From aa5e9d90f5d724e889b95e0e145460ef06323ea6 Mon Sep 17 00:00:00 2001 From: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com> Date: Mon, 7 Sep 2026 19:58:21 -0700 Subject: [PATCH 01/28] feat(devices): first-class iOS Simulator and Android Emulator support Adds a Device right-panel surface streaming simulators and emulators through a server-side expo-device-hub proxied on the environment origin, a device MCP toolkit (device_list/open/screenshot/close), and a preconfigured agent-device CLI injected into provider sessions. Co-Authored-By: Claude Fable 5 --- .../src/features/threads/thread-work-log.tsx | 12 +- apps/mobile/src/lib/threadActivity.ts | 2 +- apps/server/src/auth/RpcAuthorization.ts | 5 + apps/server/src/device/AgentDeviceShim.ts | 35 + apps/server/src/device/DeviceHost.ts | 65 ++ apps/server/src/device/DeviceHubProxy.ts | 209 ++++++ apps/server/src/device/DeviceService.ts | 475 ++++++++++++++ apps/server/src/device/DeviceToolchain.ts | 222 +++++++ apps/server/src/device/LocalDeviceHost.ts | 421 ++++++++++++ apps/server/src/mcp/McpHttpServer.ts | 183 +++++- apps/server/src/mcp/McpInvocationContext.ts | 7 +- apps/server/src/mcp/McpProviderSession.ts | 25 + .../server/src/mcp/McpSessionRegistry.test.ts | 5 + apps/server/src/mcp/McpSessionRegistry.ts | 4 +- .../src/mcp/toolkits/device/handlers.test.ts | 43 ++ .../src/mcp/toolkits/device/handlers.ts | 195 ++++++ apps/server/src/mcp/toolkits/device/tools.ts | 92 +++ .../provider/CodexDeveloperInstructions.ts | 39 +- .../src/provider/Layers/ClaudeAdapter.ts | 2 +- .../src/provider/Layers/CodexAdapter.ts | 6 +- .../provider/Layers/CodexSessionRuntime.ts | 26 +- .../src/provider/Layers/CursorAdapter.ts | 9 +- .../server/src/provider/Layers/GrokAdapter.ts | 9 +- .../provider/Layers/ProviderService.test.ts | 40 +- .../src/provider/Layers/ProviderService.ts | 77 ++- apps/server/src/server.test.ts | 16 + apps/server/src/server.ts | 10 +- apps/server/src/ws.ts | 24 + apps/web/src/components/ChatView.tsx | 37 ++ .../src/components/RightPanelTabs.test.tsx | 2 + apps/web/src/components/RightPanelTabs.tsx | 31 + .../components/chat/MessagesTimeline.logic.ts | 2 +- .../src/components/chat/MessagesTimeline.tsx | 4 + .../web/src/components/device/DevicePanel.tsx | Bin 0 -> 11038 bytes .../components/device/DeviceStreamView.tsx | 205 ++++++ .../components/device/deviceStream.test.ts | 68 ++ .../web/src/components/device/deviceStream.ts | 599 ++++++++++++++++++ .../settings/ProjectDefaultsSettings.tsx | 57 ++ .../components/settings/SettingsPanels.tsx | 5 + .../src/components/settings/settingsSearch.ts | 6 + apps/web/src/rightPanelStore.ts | 12 +- apps/web/src/routes/_chat.pull-requests.tsx | 2 + apps/web/src/state/device.ts | 68 ++ packages/client-runtime/package.json | 8 + packages/client-runtime/src/rpc/client.ts | 1 + packages/client-runtime/src/state/device.ts | 50 ++ .../src/state/deviceHubAccess.ts | 72 +++ .../src/work-log/presentation.test.ts | 13 + .../src/work-log/presentation.ts | 15 +- packages/contracts/src/device.ts | 300 +++++++++ packages/contracts/src/index.ts | 1 + packages/contracts/src/rpc.ts | 50 ++ packages/contracts/src/settings.ts | 9 + 53 files changed, 3829 insertions(+), 46 deletions(-) create mode 100644 apps/server/src/device/AgentDeviceShim.ts create mode 100644 apps/server/src/device/DeviceHost.ts create mode 100644 apps/server/src/device/DeviceHubProxy.ts create mode 100644 apps/server/src/device/DeviceService.ts create mode 100644 apps/server/src/device/DeviceToolchain.ts create mode 100644 apps/server/src/device/LocalDeviceHost.ts create mode 100644 apps/server/src/mcp/toolkits/device/handlers.test.ts create mode 100644 apps/server/src/mcp/toolkits/device/handlers.ts create mode 100644 apps/server/src/mcp/toolkits/device/tools.ts create mode 100644 apps/web/src/components/device/DevicePanel.tsx create mode 100644 apps/web/src/components/device/DeviceStreamView.tsx create mode 100644 apps/web/src/components/device/deviceStream.test.ts create mode 100644 apps/web/src/components/device/deviceStream.ts create mode 100644 apps/web/src/state/device.ts create mode 100644 packages/client-runtime/src/state/device.ts create mode 100644 packages/client-runtime/src/state/deviceHubAccess.ts create mode 100644 packages/contracts/src/device.ts diff --git a/apps/mobile/src/features/threads/thread-work-log.tsx b/apps/mobile/src/features/threads/thread-work-log.tsx index a104d2297cd9..f488e10cdd9b 100644 --- a/apps/mobile/src/features/threads/thread-work-log.tsx +++ b/apps/mobile/src/features/threads/thread-work-log.tsx @@ -76,7 +76,7 @@ export const THREAD_DISCLOSURE_TRANSITION_MS = 180; const WORK_LOG_LAYOUT_TRANSITION = LinearTransition.duration(THREAD_DISCLOSURE_TRANSITION_MS); const WORK_LOG_DETAIL_ENTER_TRANSITION = FadeIn.duration(140); const WORK_LOG_DETAIL_EXIT_TRANSITION = FadeOut.duration(120); -type WorkContentIcon = AppSymbolName | "browser" | "t3-code"; +type WorkContentIcon = AppSymbolName | "browser" | "device" | "t3-code"; function WorkLogIcon(props: { readonly icon: WorkContentIcon; @@ -92,7 +92,13 @@ function WorkLogIcon(props: { } return ( ()("DeviceHostError", { + hostId: Schema.String, + step: Schema.String, + detail: Schema.String, + cause: Schema.optional(Schema.Defect()), +}) { + override get message(): string { + return `Device host ${this.hostId} failed while ${this.step}: ${this.detail}`; + } +} + +export interface DeviceHubEndpoint { + /** Loopback origin of expo-device-hub, e.g. `http://127.0.0.1:3400`. */ + readonly origin: string; +} + +export interface AgentDeviceEndpoint { + readonly baseUrl: string; + readonly token: string; + /** Absolute path of the agent-device entry script for the provider PATH shim. */ + readonly entryPath: string; +} + +export interface DeviceHostReady { + readonly hub: DeviceHubEndpoint; + readonly agentDevice: AgentDeviceEndpoint; +} + +export interface DeviceHost { + readonly id: DeviceHostId; + readonly summary: Effect.Effect; + readonly platformAvailability: ( + platform: DevicePlatform, + ) => Effect.Effect; + /** + * Installs tools on first use and starts the helper processes. Idempotent: + * concurrent callers share one start, and a ready host returns immediately. + */ + readonly ensureReady: ( + onPhase: (phase: "installing" | "starting") => Effect.Effect, + ) => Effect.Effect; + /** Current endpoints when already running, without starting anything. */ + readonly current: Effect.Effect; + /** Stops helpers. Devices themselves keep running; the user owns those. */ + readonly stop: Effect.Effect; +} diff --git a/apps/server/src/device/DeviceHubProxy.ts b/apps/server/src/device/DeviceHubProxy.ts new file mode 100644 index 000000000000..cce7f1176af4 --- /dev/null +++ b/apps/server/src/device/DeviceHubProxy.ts @@ -0,0 +1,209 @@ +/** + * Same-origin proxy in front of expo-device-hub. + * + * The hub binds loopback and is never reachable directly: serve-sim exposes a + * shell-exec route and serve-emu's action routes are unauthenticated, so the + * only way to a device stream is through this route, which requires an + * environment session with the orchestration read scope. Reusing the T3 + * origin is also what makes remote connections work unchanged — Tailscale and + * T3 Connect already carry `/api/*` and WebSocket upgrades for the app itself. + * + * Only the routes the Device panel needs are forwarded. Anything under the + * hub's dashboard, exec, or WebRTC surface is rejected here. + */ +import { AuthOrchestrationReadScope } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; +import * as Scope from "effect/Scope"; +import * as Stream from "effect/Stream"; +import { + HttpClient, + HttpClientRequest, + HttpRouter, + HttpServerRequest, + HttpServerResponse, +} from "effect/unstable/http"; +import * as Socket from "effect/unstable/socket/Socket"; +import * as NodeSocket from "@effect/platform-node/NodeSocket"; + +import * as EnvironmentAuth from "../auth/EnvironmentAuth.ts"; +import { + failEnvironmentAuthInvalid, + failEnvironmentInternal, + failEnvironmentScopeRequired, +} from "../auth/http.ts"; +import { DEVICE_HUB_ROUTE_PREFIX, DeviceService } from "./DeviceService.ts"; + +const ALLOWED_PATHS: ReadonlyArray = [ + /^\/api\/devices$/, + /^\/vendor\/serve-sim\/api$/, + /^\/vendor\/serve-sim\/api\/screenshot$/, + /^\/vendor\/serve-sim\/helper\/[^/]+\/(stream\.mjpeg|stream\.avcc|config|health|ax|foreground)$/, + /^\/vendor\/serve-sim\/appstate$/, + /^\/vendor\/serve-emu\/api\/(devices|screenshot|stream-mode|stream-settings)$/, + /^\/vendor\/serve-emu\/health$/, +]; + +const ALLOWED_WS_PATHS: ReadonlyArray = [ + /^\/api\/devices\/ws$/, + /^\/vendor\/serve-sim\/helper\/ws$/, + /^\/vendor\/serve-emu\/ws$/, +]; + +/** Hop-by-hop and origin headers that must not cross the proxy. */ +const DROPPED_REQUEST_HEADERS = new Set([ + "host", + "connection", + "upgrade", + "sec-websocket-key", + "sec-websocket-version", + "sec-websocket-extensions", + "sec-websocket-protocol", + "cookie", + "authorization", + "dpop", + "content-length", + "accept-encoding", +]); + +const isWebSocketUpgrade = (request: HttpServerRequest.HttpServerRequest) => + request.headers.upgrade?.toLowerCase() === "websocket"; + +/** + * `` and WebSocket cannot set headers, so every proxied request + * authenticates the way the `/ws` upgrade does: a cookie for browser + * sessions, or a short-lived `wsTicket` minted over authenticated HTTP for + * bearer and DPoP clients. The upgrade authenticator already implements that + * fallback order, so it is used for plain requests as well. + */ +const authenticate = Effect.gen(function* () { + const request = yield* HttpServerRequest.HttpServerRequest; + const serverAuth = yield* EnvironmentAuth.EnvironmentAuth; + const session = yield* serverAuth.authenticateWebSocketUpgrade(request).pipe( + Effect.catchIf(EnvironmentAuth.isServerAuthCredentialError, (error) => + failEnvironmentAuthInvalid( + EnvironmentAuth.serverAuthCredentialReason(error), + EnvironmentAuth.serverAuthDpopFailureReason(error), + ), + ), + Effect.catchIf(EnvironmentAuth.isServerAuthInternalError, (error) => + failEnvironmentInternal("internal_error", error), + ), + ); + if (!session.scopes.includes(AuthOrchestrationReadScope)) { + return yield* failEnvironmentScopeRequired(AuthOrchestrationReadScope); + } +}); + +const forwardHeaders = (request: HttpServerRequest.HttpServerRequest, origin: string) => { + const headers: Record = {}; + for (const [name, value] of Object.entries(request.headers)) { + if (DROPPED_REQUEST_HEADERS.has(name) || value === undefined) continue; + headers[name] = value; + } + // serve-emu refuses mutations whose Origin differs from the request origin. + if (request.headers.origin !== undefined) headers.origin = origin; + return headers; +}; + +/** + * Pipe a client WebSocket to the hub's with no framing changes. Frames are + * opaque: H.264 access units one way, input packets the other. + */ +const proxyWebSocket = Effect.fn("DeviceHubProxy.proxyWebSocket")(function* ( + request: HttpServerRequest.HttpServerRequest, + upstreamUrl: string, +) { + const client = yield* request.upgrade; + const upstream = yield* Socket.makeWebSocket(upstreamUrl, { + openTimeout: "10 seconds", + }).pipe(Effect.provide(NodeSocket.layerWebSocketConstructor)); + yield* Effect.scoped( + Effect.gen(function* () { + const writeToClient = yield* client.writer; + const writeToUpstream = yield* upstream.writer; + const downstream = upstream.runRaw((data) => writeToClient(data)); + const upstreamPump = client.runRaw((data) => writeToUpstream(data)); + // Whichever side closes first ends the other via scope teardown. + yield* Effect.raceFirst(downstream, upstreamPump); + }), + ).pipe(Effect.catchCause(() => Effect.void)); + return HttpServerResponse.empty(); +}); + +const proxyHttp = Effect.fn("DeviceHubProxy.proxyHttp")(function* ( + request: HttpServerRequest.HttpServerRequest, + upstreamUrl: string, + hubOrigin: string, +) { + const httpClient = yield* HttpClient.HttpClient; + const scope = yield* Scope.make(); + const method = request.method; + const upstreamRequest = HttpClientRequest.make(method)(upstreamUrl).pipe( + HttpClientRequest.setHeaders(forwardHeaders(request, hubOrigin)), + method === "GET" || method === "HEAD" + ? (self) => self + : HttpClientRequest.bodyStream(request.stream), + ); + const response = yield* httpClient.execute(upstreamRequest).pipe(Scope.provide(scope)); + const headers: Record = {}; + for (const [name, value] of Object.entries(response.headers)) { + if (name === "content-encoding" || name === "transfer-encoding" || name === "connection") { + continue; + } + if (value !== undefined) headers[name] = value; + } + // Long-lived MJPEG and AVCC responses must not be buffered by compression. + headers["cache-control"] = "no-store, no-transform"; + return HttpServerResponse.stream( + response.stream.pipe(Stream.ensuring(Scope.close(scope, undefined as never))), + { + status: response.status, + headers, + ...(headers["content-type"] ? { contentType: headers["content-type"] } : {}), + }, + ); +}); + +const handler = Effect.gen(function* () { + const request = yield* HttpServerRequest.HttpServerRequest; + const url = HttpServerRequest.toURL(request); + if (Option.isNone(url)) { + return HttpServerResponse.text("Bad Request", { status: 400 }); + } + const hubPath = url.value.pathname.slice(DEVICE_HUB_ROUTE_PREFIX.length) || "/"; + const upgrade = isWebSocketUpgrade(request); + const allowed = (upgrade ? ALLOWED_WS_PATHS : ALLOWED_PATHS).some((pattern) => + pattern.test(hubPath), + ); + if (!allowed) { + return HttpServerResponse.text("Not Found", { status: 404 }); + } + yield* authenticate; + const devices = yield* DeviceService; + const ready = yield* devices.currentReadiness(); + if (!ready) { + return HttpServerResponse.text("Device hub is not running", { status: 503 }); + } + // The hub runs in standalone mode at its origin root; the panel builds every + // stream and socket URL itself, so nothing depends on the hub knowing the + // T3 prefix. + // The ticket authenticates here and must not travel on to the hub. + const upstreamSearch = new URLSearchParams(url.value.search); + upstreamSearch.delete("wsTicket"); + const search = upstreamSearch.size > 0 ? `?${upstreamSearch.toString()}` : ""; + const upstreamPath = `${hubPath}${search}`; + if (upgrade) { + return yield* proxyWebSocket( + request, + `${ready.hub.origin.replace(/^http/, "ws")}${upstreamPath}`, + ); + } + return yield* proxyHttp(request, `${ready.hub.origin}${upstreamPath}`, ready.hub.origin); +}); + +export const deviceHubProxyRouteLayer = HttpRouter.add( + "*", + `${DEVICE_HUB_ROUTE_PREFIX}/*`, + handler, +); diff --git a/apps/server/src/device/DeviceService.ts b/apps/server/src/device/DeviceService.ts new file mode 100644 index 000000000000..a0d254cd9994 --- /dev/null +++ b/apps/server/src/device/DeviceService.ts @@ -0,0 +1,475 @@ +/** + * Device discovery, per-thread device sessions, and the state stream clients + * render the Device panel from. + * + * Discovery and boot go through expo-device-hub's JSON API rather than + * shelling out to simctl and adb here: the hub already normalizes both + * platforms into one device shape and is the process that has to know a + * device is booted before it can stream it. Sessions are the server's own + * bookkeeping — which thread is looking at which device — so the panel and + * the `device_*` tools agree, and so a `device_open` from an agent surfaces in + * every connected client the way `preview_open` does. + */ +import { + type DeviceCloseInput, + type DeviceError, + type DeviceHostId, + type DeviceId, + DeviceBootError, + DeviceHostUnavailableError, + DeviceNotFoundError, + DeviceOperationError, + type DeviceOpenInput, + type DevicePlatform, + DevicePlatformUnavailableError, + type DeviceServiceState, + type DeviceSession, + type DeviceShutdownInput, + type DeviceSummary, + LOCAL_DEVICE_HOST_ID, + type ThreadId, +} from "@t3tools/contracts"; +import * as Context from "effect/Context"; +import * as DateTime from "effect/DateTime"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as PubSub from "effect/PubSub"; +import * as Schema from "effect/Schema"; +import * as Scope from "effect/Scope"; +import * as Stream from "effect/Stream"; +import * as SynchronizedRef from "effect/SynchronizedRef"; +import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"; + +import type { DeviceHost, DeviceHostReady } from "./DeviceHost.ts"; +import * as LocalDeviceHost from "./LocalDeviceHost.ts"; + +/** Origin-relative prefix the hub is proxied under. See DeviceHubProxy. */ +export const DEVICE_HUB_ROUTE_PREFIX = "/api/device-hub"; + +const BOOT_TIMEOUT = Duration.minutes(3); +const SCREENSHOT_TIMEOUT = Duration.seconds(20); + +const HubDevice = Schema.Struct({ + id: Schema.String, + name: Schema.String, + version: Schema.String, + platform: Schema.Literals(["ios", "android"]), + booted: Schema.Boolean, + physical: Schema.Boolean, +}); +const HubDeviceList = Schema.Struct({ + simulators: Schema.Array(HubDevice), + emulators: Schema.Array(HubDevice), + errors: Schema.optional(Schema.Array(Schema.Struct({ message: Schema.String }))), +}); +const HubActionResult = Schema.Struct({ + ok: Schema.Boolean, + id: Schema.optional(Schema.String), + serial: Schema.optional(Schema.String), + error: Schema.optional(Schema.String), +}); + +export interface DeviceScreenshot { + readonly device: DeviceSummary; + readonly png: Uint8Array; +} + +export interface DeviceReadiness extends DeviceHostReady { + readonly hostId: DeviceHostId; +} + +export class DeviceService extends Context.Service< + DeviceService, + { + readonly state: Effect.Effect; + readonly subscribe: Effect.Effect, never, Scope.Scope>; + /** Refreshes device discovery; starts the host helpers on first call. */ + readonly list: Effect.Effect; + readonly open: (input: DeviceOpenInput) => Effect.Effect; + readonly close: (input: DeviceCloseInput) => Effect.Effect; + readonly shutdown: (input: DeviceShutdownInput) => Effect.Effect; + readonly screenshot: (input: { + readonly hostId?: DeviceHostId | undefined; + readonly deviceId: DeviceId; + }) => Effect.Effect; + /** Host endpoints for the proxy and the provider environment. */ + readonly readiness: (hostId?: DeviceHostId) => Effect.Effect; + readonly currentReadiness: (hostId?: DeviceHostId) => Effect.Effect; + readonly sessionsForThread: (threadId: ThreadId) => Effect.Effect>; + } +>()("t3/device/DeviceService") {} + +interface ServiceState { + readonly state: DeviceServiceState; +} + +const vendorPrefix = (platform: DevicePlatform) => + platform === "ios" ? "/vendor/serve-sim" : "/vendor/serve-emu"; + +export const make = Effect.gen(function* () { + const localHost = yield* LocalDeviceHost.make(); + const hosts: ReadonlyMap = new Map([[localHost.id, localHost]]); + const httpClient = (yield* HttpClient.HttpClient).pipe(HttpClient.withScope); + const statePubSub = yield* PubSub.unbounded(); + const initialHosts = yield* Effect.forEach(hosts.values(), (host) => host.summary); + const stateRef = yield* SynchronizedRef.make({ + state: { + hosts: initialHosts, + hostStatus: "idle", + devices: [], + sessions: [], + hubBasePath: DEVICE_HUB_ROUTE_PREFIX, + revision: 0, + }, + }); + + const publish = (update: (state: DeviceServiceState) => DeviceServiceState) => + SynchronizedRef.updateAndGetEffect(stateRef, ({ state }) => { + const next = { ...update(state), revision: state.revision + 1 }; + return PubSub.publish(statePubSub, next).pipe(Effect.as({ state: next })); + }).pipe(Effect.map(({ state }) => state)); + + const resolveHost = (hostId: DeviceHostId | undefined) => + Effect.gen(function* () { + const id = hostId ?? LOCAL_DEVICE_HOST_ID; + const host = hosts.get(id); + if (!host) { + return yield* new DeviceHostUnavailableError({ hostId: id, reason: "Unknown host." }); + } + return host; + }); + + const readiness: DeviceService["Service"]["readiness"] = Effect.fn("DeviceService.readiness")( + function* (hostId) { + const host = yield* resolveHost(hostId); + const ready = yield* host + .ensureReady((phase) => + publish((state) => ({ ...state, hostStatus: phase, hostStatusDetail: undefined })).pipe( + Effect.asVoid, + ), + ) + .pipe( + Effect.tapError((error) => + publish((state) => ({ + ...state, + hostStatus: "failed", + hostStatusDetail: error.message, + })), + ), + Effect.mapError( + (error) => new DeviceHostUnavailableError({ hostId: host.id, reason: error.message }), + ), + ); + yield* SynchronizedRef.get(stateRef).pipe( + Effect.flatMap(({ state }) => + state.hostStatus === "ready" + ? Effect.void + : publish((current) => ({ + ...current, + hostStatus: "ready", + hostStatusDetail: undefined, + })), + ), + ); + return { hostId: host.id, ...ready }; + }, + ); + + const currentReadiness: DeviceService["Service"]["currentReadiness"] = (hostId) => + resolveHost(hostId).pipe( + Effect.flatMap((host) => + host.current.pipe(Effect.map((ready) => (ready ? { hostId: host.id, ...ready } : null))), + ), + Effect.orElseSucceed(() => null), + ); + + const hubJson = ( + ready: DeviceReadiness, + request: HttpClientRequest.HttpClientRequest, + schema: Schema.Codec, + operation: string, + timeout: Duration.Input = Duration.seconds(15), + ) => + httpClient.execute(request).pipe( + Effect.flatMap(HttpClientResponse.filterStatusOk), + Effect.flatMap(HttpClientResponse.schemaBodyJson(schema)), + Effect.scoped, + Effect.timeout(timeout), + Effect.mapError( + (cause) => + new DeviceOperationError({ + operation, + detail: `${ready.hub.origin}: ${cause instanceof Error ? cause.message : String(cause)}`, + }), + ), + ); + + const fetchDevices = Effect.fn("DeviceService.fetchDevices")(function* (ready: DeviceReadiness) { + const list = yield* hubJson( + ready, + HttpClientRequest.get(`${ready.hub.origin}/api/devices`), + HubDeviceList, + "list", + ); + const toSummary = (device: typeof HubDevice.Type): DeviceSummary => ({ + hostId: ready.hostId, + id: device.id, + platform: device.platform, + name: device.name, + version: device.version, + booted: device.booted, + physical: device.physical, + }); + return [...list.simulators, ...list.emulators].map(toSummary); + }); + + const refresh = Effect.fn("DeviceService.refresh")(function* (ready: DeviceReadiness) { + const devices = yield* fetchDevices(ready); + const hostSummaries = yield* Effect.forEach(hosts.values(), (host) => host.summary); + return yield* publish((state) => ({ ...state, hosts: hostSummaries, devices })); + }); + + const list: DeviceService["Service"]["list"] = Effect.gen(function* () { + const ready = yield* readiness(); + return yield* refresh(ready); + }).pipe(Effect.withSpan("DeviceService.list")); + + const findDevice = ( + state: DeviceServiceState, + hostId: DeviceHostId, + deviceId: DeviceId, + ): DeviceSummary | undefined => + state.devices.find((device) => device.hostId === hostId && device.id === deviceId); + + const ensurePlatform = Effect.fn("DeviceService.ensurePlatform")(function* ( + host: DeviceHost, + platform: DevicePlatform, + ) { + const availability = yield* host.platformAvailability(platform); + if (!availability.available) { + return yield* new DevicePlatformUnavailableError({ + hostId: host.id, + platform, + reason: availability.reason ?? "Platform toolchain missing.", + }); + } + }); + + /** + * Boot through the hub so its device list and the streaming helper both see + * the device come up. Android AVDs change id when they boot (AVD name to + * emulator serial), so the returned id is authoritative. + */ + const boot = Effect.fn("DeviceService.boot")(function* ( + ready: DeviceReadiness, + device: DeviceSummary, + ) { + const result = yield* HttpClientRequest.post(`${ready.hub.origin}/api/devices/boot`).pipe( + HttpClientRequest.bodyJson({ platform: device.platform, id: device.id, name: device.name }), + Effect.mapError( + (cause) => new DeviceOperationError({ operation: "boot", detail: String(cause) }), + ), + Effect.flatMap((request) => hubJson(ready, request, HubActionResult, "boot", BOOT_TIMEOUT)), + ); + if (!result.ok) { + return yield* new DeviceBootError({ + hostId: ready.hostId, + deviceId: device.id, + detail: result.error ?? "The device hub reported a boot failure.", + }); + } + if (device.platform === "ios") { + // Booting alone does not attach a serve-sim helper; the grid start + // does both and is idempotent for a booted simulator. + yield* HttpClientRequest.post( + `${ready.hub.origin}${vendorPrefix("ios")}/grid/api/start`, + ).pipe( + HttpClientRequest.bodyJson({ udid: device.id }), + Effect.mapError( + (cause) => new DeviceOperationError({ operation: "boot", detail: String(cause) }), + ), + Effect.flatMap((request) => + hubJson(ready, request, HubActionResult, "attach stream", BOOT_TIMEOUT), + ), + ); + } + return result.serial ?? result.id ?? device.id; + }); + + const open: DeviceService["Service"]["open"] = Effect.fn("DeviceService.open")(function* (input) { + const host = yield* resolveHost(input.hostId); + yield* ensurePlatform(host, input.platform); + const ready = yield* readiness(host.id); + let state = yield* refresh(ready); + let device = findDevice(state, host.id, input.deviceId); + if (!device) { + return yield* new DeviceNotFoundError({ hostId: host.id, deviceId: input.deviceId }); + } + if (!device.booted && input.boot !== false) { + const bootedId = yield* boot(ready, device); + state = yield* refresh(ready); + device = findDevice(state, host.id, bootedId) ?? findDevice(state, host.id, device.id); + if (!device) { + return yield* new DeviceNotFoundError({ hostId: host.id, deviceId: bootedId }); + } + } else if (device.platform === "ios" && device.booted) { + // A simulator booted outside T3 has no helper attached yet. + yield* HttpClientRequest.post( + `${ready.hub.origin}${vendorPrefix("ios")}/grid/api/start`, + ).pipe( + HttpClientRequest.bodyJson({ udid: device.id }), + Effect.mapError( + (cause) => new DeviceOperationError({ operation: "open", detail: String(cause) }), + ), + Effect.flatMap((request) => + hubJson(ready, request, HubActionResult, "attach stream", BOOT_TIMEOUT), + ), + ); + } + const openedAt = DateTime.formatIso(yield* DateTime.now); + const session: DeviceSession = { + threadId: input.threadId, + hostId: host.id, + deviceId: device.id, + platform: device.platform, + openedAt, + }; + yield* publish((current) => ({ + ...current, + sessions: [ + ...current.sessions.filter( + (existing) => + !( + existing.threadId === session.threadId && + existing.hostId === session.hostId && + existing.deviceId === session.deviceId + ), + ), + session, + ], + })); + return session; + }); + + const shutdownDevice = Effect.fn("DeviceService.shutdownDevice")(function* ( + hostId: DeviceHostId, + deviceId: DeviceId, + platform: DevicePlatform, + ) { + const ready = yield* readiness(hostId); + yield* HttpClientRequest.post(`${ready.hub.origin}/api/devices/shutdown`).pipe( + HttpClientRequest.bodyJson({ platform, id: deviceId }), + Effect.mapError( + (cause) => new DeviceOperationError({ operation: "shutdown", detail: String(cause) }), + ), + Effect.flatMap((request) => hubJson(ready, request, HubActionResult, "shutdown")), + Effect.flatMap((result) => + result.ok + ? Effect.void + : Effect.fail( + new DeviceOperationError({ + operation: "shutdown", + detail: result.error ?? "The device hub reported a shutdown failure.", + }), + ), + ), + ); + yield* refresh(ready); + }); + + const close: DeviceService["Service"]["close"] = Effect.fn("DeviceService.close")( + function* (input) { + const { state } = yield* SynchronizedRef.get(stateRef); + const closing = state.sessions.filter( + (session) => + session.threadId === input.threadId && + (input.deviceId === undefined || session.deviceId === input.deviceId), + ); + if (closing.length === 0) return; + yield* publish((current) => ({ + ...current, + sessions: current.sessions.filter((session) => !closing.includes(session)), + })); + if (input.shutdown) { + yield* Effect.forEach( + closing, + (session) => shutdownDevice(session.hostId, session.deviceId, session.platform), + { discard: true }, + ); + } + }, + ); + + const shutdown: DeviceService["Service"]["shutdown"] = Effect.fn("DeviceService.shutdown")( + function* (input) { + const host = yield* resolveHost(input.hostId); + yield* shutdownDevice(host.id, input.deviceId, input.platform); + // Sessions on a powered-off device are stale in every thread. + yield* publish((current) => ({ + ...current, + sessions: current.sessions.filter( + (session) => !(session.hostId === host.id && session.deviceId === input.deviceId), + ), + })); + }, + ); + + const screenshot: DeviceService["Service"]["screenshot"] = Effect.fn("DeviceService.screenshot")( + function* (input) { + const host = yield* resolveHost(input.hostId); + const ready = yield* readiness(host.id); + const { state } = yield* SynchronizedRef.get(stateRef); + const device = findDevice(state, host.id, input.deviceId); + if (!device) { + return yield* new DeviceNotFoundError({ hostId: host.id, deviceId: input.deviceId }); + } + const url = `${ready.hub.origin}${vendorPrefix(device.platform)}/api/screenshot?device=${encodeURIComponent(device.id)}`; + const png = yield* httpClient.execute(HttpClientRequest.post(url)).pipe( + Effect.flatMap(HttpClientResponse.filterStatusOk), + Effect.flatMap((response) => response.arrayBuffer), + Effect.map((buffer) => new Uint8Array(buffer)), + Effect.scoped, + Effect.timeout(SCREENSHOT_TIMEOUT), + Effect.mapError( + (cause) => + new DeviceOperationError({ + operation: "screenshot", + detail: cause instanceof Error ? cause.message : String(cause), + }), + ), + ); + return { device, png }; + }, + ); + + const sessionsForThread: DeviceService["Service"]["sessionsForThread"] = (threadId) => + SynchronizedRef.get(stateRef).pipe( + Effect.map(({ state }) => state.sessions.filter((session) => session.threadId === threadId)), + ); + + return DeviceService.of({ + state: SynchronizedRef.get(stateRef).pipe(Effect.map(({ state }) => state)), + subscribe: PubSub.subscribe(statePubSub), + list, + open, + close, + shutdown, + screenshot, + readiness, + currentReadiness, + sessionsForThread, + }); +}).pipe(Effect.withSpan("DeviceService.make")); + +export const layer = Layer.effect(DeviceService, make); + +/** State stream for WS subscribers: current snapshot first, then every change. */ +export const stateStream = (service: DeviceService["Service"]): Stream.Stream => + Stream.unwrap( + Effect.gen(function* () { + const initial = yield* service.state; + const subscription = yield* service.subscribe; + return Stream.concat(Stream.make(initial), Stream.fromSubscription(subscription)); + }), + ); diff --git a/apps/server/src/device/DeviceToolchain.ts b/apps/server/src/device/DeviceToolchain.ts new file mode 100644 index 000000000000..b8959ba1dc5b --- /dev/null +++ b/apps/server/src/device/DeviceToolchain.ts @@ -0,0 +1,222 @@ +/** + * Pinned installs of the two external tools device support is built on. + * + * `expo-device-hub` streams simulator and emulator screens and `agent-device` + * drives them. Both are npm-installed once into `/device/tools/` + * and executed from there with the server's own Node, never `npx`: an ephemeral + * npx cache would make every first `device_open` after a reboot depend on the + * registry, and the pinned versions are part of the contract the injected + * agent instructions describe. + * + * Install follows the pinned-runtime recipe: stage into a temp sibling, write a + * sentinel only after npm exits 0, then rename into place. npm extracts files + * before it finishes, so an entry file alone does not prove a usable tree. + */ +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Option from "effect/Option"; +import * as Path from "effect/Path"; +import * as PlatformError from "effect/PlatformError"; +import * as Schema from "effect/Schema"; +import * as Semaphore from "effect/Semaphore"; + +import * as ProcessRunner from "../processRunner.ts"; + +export const DEVICE_HUB_PACKAGE = "expo-device-hub"; +export const DEVICE_HUB_VERSION = "0.9.0"; +export const AGENT_DEVICE_PACKAGE = "agent-device"; +export const AGENT_DEVICE_VERSION = "0.20.10"; + +const DEVICE_TOOLS_DIR = "device"; +const INSTALL_TIMEOUT = Duration.minutes(10); +const installLock = Semaphore.makeUnsafe(1); + +export interface DeviceToolPaths { + readonly installDir: string; + /** Absolute path of the tool's entry script, run with the server's Node. */ + readonly entryPath: string; + readonly sentinelPath: string; +} + +export interface DeviceToolchainPaths { + readonly hub: DeviceToolPaths; + readonly agentDevice: DeviceToolPaths; +} + +export class DeviceToolchainInstallError extends Schema.TaggedError()( + "DeviceToolchainInstallError", + { + tool: Schema.String, + step: Schema.String, + exitCode: Schema.optional(Schema.Number), + stderrTail: Schema.optional(Schema.String), + cause: Schema.optional(Schema.Defect()), + }, +) { + override get message(): string { + const suffix = this.exitCode === undefined ? "" : ` (exit code ${this.exitCode})`; + return `Installing ${this.tool} failed while ${this.step}${suffix}.`; + } +} + +interface ToolSpec { + readonly name: string; + readonly version: string; + readonly entry: ReadonlyArray; +} + +const HUB_SPEC: ToolSpec = { + name: DEVICE_HUB_PACKAGE, + version: DEVICE_HUB_VERSION, + entry: ["dist", "server", "cli.mjs"], +}; + +const AGENT_DEVICE_SPEC: ToolSpec = { + name: AGENT_DEVICE_PACKAGE, + version: AGENT_DEVICE_VERSION, + entry: ["bin", "agent-device.mjs"], +}; + +const toolPaths = (path: Path.Path, baseDir: string, spec: ToolSpec): DeviceToolPaths => { + const installDir = path.join(baseDir, DEVICE_TOOLS_DIR, "tools", `${spec.name}@${spec.version}`); + return { + installDir, + entryPath: path.join(installDir, "node_modules", spec.name, ...spec.entry), + sentinelPath: path.join(installDir, ".install-complete"), + }; +}; + +export const deviceToolchainPaths = (path: Path.Path, baseDir: string): DeviceToolchainPaths => ({ + hub: toolPaths(path, baseDir, HUB_SPEC), + agentDevice: toolPaths(path, baseDir, AGENT_DEVICE_SPEC), +}); + +/** The agent-device daemon state (daemon.json, sessions) lives beside the tools. */ +export const agentDeviceStateDir = (path: Path.Path, stateDir: string): string => + path.join(stateDir, DEVICE_TOOLS_DIR, "agent-device"); + +const isInstalled = Effect.fn("DeviceToolchain.isInstalled")(function* ( + fs: FileSystem.FileSystem, + paths: DeviceToolPaths, + version: string, +) { + const [entryExists, sentinel] = yield* Effect.all([ + fs.exists(paths.entryPath), + fs.readFileString(paths.sentinelPath).pipe(Effect.option), + ]).pipe(Effect.orElseSucceed(() => [false, Option.none()] as const)); + return entryExists && Option.isSome(sentinel) && sentinel.value.trim() === version; +}); + +const installTool = Effect.fn("DeviceToolchain.installTool")(function* ( + spec: ToolSpec, + paths: DeviceToolPaths, +) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const runner = yield* ProcessRunner.ProcessRunner; + const fail = (step: string) => (cause: unknown) => + new DeviceToolchainInstallError({ tool: spec.name, step, cause }); + + if (yield* isInstalled(fs, paths, spec.version)) return paths; + + const parentDir = path.dirname(paths.installDir); + yield* fs + .remove(paths.installDir, { recursive: true, force: true }) + .pipe(Effect.mapError(fail("removing an incomplete install"))); + yield* fs + .makeDirectory(parentDir, { recursive: true }) + .pipe(Effect.mapError(fail("preparing the install directory"))); + const stagingDir = yield* fs + .makeTempDirectory({ directory: parentDir, prefix: ".staging-" }) + .pipe(Effect.mapError(fail("preparing the install directory"))); + + return yield* Effect.gen(function* () { + const installArgs = [ + "install", + "--prefix", + stagingDir, + "--no-fund", + "--no-audit", + `${spec.name}@${spec.version}`, + ]; + const result = yield* runner + .run({ command: "npm", args: installArgs, timeout: INSTALL_TIMEOUT }) + .pipe( + Effect.catchTags({ + ProcessSpawnError: (error) => + error.cause instanceof PlatformError.PlatformError && + error.cause.reason._tag === "NotFound" + ? runner.run({ + command: "pnpm", + args: ["--package=npm@11", "dlx", "npm", ...installArgs], + timeout: INSTALL_TIMEOUT, + }) + : Effect.fail(error), + }), + Effect.mapError(fail("running npm install")), + ); + if (result.code !== 0) { + return yield* new DeviceToolchainInstallError({ + tool: spec.name, + step: "running npm install", + exitCode: Number(result.code), + stderrTail: result.stderr.slice(-2_000), + }); + } + const stagedEntry = path.join(stagingDir, "node_modules", spec.name, ...spec.entry); + if (!(yield* fs.exists(stagedEntry).pipe(Effect.orElseSucceed(() => false)))) { + return yield* new DeviceToolchainInstallError({ + tool: spec.name, + step: "verifying the installed entry point", + }); + } + yield* fs + .writeFileString(path.join(stagingDir, ".install-complete"), `${spec.version}\n`) + .pipe(Effect.mapError(fail("recording the completed install"))); + yield* fs.rename(stagingDir, paths.installDir).pipe( + Effect.catch((cause) => + // A concurrent server may have published the same version first. + isInstalled(fs, paths, spec.version).pipe( + Effect.flatMap((published) => + published ? Effect.void : Effect.fail(fail("publishing the install")(cause)), + ), + ), + ), + ); + return paths; + }).pipe( + Effect.ensuring(fs.remove(stagingDir, { recursive: true, force: true }).pipe(Effect.ignore)), + ); +}); + +/** + * Installs whichever of the two tools is missing and returns their paths. + * Serialized process-wide so two threads opening devices at once do not race + * npm against the same directory. + */ +export const ensureDeviceToolchain = Effect.fn("DeviceToolchain.ensure")(function* ( + baseDir: string, +) { + const path = yield* Path.Path; + const paths = deviceToolchainPaths(path, baseDir); + return yield* installLock.withPermit( + Effect.all( + [installTool(HUB_SPEC, paths.hub), installTool(AGENT_DEVICE_SPEC, paths.agentDevice)], + { concurrency: 1 }, + ).pipe(Effect.as(paths)), + ); +}); + +export const isDeviceToolchainInstalled = Effect.fn("DeviceToolchain.isInstalled")(function* ( + baseDir: string, +) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const paths = deviceToolchainPaths(path, baseDir); + const [hub, agentDevice] = yield* Effect.all([ + isInstalled(fs, paths.hub, DEVICE_HUB_VERSION), + isInstalled(fs, paths.agentDevice, AGENT_DEVICE_VERSION), + ]); + return hub && agentDevice; +}); diff --git a/apps/server/src/device/LocalDeviceHost.ts b/apps/server/src/device/LocalDeviceHost.ts new file mode 100644 index 000000000000..6a6a7c610473 --- /dev/null +++ b/apps/server/src/device/LocalDeviceHost.ts @@ -0,0 +1,421 @@ +/** + * The device host that is this machine. + * + * Runs expo-device-hub as a supervised child on a loopback port and starts the + * agent-device daemon in HTTP mode under a T3-owned state directory. Both are + * lazy: nothing is installed or spawned until a device is first listed with + * the intent to open one, so a server that never touches simulators pays + * nothing. + * + * The hub runs in its standalone mode (origin root). The T3 proxy strips its + * own prefix, and the Device panel derives stream and socket URLs from the + * prefix itself rather than from anything the hub prints. + */ +import { + type DeviceHostSummary, + type DevicePlatform, + type DevicePlatformAvailability, + LOCAL_DEVICE_HOST_ID, +} from "@t3tools/contracts"; +import { waitForHttpReady } from "@t3tools/shared/httpReadiness"; +import { HostProcessEnvironment, HostProcessPlatform } from "@t3tools/shared/hostProcess"; +import * as NetService from "@t3tools/shared/Net"; +import { isCommandAvailable } from "@t3tools/shared/shell"; +import * as Clock from "effect/Clock"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; +import * as Ref from "effect/Ref"; +import * as Schema from "effect/Schema"; +import * as Scope from "effect/Scope"; +import * as Semaphore from "effect/Semaphore"; +import * as Stream from "effect/Stream"; +import { HttpClient } from "effect/unstable/http"; +import * as ChildProcess from "effect/unstable/process/ChildProcess"; +import * as ChildProcessSpawner from "effect/unstable/process/ChildProcessSpawner"; + +import * as ServerConfig from "../config.ts"; +import * as ProcessRunner from "../processRunner.ts"; +import { + type AgentDeviceEndpoint, + type DeviceHost, + DeviceHostError, + type DeviceHostReady, + type DeviceHubEndpoint, +} from "./DeviceHost.ts"; +import { + agentDeviceStateDir, + type DeviceToolchainPaths, + ensureDeviceToolchain, + isDeviceToolchainInstalled, +} from "./DeviceToolchain.ts"; + +const HUB_READY_TIMEOUT_MS = 30_000; +const DAEMON_READY_TIMEOUT_MS = 30_000; +const DAEMON_POLL_MS = 100; +const HUB_RESTART_STABLE_UPTIME_MS = 60_000; +const HUB_RESTART_MAX_DELAY_MS = 30_000; + +const AgentDeviceDaemonFile = Schema.Struct({ + httpPort: Schema.Int, + token: Schema.String, + pid: Schema.optional(Schema.Int), +}); +const decodeDaemonFile = Schema.decodeUnknownEffect(Schema.fromJsonString(AgentDeviceDaemonFile)); + +interface HubProcess { + readonly child: ChildProcessSpawner.ChildProcessHandle; + readonly scope: Scope.Closeable; + readonly origin: string; + readonly startedAtMillis: number; +} + +interface RunningHost { + readonly hub: HubProcess; + readonly agentDevice: AgentDeviceEndpoint; +} + +const platformReason = Effect.fn("LocalDeviceHost.platformReason")(function* ( + platform: DevicePlatform, +): Effect.fn.Return { + const hostPlatform = yield* HostProcessPlatform; + const environment = yield* HostProcessEnvironment; + if (platform === "ios") { + if (hostPlatform !== "darwin") return "iOS Simulators need macOS with Xcode."; + if (!(yield* isCommandAvailable("xcrun"))) return "Xcode command line tools were not found."; + return null; + } + const sdkRoot = environment.ANDROID_HOME?.trim() || environment.ANDROID_SDK_ROOT?.trim(); + if (!sdkRoot && !(yield* isCommandAvailable("adb"))) { + return "Android SDK was not found. Set ANDROID_HOME or put adb on PATH."; + } + return null; +}); + +export const make = Effect.fn("LocalDeviceHost.make")(function* () { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const config = yield* ServerConfig.ServerConfig; + const path = yield* Path.Path; + const fs = yield* FileSystem.FileSystem; + const net = yield* NetService.NetService; + const runner = yield* ProcessRunner.ProcessRunner; + const httpClient = yield* HttpClient.HttpClient; + const hostEnvironment = yield* HostProcessEnvironment; + const startLock = yield* Semaphore.make(1); + const runningRef = yield* Ref.make(null); + const restartDelayRef = yield* Ref.make(0); + const hostId = LOCAL_DEVICE_HOST_ID; + + const platformAvailability = Effect.fn("LocalDeviceHost.platformAvailability")(function* ( + platform: DevicePlatform, + ): Effect.fn.Return { + const reason = yield* platformReason(platform).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Path.Path, path), + ); + return reason === null ? { platform, available: true } : { platform, available: false, reason }; + }); + + const summary: Effect.Effect = Effect.gen(function* () { + const platforms = yield* Effect.all([ + platformAvailability("ios"), + platformAvailability("android"), + ]); + return { id: hostId, kind: "local", label: "This machine", platforms }; + }); + + const hubEnvironment = (): NodeJS.ProcessEnv => ({ + ...hostEnvironment, + FORCE_COLOR: "0", + NO_COLOR: "1", + }); + + const stopHub = (hub: HubProcess | undefined) => + hub ? Scope.close(hub.scope, Exit.void).pipe(Effect.ignore) : Effect.void; + + const spawnHub = Effect.fn("LocalDeviceHost.spawnHub")(function* ( + tools: DeviceToolchainPaths, + ): Effect.fn.Return { + const port = yield* net.reserveLoopbackPort("127.0.0.1").pipe( + Effect.mapError( + (cause) => + new DeviceHostError({ + hostId, + step: "reserving a port for the device hub", + detail: cause.message, + cause, + }), + ), + ); + const origin = `http://127.0.0.1:${port}`; + const scope = yield* Scope.make("sequential"); + const child = yield* spawner + .spawn( + ChildProcess.make( + process.execPath, + [ + tools.hub.entryPath, + "--port", + String(port), + "--host", + "127.0.0.1", + "--hide-sidebar", + "--hide-boot-device", + ], + { + detached: false, + shell: false, + stdout: "pipe", + stderr: "pipe", + env: hubEnvironment(), + }, + ), + ) + .pipe( + Effect.provideService(Scope.Scope, scope), + Effect.mapError( + (cause) => + new DeviceHostError({ + hostId, + step: "starting the device hub", + detail: String(cause), + cause, + }), + ), + ); + const startedAtMillis = yield* Clock.currentTimeMillis; + const hub: HubProcess = { child, scope, origin, startedAtMillis }; + yield* Effect.forkIn(observeHubOutput(hub), scope); + yield* waitForHttpReady({ + baseUrl: origin, + path: "/readyz", + timeoutMs: HUB_READY_TIMEOUT_MS, + makeError: (info) => + new DeviceHostError({ + hostId, + step: "waiting for the device hub to answer", + detail: `No response from ${info.requestUrl} after ${info.attempt} attempts.`, + cause: info.cause, + }), + }).pipe( + Effect.provideService(HttpClient.HttpClient, httpClient), + Effect.tapError(() => stopHub(hub)), + ); + yield* Effect.logInfo("Device hub started", { pid: Number(child.pid), port }); + return hub; + }); + + const observeHubOutput = (hub: HubProcess) => + hub.child.all.pipe( + Stream.decodeText(), + Stream.splitLines, + Stream.map((line) => line.trim()), + Stream.filter((line) => line.length > 0), + Stream.runForEach((line) => + Effect.logDebug("Device hub output", { pid: Number(hub.child.pid), output: line }), + ), + Effect.catchCause(() => Effect.void), + ); + + /** + * Restart the hub when it dies under us, with the same doubling backoff the + * relay connector uses so a hub that crashes on boot cannot spin. + */ + const superviseHub = (hub: HubProcess, tools: DeviceToolchainPaths): Effect.Effect => + Effect.gen(function* () { + yield* Effect.result(hub.child.exitCode); + const running = yield* Ref.get(runningRef); + if (running?.hub.child.pid !== hub.child.pid) return; + const uptime = (yield* Clock.currentTimeMillis) - hub.startedAtMillis; + const delay = yield* Ref.modify(restartDelayRef, (current) => { + if (uptime >= HUB_RESTART_STABLE_UPTIME_MS) return [0, 0]; + const next = current === 0 ? 1_000 : Math.min(current * 2, HUB_RESTART_MAX_DELAY_MS); + return [current, next]; + }); + yield* Effect.logWarning("Device hub exited; restarting", { + pid: Number(hub.child.pid), + delayMs: delay, + }); + yield* Effect.sleep(Duration.millis(delay)); + yield* startLock.withPermits(1)( + Effect.gen(function* () { + const current = yield* Ref.get(runningRef); + if (current?.hub.child.pid !== hub.child.pid) return; + const replacement = yield* spawnHub(tools); + yield* Ref.set(runningRef, { ...current, hub: replacement }); + yield* Effect.forkDetach(superviseHub(replacement, tools)); + }), + ); + }).pipe( + Effect.catchCause((cause) => Effect.logWarning("Device hub supervisor failed", { cause })), + ); + + const daemonFilePath = () => path.join(agentDeviceStateDir(path, config.stateDir), "daemon.json"); + + const readDaemonFile = Effect.fn("LocalDeviceHost.readDaemonFile")(function* () { + const raw = yield* fs.readFileString(daemonFilePath()); + return yield* decodeDaemonFile(raw); + }); + + /** + * agent-device auto-starts its daemon on any command. A trivial `devices` + * call in HTTP mode is the documented way to bring it up; its output is the + * daemon.json this reads back. + */ + const startAgentDeviceDaemon = Effect.fn("LocalDeviceHost.startAgentDeviceDaemon")(function* ( + tools: DeviceToolchainPaths, + ): Effect.fn.Return { + const stateDir = agentDeviceStateDir(path, config.stateDir); + yield* fs.makeDirectory(stateDir, { recursive: true }).pipe(Effect.ignore); + const existing = yield* readDaemonFile().pipe(Effect.option); + const daemonEnvironment: NodeJS.ProcessEnv = { + ...hostEnvironment, + AGENT_DEVICE_STATE_DIR: stateDir, + AGENT_DEVICE_DAEMON_SERVER_MODE: "http", + // The daemon idles out after five minutes by default; the server owns + // its lifetime here and stops it explicitly. + AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: "0", + AGENT_DEVICE_NO_UPDATE_NOTIFIER: "1", + FORCE_COLOR: "0", + NO_COLOR: "1", + }; + const toEndpoint = (file: typeof AgentDeviceDaemonFile.Type): AgentDeviceEndpoint => ({ + baseUrl: `http://127.0.0.1:${file.httpPort}`, + token: file.token, + entryPath: tools.agentDevice.entryPath, + }); + if (existing._tag === "Some") { + const alive = yield* httpClient + .get(`http://127.0.0.1:${existing.value.httpPort}/health`) + .pipe( + Effect.timeout(Duration.seconds(2)), + Effect.map((response) => response.status === 200), + Effect.orElseSucceed(() => false), + ); + if (alive) return toEndpoint(existing.value); + yield* fs.remove(daemonFilePath(), { force: true }).pipe(Effect.ignore); + } + // There is no `daemon start`; the first command in a state dir spawns the + // daemon and blocks until it answers. `devices` is the cheapest one. + yield* runner + .run({ + command: process.execPath, + args: [tools.agentDevice.entryPath, "devices", "--json"], + env: daemonEnvironment, + timeout: Duration.millis(DAEMON_READY_TIMEOUT_MS), + timeoutBehavior: "timedOutResult", + }) + .pipe(Effect.ignore); + const deadline = (yield* Clock.currentTimeMillis) + DAEMON_READY_TIMEOUT_MS; + while (true) { + const file = yield* readDaemonFile().pipe(Effect.option); + if (file._tag === "Some") return toEndpoint(file.value); + if ((yield* Clock.currentTimeMillis) > deadline) { + return yield* new DeviceHostError({ + hostId, + step: "starting the agent-device daemon", + detail: `daemon.json did not appear in ${stateDir}.`, + }); + } + yield* Effect.sleep(Duration.millis(DAEMON_POLL_MS)); + } + }); + + const stopAgentDeviceDaemon = (tools: DeviceToolchainPaths | null) => + tools + ? runner + .run({ + command: process.execPath, + args: [ + tools.agentDevice.entryPath, + "daemon", + "stop", + "--state-dir", + agentDeviceStateDir(path, config.stateDir), + ], + env: { ...hostEnvironment, AGENT_DEVICE_NO_UPDATE_NOTIFIER: "1" }, + timeout: Duration.seconds(10), + timeoutBehavior: "timedOutResult", + }) + .pipe(Effect.ignore) + : Effect.void; + + let toolsRef: DeviceToolchainPaths | null = null; + + const ensureReady: DeviceHost["ensureReady"] = (onPhase) => + startLock.withPermits(1)( + Effect.gen(function* (): Generator, DeviceHostReady> { + const running = yield* Ref.get(runningRef); + if (running) { + const alive = yield* running.hub.child.isRunning.pipe(Effect.orElseSucceed(() => false)); + if (alive) return toReady(running); + yield* Ref.set(runningRef, null); + } + const installed = yield* isDeviceToolchainInstalled(config.baseDir).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Path.Path, path), + ); + if (!installed) yield* onPhase("installing"); + const tools = yield* ensureDeviceToolchain(config.baseDir).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Path.Path, path), + Effect.provideService(ProcessRunner.ProcessRunner, runner), + Effect.mapError( + (cause) => + new DeviceHostError({ + hostId, + step: `installing ${cause.tool}`, + detail: cause.message, + cause, + }), + ), + ); + toolsRef = tools; + yield* onPhase("starting"); + const hub = yield* spawnHub(tools); + const agentDevice = yield* startAgentDeviceDaemon(tools).pipe( + Effect.tapError(() => stopHub(hub)), + ); + const next: RunningHost = { hub, agentDevice }; + yield* Ref.set(runningRef, next); + yield* Ref.set(restartDelayRef, 0); + yield* Effect.forkDetach(superviseHub(hub, tools)); + return toReady(next); + }), + ); + + const toReady = (running: RunningHost): DeviceHostReady => ({ + hub: { origin: running.hub.origin } satisfies DeviceHubEndpoint, + agentDevice: running.agentDevice, + }); + + const current: DeviceHost["current"] = Ref.get(runningRef).pipe( + Effect.map((running) => (running ? toReady(running) : null)), + ); + + const stop: DeviceHost["stop"] = startLock.withPermits(1)( + Effect.gen(function* () { + const running = yield* Ref.getAndSet(runningRef, null); + yield* stopHub(running?.hub); + yield* stopAgentDeviceDaemon(toolsRef); + }), + ); + + // Never leave the hub or daemon behind when the server's scope closes. + yield* Effect.addFinalizer(() => stop); + + const host: DeviceHost = { + id: hostId, + summary, + platformAvailability, + ensureReady, + current, + stop, + }; + return host; +}); + +/** Exposed for tests. */ +export const __testing = { AgentDeviceDaemonFile }; diff --git a/apps/server/src/mcp/McpHttpServer.ts b/apps/server/src/mcp/McpHttpServer.ts index 3f3e48ebe4b2..965426b2139a 100644 --- a/apps/server/src/mcp/McpHttpServer.ts +++ b/apps/server/src/mcp/McpHttpServer.ts @@ -16,6 +16,7 @@ import { HttpRouter, HttpServerRequest, HttpServerResponse } from "effect/unstab import packageJson from "../../package.json" with { type: "json" }; import * as ServerConfig from "../config.ts"; +import * as DeviceService from "../device/DeviceService.ts"; import * as McpInvocationContext from "./McpInvocationContext.ts"; import * as McpSessionRegistry from "./McpSessionRegistry.ts"; import * as PreviewAutomationBroker from "./PreviewAutomationBroker.ts"; @@ -28,6 +29,15 @@ import { PreviewSnapshotToolkit, PreviewStandardToolkit, } from "./toolkits/preview/tools.ts"; +import { + DeviceScreenshotToolkitHandlersLive, + DeviceStandardToolkitHandlersLive, +} from "./toolkits/device/handlers.ts"; +import { + DeviceScreenshotTool, + DeviceScreenshotToolkit, + DeviceStandardToolkit, +} from "./toolkits/device/tools.ts"; const unauthorized = HttpServerResponse.jsonUnsafe( { @@ -323,16 +333,142 @@ const previewSnapshotFailure = (cause: Cause.Cause) => { }, // Agents usually see only the text content, so name the tag there too. content: [{ type: "text", text: `Preview snapshot failed: ${errorTag}.` }], +interface ImageToolResult { + readonly screenshot: { + readonly mimeType: "image/png"; + readonly data: string; + readonly width: number; + readonly height: number; + }; + readonly [key: string]: unknown; +} + +/** + * Failures surface only their tag: the remote message may carry renderer or + * device output the agent should not see, and the tag is what it can act on. + */ +const imageToolFailure = + (toolName: string, operation: string, failureText: string) => + (cause: Cause.Cause) => { + if (Cause.hasInterrupts(cause) || cause.reasons.some(Cause.isDieReason)) { + return Effect.failCause(cause).pipe(Effect.orDie); + } + const failures = cause.reasons.filter(Cause.isFailReason); + const firstFailure = failures[0]?.error; + const errorTag = + typeof firstFailure === "object" && + firstFailure !== null && + "_tag" in firstFailure && + typeof firstFailure._tag === "string" + ? firstFailure._tag + : `${toolName}Error`; + const result = new McpSchema.CallToolResult({ + isError: true, + structuredContent: { + error: { + _tag: errorTag, + operation, + failureCount: failures.length, + }, + }, + content: [{ type: "text", text: failureText }], + }); + return Effect.logWarning(`${toolName} failed`, { + operation, + errorTag, + failureCount: failures.length, + }).pipe(Effect.as(result)); + }; + +/** + * `McpServer.toolkit` serializes every result as JSON text, which is the + * wrong shape for a screenshot: the model needs image content. Tools whose + * result carries a `screenshot` field are registered by hand so the PNG goes + * out as an image block and the rest of the payload as JSON metadata. + */ +const registerImageTool = ( + tool: T, + handle: ( + payload: Tool.Parameters, + ) => Effect.Effect<{ readonly encodedResult: unknown }, unknown, R>, + provide: ( + effect: Effect.Effect<{ readonly encodedResult: unknown }, unknown, R>, + ) => Effect.Effect< + { readonly encodedResult: unknown }, + unknown, + McpInvocationContext.McpInvocationContext + >, + operation: string, + failureText: string, +) => + Effect.gen(function* () { + const server = yield* McpServer.McpServer; + yield* server.addTool({ + tool: new McpSchema.Tool({ + name: tool.name, + description: Tool.getDescription(tool), + inputSchema: Tool.getJsonSchema(tool), + annotations: { + ...Context.getOption(tool.annotations, Tool.Title).pipe( + Option.map((title) => ({ title })), + Option.getOrUndefined, + ), + readOnlyHint: Context.get(tool.annotations, Tool.Readonly), + destructiveHint: Context.get(tool.annotations, Tool.Destructive), + idempotentHint: Context.get(tool.annotations, Tool.Idempotent), + openWorldHint: Context.get(tool.annotations, Tool.OpenWorld), + }, + }), + annotations: tool.annotations, + handle: (payload) => + Effect.withFiber((fiber) => { + const invocation = Context.getUnsafe( + fiber.context, + McpInvocationContext.McpInvocationContext, + ); + return provide(handle(payload as Tool.Parameters)).pipe( + Effect.provideService(McpInvocationContext.McpInvocationContext, invocation), + Effect.matchCauseEffect({ + onFailure: imageToolFailure(tool.name, operation, failureText), + onSuccess: ({ encodedResult }) => { + const { screenshot, ...rest } = encodedResult as ImageToolResult; + const includeImage = + (payload as { readonly includeImage?: boolean } | undefined)?.includeImage !== + false; + const metadata = { + ...rest, + screenshot: { + mimeType: screenshot.mimeType, + width: screenshot.width, + height: screenshot.height, + }, + }; + return Effect.succeed( + new McpSchema.CallToolResult({ + isError: false, + structuredContent: metadata, + content: [ + { type: "text", text: JSON.stringify(metadata) }, + ...(includeImage + ? [ + { + type: "image" as const, + data: new Uint8Array(Buffer.from(screenshot.data, "base64")), + mimeType: screenshot.mimeType, + }, + ] + : []), + ], + }), + ); + }, + }), + ); + }), + }); }); - return Effect.logWarning("preview snapshot failed", { - operation: "snapshot", - errorTag, - failureCount: failures.length, - }).pipe(Effect.as(result)); -}; const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot")(function* () { - const server = yield* McpServer.McpServer; const broker = yield* PreviewAutomationBroker.PreviewAutomationBroker; // The MCP tool runner only supplies the client, so hand the save path its services here. const saveServices = yield* Effect.context< @@ -424,6 +560,21 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot }); }); +const registerDeviceScreenshot = Effect.fn("McpHttpServer.registerDeviceScreenshot")(function* () { + const devices = yield* DeviceService.DeviceService; + const built = yield* DeviceScreenshotToolkit; + yield* registerImageTool( + DeviceScreenshotTool, + (payload) => + built + .handle("device_screenshot", payload) + .pipe(Stream.unwrap, Stream.run(Sink.last()), Effect.flatMap(Effect.fromOption)), + (effect) => effect.pipe(Effect.provideService(DeviceService.DeviceService, devices)), + "screenshot", + "Device screenshot failed.", + ); +}); + const PreviewStandardToolkitRegistrationLive = McpServer.toolkit(PreviewStandardToolkit).pipe( Layer.provide(PreviewStandardToolkitHandlersLive), ); @@ -437,6 +588,19 @@ export const PreviewToolkitRegistrationLive = Layer.mergeAll( PreviewSnapshotRegistrationLive, ); +const DeviceStandardToolkitRegistrationLive = McpServer.toolkit(DeviceStandardToolkit).pipe( + Layer.provide(DeviceStandardToolkitHandlersLive), +); + +const DeviceScreenshotRegistrationLive = Layer.effectDiscard(registerDeviceScreenshot()).pipe( + Layer.provide(DeviceScreenshotToolkitHandlersLive), +); + +export const DeviceToolkitRegistrationLive = Layer.mergeAll( + DeviceStandardToolkitRegistrationLive, + DeviceScreenshotRegistrationLive, +); + const McpTransportLive = McpServer.layerHttp({ name: "T3 Code", version: packageJson.version, @@ -444,4 +608,7 @@ const McpTransportLive = McpServer.layerHttp({ protocols: [McpProtocol.v2025_06_18], }).pipe(Layer.provide(McpAuthMiddlewareLive)); -export const layer = PreviewToolkitRegistrationLive.pipe(Layer.provideMerge(McpTransportLive)); +export const layer = Layer.mergeAll( + PreviewToolkitRegistrationLive, + DeviceToolkitRegistrationLive, +).pipe(Layer.provideMerge(McpTransportLive)); diff --git a/apps/server/src/mcp/McpInvocationContext.ts b/apps/server/src/mcp/McpInvocationContext.ts index 49273485a44d..f508c3d0cccd 100644 --- a/apps/server/src/mcp/McpInvocationContext.ts +++ b/apps/server/src/mcp/McpInvocationContext.ts @@ -7,7 +7,7 @@ import { import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; -export type McpCapability = "preview"; +export type McpCapability = "preview" | "device"; export interface McpInvocationScope { readonly environmentId: EnvironmentId; @@ -28,8 +28,11 @@ export const requireMcpCapability = Effect.fn("mcp.requireCapability")(function* ) { const invocation = yield* McpInvocationContext; if (!invocation.capabilities.has(capability)) { + // The error shape predates the second capability; "preview" is the + // wire-compatible tag, and device handlers remap it before it reaches + // the agent. return yield* new PreviewAutomationUnavailableError({ - capability, + capability: "preview", environmentId: invocation.environmentId, threadId: invocation.threadId, providerSessionId: invocation.providerSessionId, diff --git a/apps/server/src/mcp/McpProviderSession.ts b/apps/server/src/mcp/McpProviderSession.ts index d5dc582046c1..dfa7dc6e09c2 100644 --- a/apps/server/src/mcp/McpProviderSession.ts +++ b/apps/server/src/mcp/McpProviderSession.ts @@ -7,6 +7,31 @@ export interface McpProviderSessionConfig { readonly providerInstanceId: ProviderInstanceId; readonly endpoint: string; readonly authorizationHeader: string; + /** Capabilities the credential grants ("preview", "device"). */ + readonly capabilities: ReadonlySet; + /** + * Set when the session may drive devices. Adapters spread this into the + * provider subprocess environment so the `agent-device` CLI is on PATH and + * already pointed at the server's daemon; the agent never handles a token. + */ + readonly agentDeviceEnvironment?: Readonly>; +} + +/** Provider env with the device variables applied over `base`, or `base` untouched. */ +export function withAgentDeviceEnvironment( + base: NodeJS.ProcessEnv, + config: McpProviderSessionConfig | undefined, +): NodeJS.ProcessEnv { + const extra = config?.agentDeviceEnvironment; + if (!extra) return base; + const separator = extra.PATH_SEPARATOR ?? ":"; + const basePath = base.PATH ?? base.Path; + const { PATH: shimDir, PATH_SEPARATOR: _separator, ...rest } = extra; + return { + ...base, + ...rest, + ...(shimDir ? { PATH: basePath ? `${shimDir}${separator}${basePath}` : shimDir } : {}), + }; } const sessionsByThread = new Map(); diff --git a/apps/server/src/mcp/McpSessionRegistry.test.ts b/apps/server/src/mcp/McpSessionRegistry.test.ts index 1d8aead99d0d..1a4062f9e660 100644 --- a/apps/server/src/mcp/McpSessionRegistry.test.ts +++ b/apps/server/src/mcp/McpSessionRegistry.test.ts @@ -38,6 +38,7 @@ it.effect("stores only a token hash, resolves the bearer token, and revokes by t const threadId = ThreadId.make("thread-1"); const issued = yield* registry.issue({ threadId, + capabilities: new Set(["preview"]), providerInstanceId: ProviderInstanceId.make("codex"), }); expect(issued.config.endpoint).toBe("http://127.0.0.1:43123/mcp"); @@ -67,6 +68,7 @@ it.effect("builds MCP endpoints from the bound server host", () => const registry = yield* makeRegistry(() => 1_000, makeFakeHttpServer(hostname)); const issued = yield* registry.issue({ threadId: ThreadId.make(`thread-${hostname}`), + capabilities: new Set(["preview"]), providerInstanceId: ProviderInstanceId.make("codex"), }); expect(issued.config.endpoint).toBe(expectedEndpoint); @@ -80,6 +82,7 @@ it.effect("expires credentials once their session stops showing signs of life", const registry = yield* makeRegistry(() => timestamp); const issued = yield* registry.issue({ threadId: ThreadId.make("thread-2"), + capabilities: new Set(["preview"]), providerInstanceId: ProviderInstanceId.make("claude"), }); const token = issued.config.authorizationHeader.replace(/^Bearer\s+/, ""); @@ -95,6 +98,7 @@ it.effect("keeps a credential alive across turns that never touch an MCP tool", const threadId = ThreadId.make("thread-3"); const issued = yield* registry.issue({ threadId, + capabilities: new Set(["preview"]), providerInstanceId: ProviderInstanceId.make("claude"), }); const token = issued.config.authorizationHeader.replace(/^Bearer\s+/, ""); @@ -116,6 +120,7 @@ it.effect("does not keep credentials of other threads alive", () => const registry = yield* makeRegistry(() => timestamp); const issued = yield* registry.issue({ threadId: ThreadId.make("thread-4"), + capabilities: new Set(["preview"]), providerInstanceId: ProviderInstanceId.make("codex"), }); const token = issued.config.authorizationHeader.replace(/^Bearer\s+/, ""); diff --git a/apps/server/src/mcp/McpSessionRegistry.ts b/apps/server/src/mcp/McpSessionRegistry.ts index f19a4f4e8c49..abeb74043db7 100644 --- a/apps/server/src/mcp/McpSessionRegistry.ts +++ b/apps/server/src/mcp/McpSessionRegistry.ts @@ -14,6 +14,7 @@ import * as McpProviderSession from "./McpProviderSession.ts"; export interface McpCredentialRequest { readonly threadId: ThreadId; readonly providerInstanceId: ProviderInstanceId; + readonly capabilities: ReadonlySet; } export interface McpIssuedCredential { @@ -128,7 +129,7 @@ const makeWithOptions = Effect.fn("McpSessionRegistry.make")(function* ( threadId: ThreadId.make(request.threadId), providerSessionId, providerInstanceId: ProviderInstanceId.make(request.providerInstanceId), - capabilities: new Set(["preview"]), + capabilities: new Set(request.capabilities), issuedAt, }; yield* SynchronizedRef.update(state, ({ records }) => { @@ -144,6 +145,7 @@ const makeWithOptions = Effect.fn("McpSessionRegistry.make")(function* ( providerInstanceId: scope.providerInstanceId, endpoint, authorizationHeader: `Bearer ${rawToken}`, + capabilities: scope.capabilities, }, }; }, diff --git a/apps/server/src/mcp/toolkits/device/handlers.test.ts b/apps/server/src/mcp/toolkits/device/handlers.test.ts new file mode 100644 index 000000000000..e37b2a8b22c2 --- /dev/null +++ b/apps/server/src/mcp/toolkits/device/handlers.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { agentDeviceQuickStart, agentDeviceTargetArgs, pngDimensions } from "./handlers.ts"; + +const device = { + hostId: "local", + id: "ABCD-1234", + platform: "ios" as const, + name: "iPhone 17 Pro", + version: "iOS 27.0", + booted: true, + physical: false, +}; + +describe("device tool helpers", () => { + it("pins agent-device commands to the device by platform-specific flag", () => { + expect(agentDeviceTargetArgs(device)).toEqual(["--platform", "ios", "--udid", "ABCD-1234"]); + expect(agentDeviceTargetArgs({ ...device, platform: "android", id: "emulator-5554" })).toEqual([ + "--platform", + "android", + "--serial", + "emulator-5554", + ]); + }); + + it("writes the quick start around the pinned target", () => { + const text = agentDeviceQuickStart(device); + expect(text).toContain("agent-device snapshot -i --platform ios --udid ABCD-1234"); + expect(text).toContain("iPhone 17 Pro (iOS 27.0)"); + expect(text).toContain("XCTest runner"); + }); + + it("reads PNG dimensions from the IHDR chunk", () => { + const png = new Uint8Array(24); + new DataView(png.buffer).setUint32(0, 0x89504e47); + new DataView(png.buffer).setUint32(4, 0x0d0a1a0a); + new DataView(png.buffer).setUint32(12, 0x49484452); + new DataView(png.buffer).setUint32(16, 1179); + new DataView(png.buffer).setUint32(20, 2556); + expect(pngDimensions(png)).toEqual({ width: 1179, height: 2556 }); + expect(pngDimensions(new Uint8Array([1, 2, 3]))).toEqual({ width: 0, height: 0 }); + }); +}); diff --git a/apps/server/src/mcp/toolkits/device/handlers.ts b/apps/server/src/mcp/toolkits/device/handlers.ts new file mode 100644 index 000000000000..baf55201152e --- /dev/null +++ b/apps/server/src/mcp/toolkits/device/handlers.ts @@ -0,0 +1,195 @@ +import { + type DeviceError, + type DeviceHostId, + type DeviceId, + type DevicePlatform, + type DeviceSummary, + DeviceToolUnavailableError, + LOCAL_DEVICE_HOST_ID, +} from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; + +import * as DeviceService from "../../../device/DeviceService.ts"; +import * as McpInvocationContext from "../../McpInvocationContext.ts"; +import { DeviceScreenshotToolkit, DeviceStandardToolkit, DeviceToolkit } from "./tools.ts"; + +/** The flags that pin every agent-device command to one device. */ +export function agentDeviceTargetArgs(device: DeviceSummary): ReadonlyArray { + return device.platform === "ios" + ? ["--platform", "ios", "--udid", device.id] + : ["--platform", "android", "--serial", device.id]; +} + +/** + * Just-in-time guidance returned from `device_open`. This is the one place + * the agent learns how to drive the device, so it lives with the tool result + * rather than in the always-on prompt block; threads that never open a device + * never pay for it. + */ +export function agentDeviceQuickStart(device: DeviceSummary): string { + const target = agentDeviceTargetArgs(device).join(" "); + const platformNotes = + device.platform === "ios" + ? "First use builds an XCTest runner and can take a couple of minutes; later commands are fast." + : "The Android snapshot helper installs itself on first use."; + return [ + `The user is watching ${device.name} (${device.version}) in the Device panel.`, + `Drive it with the agent-device CLI, which is on PATH and already connected to this environment. Always pass ${target}.`, + "Typical loop:", + ` agent-device open ${target} # or: open `, + ` agent-device snapshot -i ${target} # accessibility tree with @eN refs`, + ` agent-device click @e3 ${target}`, + ` agent-device fill @e5 "text" ${target}`, + ` agent-device screenshot /tmp/shot.png ${target} # or call device_screenshot`, + ` agent-device install ${target}`, + "Prefer snapshot refs over coordinates. Run `agent-device help` for workflow guides and `agent-device --help` for flags.", + "Do not call simctl, adb, xcrun, or serve-sim directly while these tools are attached; use agent-device.", + platformNotes, + ].join("\n"); +} + +const requireDeviceAccess = McpInvocationContext.requireMcpCapability("device").pipe( + Effect.mapError( + () => + new DeviceToolUnavailableError({ + reason: "Agent device access is turned off for this environment.", + }), + ), +); + +const pickDevice = ( + devices: ReadonlyArray, + input: { + readonly deviceId?: DeviceId | undefined; + readonly platform?: DevicePlatform | undefined; + readonly hostId?: DeviceHostId | undefined; + }, +): Effect.Effect => + Effect.gen(function* () { + const hostId = input.hostId ?? LOCAL_DEVICE_HOST_ID; + if (input.deviceId !== undefined) { + const match = devices.find( + (device) => device.hostId === hostId && device.id === input.deviceId, + ); + if (match) return match; + return yield* new DeviceToolUnavailableError({ + reason: `No device ${input.deviceId} on host ${hostId}. Call device_list for current ids.`, + }); + } + const candidates = devices.filter( + (device) => + device.hostId === hostId && + (input.platform === undefined || device.platform === input.platform), + ); + if (candidates.length === 0) { + return yield* new DeviceToolUnavailableError({ + reason: + input.platform === undefined + ? "No simulators or emulators were found. Call device_list to see why." + : `No ${input.platform} devices were found on host ${hostId}. Call device_list to see why.`, + }); + } + const platforms = new Set(candidates.map((device) => device.platform)); + if (input.platform === undefined && platforms.size > 1) { + return yield* new DeviceToolUnavailableError({ + reason: "Both iOS and Android devices are available; pass platform or deviceId.", + }); + } + return candidates.find((device) => device.booted) ?? candidates[0]!; + }); + +const toolError = (error: DeviceError | DeviceToolUnavailableError) => error; + +const handlers = { + device_list: () => + Effect.gen(function* () { + const scope = yield* requireDeviceAccess; + const devices = yield* DeviceService.DeviceService; + const state = yield* devices.list; + const open = state.sessions + .filter((session) => session.threadId === scope.threadId) + .map((session) => ({ hostId: session.hostId, deviceId: session.deviceId })); + return { hosts: state.hosts, devices: state.devices, open }; + }).pipe(Effect.mapError(toolError)), + device_open: (input) => + Effect.gen(function* () { + const scope = yield* requireDeviceAccess; + const devices = yield* DeviceService.DeviceService; + const state = yield* devices.list; + const target = yield* pickDevice(state.devices, input); + const session = yield* devices.open({ + threadId: scope.threadId, + hostId: target.hostId, + deviceId: target.id, + platform: target.platform, + }); + const after = yield* devices.state; + const device = + after.devices.find( + (candidate) => candidate.hostId === session.hostId && candidate.id === session.deviceId, + ) ?? target; + return { + device, + agentDevice: { command: "agent-device", targetArgs: agentDeviceTargetArgs(device) }, + quickStart: agentDeviceQuickStart(device), + }; + }).pipe(Effect.mapError(toolError)), + device_screenshot: (input) => + Effect.gen(function* () { + const scope = yield* requireDeviceAccess; + const devices = yield* DeviceService.DeviceService; + const sessions = yield* devices.sessionsForThread(scope.threadId); + const target = + input.deviceId !== undefined + ? { hostId: input.hostId ?? LOCAL_DEVICE_HOST_ID, deviceId: input.deviceId } + : sessions.at(-1); + if (!target) { + return yield* new DeviceToolUnavailableError({ + reason: "No device is open in this thread. Call device_open first.", + }); + } + const shot = yield* devices.screenshot(target); + return { + device: shot.device, + screenshot: { + mimeType: "image/png" as const, + data: Buffer.from(shot.png).toString("base64"), + ...pngDimensions(shot.png), + }, + }; + }).pipe(Effect.mapError(toolError)), + device_close: (input) => + Effect.gen(function* () { + const scope = yield* requireDeviceAccess; + const devices = yield* DeviceService.DeviceService; + yield* devices.close({ + threadId: scope.threadId, + ...(input.deviceId === undefined ? {} : { deviceId: input.deviceId }), + ...(input.shutdown === undefined ? {} : { shutdown: input.shutdown }), + }); + return {}; + }).pipe(Effect.mapError(toolError)), +} satisfies Parameters[0]; + +/** Width and height from the IHDR chunk; a PNG that lacks one reports 0×0. */ +export function pngDimensions(png: Uint8Array): { width: number; height: number } { + if (png.length < 24) return { width: 0, height: 0 }; + const view = new DataView(png.buffer, png.byteOffset, png.byteLength); + const isPng = + view.getUint32(0) === 0x89504e47 && + view.getUint32(4) === 0x0d0a1a0a && + view.getUint32(12) === 0x49484452; + return isPng + ? { width: view.getUint32(16), height: view.getUint32(20) } + : { width: 0, height: 0 }; +} + +const { device_screenshot, ...standardHandlers } = handlers; + +export const DeviceStandardToolkitHandlersLive = DeviceStandardToolkit.toLayer(standardHandlers); + +export const DeviceScreenshotToolkitHandlersLive = DeviceScreenshotToolkit.toLayer({ + device_screenshot, +}); + +export const DeviceToolkitHandlersLive = DeviceToolkit.toLayer(handlers); diff --git a/apps/server/src/mcp/toolkits/device/tools.ts b/apps/server/src/mcp/toolkits/device/tools.ts new file mode 100644 index 000000000000..5e157f1f5821 --- /dev/null +++ b/apps/server/src/mcp/toolkits/device/tools.ts @@ -0,0 +1,92 @@ +import { + DeviceToolCloseInput, + DeviceToolError, + DeviceToolListResult, + DeviceToolOpenInput, + DeviceToolOpenResult, + DeviceToolScreenshotResult, + DeviceToolTargetInput, +} from "@t3tools/contracts"; +import * as Schema from "effect/Schema"; +import { Tool, Toolkit } from "effect/unstable/ai"; + +import * as McpInvocationContext from "../../McpInvocationContext.ts"; +import * as DeviceService from "../../../device/DeviceService.ts"; + +const dependencies = [McpInvocationContext.McpInvocationContext, DeviceService.DeviceService]; + +/** + * Deliberately a small surface: lifecycle, visibility for the user, and one + * image-returning verb. Driving the device (taps, typing, install, logs) + * happens through the preconfigured `agent-device` CLI, which has the + * semantic snapshot model agents need and stays current with its own + * releases. Wrapping its commands here would only lag behind it. + */ +export const DeviceListTool = Tool.make("device_list", { + description: + "List iOS Simulators and Android Emulators on this environment's device hosts, which platforms each host can run, and which devices are already open in this thread's Device panel. Call this before device_open when you do not know a device id.", + parameters: Schema.Struct({}), + success: DeviceToolListResult, + failure: DeviceToolError, + dependencies, +}) + .annotate(Tool.Title, "List devices") + .annotate(Tool.Readonly, true) + .annotate(Tool.Destructive, false) + .annotate(Tool.Idempotent, true) + .annotate(Tool.OpenWorld, false); + +export const DeviceOpenTool = Tool.make("device_open", { + description: + "Open a simulator or emulator for this thread: boots it if needed, starts its live stream, and shows it in the user's Device panel so they can watch. Returns the agent-device CLI invocation pinned to the device; drive the device with that CLI afterwards.", + parameters: DeviceToolOpenInput, + success: DeviceToolOpenResult, + failure: DeviceToolError, + dependencies, +}) + .annotate(Tool.Title, "Open device") + .annotate(Tool.Readonly, false) + .annotate(Tool.Destructive, false) + .annotate(Tool.Idempotent, true) + .annotate(Tool.OpenWorld, true); + +export const DeviceScreenshotTool = Tool.make("device_screenshot", { + description: + "Capture the current screen of an open device as a PNG image. Use it to see what the user sees; for taps and text use the agent-device CLI.", + parameters: DeviceToolTargetInput, + success: DeviceToolScreenshotResult, + failure: DeviceToolError, + dependencies, +}) + .annotate(Tool.Title, "Screenshot device") + .annotate(Tool.Readonly, true) + .annotate(Tool.Destructive, false) + .annotate(Tool.Idempotent, true) + .annotate(Tool.OpenWorld, true); + +export const DeviceCloseTool = Tool.make("device_close", { + description: + "Remove a device from this thread's Device panel. Pass shutdown=true to also power the simulator or emulator off.", + parameters: DeviceToolCloseInput, + success: Schema.Record(Schema.String, Schema.Never).annotate({ + description: "The device was closed.", + }), + failure: DeviceToolError, + dependencies, +}) + .annotate(Tool.Title, "Close device") + .annotate(Tool.Readonly, false) + .annotate(Tool.Destructive, true) + .annotate(Tool.Idempotent, true) + .annotate(Tool.OpenWorld, true); + +export const DeviceStandardToolkit = Toolkit.make(DeviceListTool, DeviceOpenTool, DeviceCloseTool); + +export const DeviceScreenshotToolkit = Toolkit.make(DeviceScreenshotTool); + +export const DeviceToolkit = Toolkit.make( + DeviceListTool, + DeviceOpenTool, + DeviceScreenshotTool, + DeviceCloseTool, +); diff --git a/apps/server/src/provider/CodexDeveloperInstructions.ts b/apps/server/src/provider/CodexDeveloperInstructions.ts index 1c2439a9ad9a..0e9f4a35a188 100644 --- a/apps/server/src/provider/CodexDeveloperInstructions.ts +++ b/apps/server/src/provider/CodexDeveloperInstructions.ts @@ -12,18 +12,39 @@ For browser work, first call \`preview_status\`. If no automation-capable previe Do not switch to global browser skills, Chrome, Node REPL browser automation, standalone Playwright, or agent-browser merely because the preview is initially closed or a first call fails. Use an alternative browser system only when the T3 preview tools are absent, the user explicitly requests another browser, or \`preview_open\` returns an explicit unsupported/unavailable error. A failed T3 preview tool call should be inspected and retried with corrected arguments when the error is actionable. `; +export const T3_CODE_DEVICE_TOOL_INSTRUCTIONS = ` + +## T3 Code devices + +The \`t3-code\` MCP server also exposes \`device_*\` tools for iOS Simulators and Android Emulators on this environment. For mobile verification, call \`device_list\`, then \`device_open\` so the user can watch the device in their Device panel; its result explains how to drive the device. Driving happens through the \`agent-device\` CLI, which is on PATH and already connected: prefer \`agent-device snapshot -i\` refs over coordinates, and use \`device_screenshot\` when you need to see the screen. Do not call simctl, adb, xcrun, or serve-sim directly while these tools are present. If \`device_list\` reports a platform as unavailable, say so instead of trying another route. +`; + +export interface T3CodeToolAvailability { + readonly browser: boolean; + readonly device: boolean; +} + +const normalizeAvailability = ( + availability: boolean | T3CodeToolAvailability, +): T3CodeToolAvailability => + typeof availability === "boolean" ? { browser: availability, device: false } : availability; + /** - * The browser block is omitted entirely when the preview tools aren't attached. - * Describing `preview_*` tools that aren't in the turn's tool list would be + * Each block is omitted entirely when its tools aren't attached. Describing + * `preview_*` or `device_*` tools that aren't in the turn's tool list would be * worse than saying nothing: the instructions actively steer the model away - * from Playwright and agent-browser, so leaving them in would talk it out of - * the only browser automation it still has. + * from Playwright, agent-browser, and raw simctl/adb, so leaving them in would + * talk it out of the only automation it still has. */ -const browserToolInstructions = (browserToolsAvailable: boolean): string => - browserToolsAvailable ? T3_CODE_BROWSER_TOOL_INSTRUCTIONS : ""; +const browserToolInstructions = (availability: boolean | T3CodeToolAvailability): string => { + const tools = normalizeAvailability(availability); + return `${tools.browser ? T3_CODE_BROWSER_TOOL_INSTRUCTIONS : ""}${ + tools.device ? T3_CODE_DEVICE_TOOL_INSTRUCTIONS : "" + }`; +}; const codexPlanModeDeveloperInstructions = ( - browserToolsAvailable: boolean, + browserToolsAvailable: boolean | T3CodeToolAvailability, ): string => `# Plan Mode (Conversational) You work in 3 phases, and you should *chat your way* to a great plan before finalizing it. A great plan is very detailed-intent- and implementation-wise-so that it can be handed to another engineer or agent to be implemented right away. It must be **decision complete**, where the implementer does not need to make any decisions. @@ -156,7 +177,7 @@ ${browserToolInstructions(browserToolsAvailable)} `; const codexDefaultModeDeveloperInstructions = ( - browserToolsAvailable: boolean, + browserToolsAvailable: boolean | T3CodeToolAvailability, ): string => `# Collaboration Mode: Default You are now in Default mode. Any previous instructions for other modes (e.g. Plan mode) are no longer active. @@ -184,7 +205,7 @@ export function buildCodexDeveloperInstructions( * it from the session's actual MCP configuration rather than re-reading the * setting, so the prompt cannot claim tools the turn doesn't have. */ - browserToolsAvailable = true, + browserToolsAvailable: boolean | T3CodeToolAvailability = true, ): string { const base = interactionMode === "plan" diff --git a/apps/server/src/provider/Layers/ClaudeAdapter.ts b/apps/server/src/provider/Layers/ClaudeAdapter.ts index 81a4a197e9db..4af5a0633654 100644 --- a/apps/server/src/provider/Layers/ClaudeAdapter.ts +++ b/apps/server/src/provider/Layers/ClaudeAdapter.ts @@ -4696,7 +4696,7 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( canUseTool, onUserDialog, supportedDialogKinds: ["resume_return"], - env: claudeEnvironment, + env: McpProviderSession.withAgentDeviceEnvironment(claudeEnvironment, mcpSession), additionalDirectories, ...(Object.keys(extraArgs).length > 0 ? { extraArgs } : {}), ...(mcpSession diff --git a/apps/server/src/provider/Layers/CodexAdapter.ts b/apps/server/src/provider/Layers/CodexAdapter.ts index 2d88e58dc1fb..b43755736ca3 100644 --- a/apps/server/src/provider/Layers/CodexAdapter.ts +++ b/apps/server/src/provider/Layers/CodexAdapter.ts @@ -2274,7 +2274,10 @@ export const makeCodexAdapter = Effect.fn("makeCodexAdapter")(function* ( ...(mcpSession ? { environment: { - ...(options?.environment ?? process.env), + ...McpProviderSession.withAgentDeviceEnvironment( + options?.environment ?? process.env, + mcpSession, + ), T3_MCP_BEARER_TOKEN: mcpSession.authorizationHeader.replace(/^Bearer\s+/, ""), }, appServerArgs: [ @@ -2283,6 +2286,7 @@ export const makeCodexAdapter = Effect.fn("makeCodexAdapter")(function* ( "-c", 'mcp_servers.t3-code.bearer_token_env_var="T3_MCP_BEARER_TOKEN"', ], + mcpCapabilities: mcpSession.capabilities, } : {}), }; diff --git a/apps/server/src/provider/Layers/CodexSessionRuntime.ts b/apps/server/src/provider/Layers/CodexSessionRuntime.ts index 78f4b9aa8e50..29499d700689 100644 --- a/apps/server/src/provider/Layers/CodexSessionRuntime.ts +++ b/apps/server/src/provider/Layers/CodexSessionRuntime.ts @@ -39,7 +39,10 @@ import * as EffectCodexSchema from "effect-codex-app-server/schema"; import { buildCodexInitializeParams } from "./CodexProvider.ts"; import { codexSessionAppServerArgs } from "./codexLaunchArgs.ts"; import { expandHomePath } from "../../pathExpansion.ts"; -import { buildCodexDeveloperInstructions } from "../CodexDeveloperInstructions.ts"; +import { + buildCodexDeveloperInstructions, + type T3CodeToolAvailability, +} from "../CodexDeveloperInstructions.ts"; const decodeV2TurnStartResponse = Schema.decodeUnknownEffect(EffectCodexSchema.V2TurnStartResponse); const PROVIDER = ProviderDriverKind.make("codex"); @@ -66,6 +69,16 @@ export function hasConfiguredMcpServer(appServerArgs: ReadonlyArray | un return appServerArgs?.some((argument) => argument.includes("mcp_servers.")) === true; } +export function configuredMcpToolAvailability( + appServerArgs: ReadonlyArray | undefined, + mcpCapabilities: ReadonlySet | undefined, +): T3CodeToolAvailability { + if (!hasConfiguredMcpServer(appServerArgs)) return { browser: false, device: false }; + // Callers predating the capability set attached the browser toolkit only. + if (mcpCapabilities === undefined) return { browser: true, device: false }; + return { browser: mcpCapabilities.has("preview"), device: mcpCapabilities.has("device") }; +} + export const CodexResumeCursorSchema = Schema.Struct({ threadId: Schema.String, }); @@ -166,6 +179,8 @@ export interface CodexSessionRuntimeOptions { readonly serviceTier?: CodexServiceTier | undefined; readonly resumeCursor?: CodexResumeCursor; readonly appServerArgs?: ReadonlyArray; + /** Capabilities the session's `t3-code` MCP credential grants; drives the prompt blocks. */ + readonly mcpCapabilities?: ReadonlySet; } export interface CodexSessionRuntimeSendTurnInput { @@ -569,7 +584,7 @@ function buildCodexCollaborationMode(input: { readonly interactionMode?: ProviderInteractionMode; readonly model?: string; readonly effort?: EffectCodexSchema.V2TurnStartParams__ReasoningEffort; - readonly browserToolsAvailable?: boolean; + readonly browserToolsAvailable?: boolean | T3CodeToolAvailability; }): EffectCodexSchema.V2TurnStartParams__CollaborationMode | undefined { if (input.interactionMode === undefined) { return undefined; @@ -603,7 +618,7 @@ export function buildTurnStartParams(input: { readonly effort?: EffectCodexSchema.V2TurnStartParams__ReasoningEffort; readonly interactionMode?: ProviderInteractionMode; /** Defaults to true so callers that predate the agent-access gate are unchanged. */ - readonly browserToolsAvailable?: boolean; + readonly browserToolsAvailable?: boolean | T3CodeToolAvailability; }): Effect.Effect< CodexTurnStartParamsWithCollaborationMode, CodexErrors.CodexAppServerProtocolParseError @@ -2352,7 +2367,10 @@ export const makeCodexSessionRuntime = ( // Derived from the session's own MCP configuration rather than the // setting, so the prompt describes the tools this turn actually // has even if the setting changed after the session started. - browserToolsAvailable: hasConfiguredMcpServer(options.appServerArgs), + browserToolsAvailable: configuredMcpToolAvailability( + options.appServerArgs, + options.mcpCapabilities, + ), }); const rawResponse = yield* client.raw.request("turn/start", params); const response = yield* decodeV2TurnStartResponse(rawResponse).pipe( diff --git a/apps/server/src/provider/Layers/CursorAdapter.ts b/apps/server/src/provider/Layers/CursorAdapter.ts index 1a77f964aac5..b19d46112fde 100644 --- a/apps/server/src/provider/Layers/CursorAdapter.ts +++ b/apps/server/src/provider/Layers/CursorAdapter.ts @@ -543,7 +543,14 @@ export function makeCursorAdapter( const mcpSession = McpProviderSession.readMcpProviderSession(input.threadId); const acp = yield* makeCursorAcpRuntime({ cursorSettings: effectiveCursorSettings, - ...(options?.environment ? { environment: options.environment } : {}), + ...(options?.environment || mcpSession?.agentDeviceEnvironment + ? { + environment: McpProviderSession.withAgentDeviceEnvironment( + options?.environment ?? process.env, + mcpSession, + ), + } + : {}), childProcessSpawner, cwd, runtimeMode: input.runtimeMode, diff --git a/apps/server/src/provider/Layers/GrokAdapter.ts b/apps/server/src/provider/Layers/GrokAdapter.ts index 25188adcffcc..a2f78a0d72d1 100644 --- a/apps/server/src/provider/Layers/GrokAdapter.ts +++ b/apps/server/src/provider/Layers/GrokAdapter.ts @@ -989,7 +989,14 @@ export function makeGrokAdapter(grokSettings: GrokSettings, options?: GrokAdapte const mcpSession = McpProviderSession.readMcpProviderSession(input.threadId); const acp = yield* makeGrokAcpRuntime({ grokSettings, - ...(options?.environment ? { environment: options.environment } : {}), + ...(options?.environment || mcpSession?.agentDeviceEnvironment + ? { + environment: McpProviderSession.withAgentDeviceEnvironment( + options?.environment ?? process.env, + mcpSession, + ), + } + : {}), childProcessSpawner, cwd, runtimeMode: input.runtimeMode, diff --git a/apps/server/src/provider/Layers/ProviderService.test.ts b/apps/server/src/provider/Layers/ProviderService.test.ts index e7647866d604..0b2d8562ebc7 100644 --- a/apps/server/src/provider/Layers/ProviderService.test.ts +++ b/apps/server/src/provider/Layers/ProviderService.test.ts @@ -4801,12 +4801,16 @@ describe("agent browser access", () => { const projectId = ProjectId.make("project-browser-access"); const startSessionWith = ( - enableAgentBrowserAccess: boolean, + access: boolean | { readonly browser: boolean; readonly device: boolean }, threadId: ThreadId, projectOverride?: boolean, ) => Effect.gen(function* () { - const issued: Array = []; + const enableAgentBrowserAccess = typeof access === "boolean" ? access : access.browser; + // Device access shares the credential; a browser-only test turns it off + // too so "no credential at all" stays observable. + const enableAgentDeviceAccess = typeof access === "boolean" ? access : access.device; + const issued: Array<{ threadId: ThreadId; capabilities: ReadonlyArray }> = []; const codex = makeFakeCodexAdapter(); const providerAdapterLayer = Layer.succeed( ProviderAdapterRegistry.ProviderAdapterRegistry, @@ -4865,7 +4869,10 @@ describe("agent browser access", () => { const providerLayer = makeProviderServiceLive({ issueMcpCredential: (request) => Effect.sync(() => { - issued.push(request.threadId); + issued.push({ + threadId: request.threadId, + capabilities: [...request.capabilities].toSorted(), + }); return undefined; }), revokeMcpCredential: (revoked) => Effect.sync(() => void revokedThreads.push(revoked)), @@ -4876,6 +4883,7 @@ describe("agent browser access", () => { Layer.provide( ServerSettings.ServerSettingsService.layerTest({ enableAgentBrowserAccess, + enableAgentDeviceAccess, projectAgentBrowserAccessOverrides: projectOverride === undefined ? {} : { [projectId]: projectOverride }, }), @@ -4934,7 +4942,17 @@ describe("agent browser access", () => { const issued = yield* startSessionWith(true, threadId); - assert.deepEqual(issued, [threadId]); + assert.deepEqual(issued, [{ threadId, capabilities: ["device", "preview"] }]); + }).pipe(Effect.provide(NodeServices.layer)), + ); + + it.effect("drops only the preview capability when browser access alone is off", () => + Effect.gen(function* () { + const threadId = asThreadId("thread-browser-off-device-on"); + + const issued = yield* startSessionWith({ browser: false, device: true }, threadId); + + assert.deepEqual(issued, [{ threadId, capabilities: ["device"] }]); }).pipe(Effect.provide(NodeServices.layer)), ); @@ -4942,17 +4960,25 @@ describe("agent browser access", () => { Effect.gen(function* () { const threadId = asThreadId("thread-project-browser-off"); revokedThreads.length = 0; - const issued = yield* startSessionWith(true, threadId, false); + const issued = yield* startSessionWith({ browser: true, device: false }, threadId, false); assert.deepEqual(issued, []); assert.deepEqual(revokedThreads, [threadId]); }).pipe(Effect.provide(NodeServices.layer)), ); + it.effect("a project browser override leaves device access alone", () => + Effect.gen(function* () { + const threadId = asThreadId("thread-project-browser-off-device-on"); + const issued = yield* startSessionWith(true, threadId, false); + assert.deepEqual(issued, [{ threadId, capabilities: ["device"] }]); + }).pipe(Effect.provide(NodeServices.layer)), + ); + it.effect("requests an MCP credential when the project overrides browser access to on", () => Effect.gen(function* () { const threadId = asThreadId("thread-project-browser-on"); - const issued = yield* startSessionWith(false, threadId, true); - assert.deepEqual(issued, [threadId]); + const issued = yield* startSessionWith({ browser: false, device: false }, threadId, true); + assert.deepEqual(issued, [{ threadId, capabilities: ["preview"] }]); }).pipe(Effect.provide(NodeServices.layer)), ); }); diff --git a/apps/server/src/provider/Layers/ProviderService.ts b/apps/server/src/provider/Layers/ProviderService.ts index 016bcd5c4a28..85a5b8cb9785 100644 --- a/apps/server/src/provider/Layers/ProviderService.ts +++ b/apps/server/src/provider/Layers/ProviderService.ts @@ -34,6 +34,7 @@ import { type ProviderSession, } from "@t3tools/contracts"; import { expandAssistantCitationsForProvider } from "@t3tools/shared/assistantCitations"; +import { HostProcessPlatform } from "@t3tools/shared/hostProcess"; import { causeErrorTag } from "@t3tools/shared/observability"; import { getModelSelectionStringOptionValue } from "@t3tools/shared/model"; import { resolveProjectAgentBrowserAccess } from "@t3tools/shared/serverSettings"; @@ -43,6 +44,7 @@ import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; +import * as Path from "effect/Path"; import * as PubSub from "effect/PubSub"; import * as Ref from "effect/Ref"; import * as Schema from "effect/Schema"; @@ -52,6 +54,9 @@ import * as Stream from "effect/Stream"; import { appendUserInputAttachmentPaths } from "../userInputAttachments.ts"; import { resolveAttachmentPath } from "../../attachmentStore.ts"; import * as ServerConfig from "../../config.ts"; +import { ensureAgentDeviceShim } from "../../device/AgentDeviceShim.ts"; +import * as DeviceService from "../../device/DeviceService.ts"; +import type * as McpInvocationContext from "../../mcp/McpInvocationContext.ts"; import { increment, providerMetricAttributes, @@ -251,6 +256,8 @@ export interface ProviderServiceLiveOptions { readonly issueMcpCredential?: typeof McpSessionRegistry.issueActiveMcpCredential; /** Same seam as `issueMcpCredential`, for observing the deny path's revoke. */ readonly revokeMcpCredential?: typeof McpSessionRegistry.revokeActiveMcpThread; + /** Overrides the device host lookup used to build the agent-device environment. */ + readonly deviceReadiness?: () => Effect.Effect; } interface TurnAnalyticsMetadata { @@ -481,7 +488,16 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( options?.issueMcpCredential ?? McpSessionRegistry.issueActiveMcpCredential; const revokeMcpCredential = options?.revokeMcpCredential ?? McpSessionRegistry.revokeActiveMcpThread; + const deviceReadiness = + options?.deviceReadiness ?? + (() => + Effect.serviceOption(DeviceService.DeviceService).pipe( + Effect.flatMap((service) => + Option.isSome(service) ? service.value.currentReadiness() : Effect.succeed(null), + ), + )); const fileSystem = yield* FileSystem.FileSystem; + const pathService = yield* Path.Path; const runtimeEventPubSub = yield* PubSub.unbounded(); const pendingCompactions = new Map(); const timedOutNativeCompactions = new Set(); @@ -889,9 +905,56 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( ), ); + const agentDeviceAccessEnabled = serverSettings.getSettings.pipe( + Effect.map((settings) => settings.enableAgentDeviceAccess), + Effect.catch((cause) => + Effect.logWarning( + "Could not read server settings; withholding agent device access for this session.", + { cause }, + ).pipe(Effect.as(false)), + ), + ); + + const agentAccessCapabilities = Effect.fn("ProviderService.agentAccessCapabilities")( + function* (threadId: ThreadId) { + const capabilities = new Set(); + if (yield* agentBrowserAccessEnabled(threadId)) capabilities.add("preview"); + if (yield* agentDeviceAccessEnabled) capabilities.add("device"); + return capabilities; + }, + ); + + /** + * The device host only starts when a device is first opened, so a session + * prepared before that gets the tools without the CLI environment; the + * `device_open` result tells the agent the CLI is ready, and by then the + * next session restart picks the environment up. Sessions prepared after + * the host is running get it immediately. + */ + const hostPlatform = yield* HostProcessPlatform; + const agentDeviceEnvironment = Effect.gen(function* () { + const readiness = yield* deviceReadiness(); + if (!readiness) return undefined; + const shimDir = yield* ensureAgentDeviceShim({ + entryPath: readiness.agentDevice.entryPath, + stateDir: serverConfig.stateDir, + fs: fileSystem, + path: pathService, + }).pipe(Effect.orElseSucceed(() => undefined)); + if (!shimDir) return undefined; + return { + PATH: shimDir, + PATH_SEPARATOR: hostPlatform === "win32" ? ";" : ":", + AGENT_DEVICE_DAEMON_BASE_URL: readiness.agentDevice.baseUrl, + AGENT_DEVICE_DAEMON_AUTH_TOKEN: readiness.agentDevice.token, + AGENT_DEVICE_NO_UPDATE_NOTIFIER: "1", + } satisfies Record; + }); + const prepareMcpSession = (threadId: ThreadId, providerInstanceId: ProviderInstanceId) => Effect.gen(function* () { - if (!(yield* agentBrowserAccessEnabled(threadId))) { + const capabilities = yield* agentAccessCapabilities(threadId); + if (capabilities.size === 0) { // Revoke as well as clear. Every other prepare path reaches // `issueActiveMcpCredential`, which revokes the thread first, so // skipping it here would leave a previously issued bearer token valid @@ -902,9 +965,17 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( yield* Effect.sync(() => McpProviderSession.clearMcpProviderSession(threadId)); return undefined; } - const credential = yield* issueMcpCredential({ threadId, providerInstanceId }); + const credential = yield* issueMcpCredential({ threadId, providerInstanceId, capabilities }); if (credential) { - yield* Effect.sync(() => McpProviderSession.setMcpProviderSession(credential.config)); + const deviceEnvironment = capabilities.has("device") + ? yield* agentDeviceEnvironment + : undefined; + yield* Effect.sync(() => + McpProviderSession.setMcpProviderSession({ + ...credential.config, + ...(deviceEnvironment ? { agentDeviceEnvironment: deviceEnvironment } : {}), + }), + ); } return credential; }); diff --git a/apps/server/src/server.test.ts b/apps/server/src/server.test.ts index 2f32b6524d7b..a45a5ff0039b 100644 --- a/apps/server/src/server.test.ts +++ b/apps/server/src/server.test.ts @@ -5,6 +5,7 @@ import * as NodeCrypto from "node:crypto"; import { HostProcessEnvironment, HostProcessPlatform } from "@t3tools/shared/hostProcess"; import { + type DeviceServiceState, AuthAccessTokenType, AuthStandardClientScopes, AuthEnvironmentBootstrapTokenType, @@ -97,6 +98,7 @@ const encodeTestJson = Schema.encodeUnknownSync(Schema.fromJsonString(Schema.Unk import * as BackgroundPolicy from "./background/BackgroundPolicy.ts"; import * as ServerConfig from "./config.ts"; +import * as DeviceService from "./device/DeviceService.ts"; import { HTTP_ROUTER_CONFIG, makeRoutesLayer } from "./server.ts"; import { isThreadDetailEvent, @@ -802,6 +804,11 @@ const buildAppUnderTest = (options?: { listBindings: () => Effect.succeed([]), ...options?.layers?.providerSessionDirectory, }), + Layer.mock(DeviceService.DeviceService)({ + state: Effect.succeed(EMPTY_DEVICE_STATE), + currentReadiness: () => Effect.succeed(null), + sessionsForThread: () => Effect.succeed([]), + }), ), ), Layer.provide( @@ -1652,6 +1659,15 @@ const NodeHttpServerTestWithWsDeflate = HttpServer.layerTestClient.pipe( ), ); +const EMPTY_DEVICE_STATE: DeviceServiceState = { + hosts: [], + hostStatus: "idle", + devices: [], + sessions: [], + hubBasePath: DeviceService.DEVICE_HUB_ROUTE_PREFIX, + revision: 0, +}; + it.layer(NodeServices.layer)("server router seam", (it) => { it.effect("parks HTTP ingress until command readiness", () => Effect.gen(function* () { diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index fd8ee4a4f699..0a2b7bacef81 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -57,6 +57,8 @@ import * as TerminalManager from "./terminal/Manager.ts"; import * as McpHttpServer from "./mcp/McpHttpServer.ts"; import * as McpSessionRegistry from "./mcp/McpSessionRegistry.ts"; import * as PreviewAutomationBroker from "./mcp/PreviewAutomationBroker.ts"; +import * as DeviceService from "./device/DeviceService.ts"; +import { deviceHubProxyRouteLayer } from "./device/DeviceHubProxy.ts"; import * as PreviewManager from "./preview/Manager.ts"; import * as PortScanner from "./preview/PortScanner.ts"; import * as ProcessRunner from "./processRunner.ts"; @@ -382,6 +384,11 @@ const PreviewLayerLive = Layer.empty.pipe( Layer.provideMerge(PortScannerLayerLive), ); +const DeviceLayerLive = DeviceService.layer.pipe( + Layer.provide(ProcessRunner.layer), + Layer.provide(NetService.layer), +); + const WorkspaceEntriesLayerLive = WorkspaceEntries.layer.pipe(Layer.provide(WorkspacePaths.layer)); const WorkspaceFileSystemLayerLive = WorkspaceFileSystem.layer.pipe( @@ -465,7 +472,7 @@ const RuntimeCoreDependenciesLive = ReactorLayerLive.pipe( Layer.provideMerge(GitLayerLive), Layer.provideMerge(VcsLayerLive), Layer.provideMerge(ProviderRuntimeLayerLive), - Layer.provideMerge(Layer.mergeAll(TerminalLayerLive, PreviewLayerLive)), + Layer.provideMerge(Layer.mergeAll(TerminalLayerLive, PreviewLayerLive, DeviceLayerLive)), Layer.provideMerge(PersistenceLayerLive), // Both read a user-owned file out of the state directory and stream changes // to clients; neither depends on the other. @@ -549,6 +556,7 @@ export const makeRoutesLayer = Layer.mergeAll( otlpTracesProxyRouteLayer, assetRouteLayer, attachmentUploadRouteLayer, + deviceHubProxyRouteLayer, staticAndDevRouteLayer, websocketRpcRouteLayer, ), diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index 740be1330817..f415f4d9b06c 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -113,6 +113,7 @@ import * as ServerRuntimeStartup from "./serverRuntimeStartup.ts"; import * as ServerSettings from "./serverSettings.ts"; import * as TerminalManager from "./terminal/Manager.ts"; import * as PreviewAutomationBroker from "./mcp/PreviewAutomationBroker.ts"; +import * as DeviceService from "./device/DeviceService.ts"; import * as PreviewManager from "./preview/Manager.ts"; import { issueAssetUrl } from "./assets/AssetAccess.ts"; import { deletePendingAttachment, issueAttachmentUploadUrl } from "./assets/AttachmentUpload.ts"; @@ -519,6 +520,7 @@ const makeWsRpcLayer = ( const vcsStatusBroadcaster = yield* VcsStatusBroadcaster.VcsStatusBroadcaster; const terminalManager = yield* TerminalManager.TerminalManager; const previewManager = yield* PreviewManager.PreviewManager; + const deviceService = yield* DeviceService.DeviceService; const portDiscovery = yield* PortScanner.PortDiscovery; const providerRegistry = yield* ProviderRegistry.ProviderRegistry; const providerService = yield* ProviderService.ProviderService; @@ -2685,6 +2687,28 @@ const makeWsRpcLayer = ( observeRpcStream(WS_METHODS.subscribePreviewEvents, previewManager.events, { "rpc.aggregate": "preview", }), + [WS_METHODS.deviceList]: (_input) => + observeRpcEffect(WS_METHODS.deviceList, deviceService.list, { + "rpc.aggregate": "device", + }), + [WS_METHODS.deviceOpen]: (input) => + observeRpcEffect(WS_METHODS.deviceOpen, deviceService.open(input), { + "rpc.aggregate": "device", + }), + [WS_METHODS.deviceClose]: (input) => + observeRpcEffect(WS_METHODS.deviceClose, deviceService.close(input), { + "rpc.aggregate": "device", + }), + [WS_METHODS.deviceShutdown]: (input) => + observeRpcEffect(WS_METHODS.deviceShutdown, deviceService.shutdown(input), { + "rpc.aggregate": "device", + }), + [WS_METHODS.subscribeDeviceState]: (_input) => + observeRpcStream( + WS_METHODS.subscribeDeviceState, + DeviceService.stateStream(deviceService), + { "rpc.aggregate": "device" }, + ), [WS_METHODS.subscribeDiscoveredLocalServers]: (input) => observeRpcStream( WS_METHODS.subscribeDiscoveredLocalServers, diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index 0fbef88c81e1..62ede3749c58 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -203,6 +203,7 @@ import { PullRequestDetailGhost } from "./pullRequest/PullRequestGhosts"; import { PullRequestsUnavailableState } from "./pullRequest/PullRequestsUnavailableState"; import { RightPanelTabs } from "./RightPanelTabs"; import { AgentsPanel } from "./AgentsPanel"; +import { useDeviceState } from "~/state/device"; import { deriveAgentPanelModel, foldSubagentActivities, @@ -554,6 +555,9 @@ const PreviewPanel = lazy(() => import("./preview/PreviewPanel").then((module) => ({ default: module.PreviewPanel })), ); const DiffPanel = lazy(() => import("./DiffPanel")); +const DevicePanel = lazy(() => + import("./device/DevicePanel").then((module) => ({ default: module.DevicePanel })), +); const FilePreviewPanel = lazy(() => import("./files/FilePreviewPanel")); const EMPTY_PENDING_FILE_SURFACE_IDS: ReadonlySet = new Set(); const TYPE_TO_FOCUS_EDITABLE_SELECTOR = [ @@ -4127,6 +4131,26 @@ export default function ChatView(props: ChatViewProps) { if (!activeThreadRef) return; useRightPanelStore.getState().open(activeThreadRef, "agents"); }, [activeThreadRef]); + const addDeviceSurface = useCallback(() => { + if (!activeThreadRef || !isServerThread) return; + useRightPanelStore.getState().open(activeThreadRef, "device"); + }, [activeThreadRef, isServerThread]); + // An agent's `device_open` surfaces in every client the same way a + // `preview_open` does: the thread gains a device session and the panel + // opens on it. Closing the last session leaves the tab in place so the + // user keeps their picker; only new sessions raise the panel. + const { state: deviceState } = useDeviceState(activeThreadRef?.environmentId ?? null); + const threadDeviceSessionCount = activeThreadRef + ? deviceState.sessions.filter((session) => session.threadId === activeThreadRef.threadId).length + : 0; + const previousDeviceSessionCount = useRef(threadDeviceSessionCount); + useEffect(() => { + const previous = previousDeviceSessionCount.current; + previousDeviceSessionCount.current = threadDeviceSessionCount; + if (!activeThreadRef || threadDeviceSessionCount <= previous) return; + if (shouldUseRightPanelSheet) return; + useRightPanelStore.getState().open(activeThreadRef, "device"); + }, [activeThreadRef, shouldUseRightPanelSheet, threadDeviceSessionCount]); const openFileSurface = useCallback( (relativePath: string) => { if (!activeThreadRef || !activeProject) return; @@ -8069,6 +8093,15 @@ export default function ChatView(props: ChatViewProps) { environmentId={activeThreadRef?.environmentId ?? null} threadId={activeThreadRef?.threadId ?? null} /> + ) : renderedRightPanelSurface?.kind === "device" ? ( + + + ) : (renderedRightPanelSurface?.kind === "files" || renderedRightPanelSurface?.kind === "file") && ((activeProject && activeWorkspaceRoot) || @@ -8621,12 +8654,14 @@ export default function ChatView(props: ChatViewProps) { onAddFiles={addFilesSurface} onAddPullRequest={addPullRequestSurface} onAddAgents={addAgentsSurface} + onAddDevice={addDeviceSurface} browserAvailable={isPreviewSupportedInRuntime()} terminalAvailable={activeProject !== null} diffAvailable={isServerThread && isGitRepo} filesAvailable={activeProject !== null} pullRequestAvailable={pullRequestSurfaceAvailable} agentsAvailable + deviceAvailable={isServerThread} liveAgentCount={agentPanelModel.liveCount} > {rightPanelContent} @@ -8671,12 +8706,14 @@ export default function ChatView(props: ChatViewProps) { onAddFiles={addFilesSurface} onAddPullRequest={addPullRequestSurface} onAddAgents={addAgentsSurface} + onAddDevice={addDeviceSurface} browserAvailable={isPreviewSupportedInRuntime()} terminalAvailable={activeProject !== null} diffAvailable={isServerThread && isGitRepo} filesAvailable={activeProject !== null} pullRequestAvailable={pullRequestSurfaceAvailable} agentsAvailable + deviceAvailable={isServerThread} liveAgentCount={agentPanelModel.liveCount} > {rightPanelContent} diff --git a/apps/web/src/components/RightPanelTabs.test.tsx b/apps/web/src/components/RightPanelTabs.test.tsx index 81367d75580d..82ead6644d42 100644 --- a/apps/web/src/components/RightPanelTabs.test.tsx +++ b/apps/web/src/components/RightPanelTabs.test.tsx @@ -120,6 +120,7 @@ function renderTabs( onAddDiff={() => undefined} onAddFiles={() => undefined} onAddAgents={() => undefined} + onAddDevice={() => undefined} liveAgentCount={0} browserAvailable terminalAvailable={false} @@ -127,6 +128,7 @@ function renderTabs( filesAvailable={false} pullRequestAvailable={false} agentsAvailable={false} + deviceAvailable={false} >
content
, diff --git a/apps/web/src/components/RightPanelTabs.tsx b/apps/web/src/components/RightPanelTabs.tsx index e9dab0c9d2b4..b6b897b23a1e 100644 --- a/apps/web/src/components/RightPanelTabs.tsx +++ b/apps/web/src/components/RightPanelTabs.tsx @@ -8,6 +8,7 @@ import type { import { getTerminalLabel } from "@t3tools/shared/terminalLabels"; import { Bot, + Smartphone, ChevronDown, ChevronLeft, ChevronRight, @@ -105,12 +106,14 @@ interface RightPanelTabsProps { onAddFiles: () => void; onAddPullRequest: () => void; onAddAgents: () => void; + onAddDevice: () => void; browserAvailable: boolean; terminalAvailable: boolean; diffAvailable: boolean; filesAvailable: boolean; pullRequestAvailable: boolean; agentsAvailable: boolean; + deviceAvailable: boolean; pullRequestStatusSeeds?: Readonly>; /** Running + waiting subagents; badges the Agents card in the empty state. */ liveAgentCount: number; @@ -140,6 +143,7 @@ const SURFACE_DISABLED_REASONS = { diff: "Diff is only available for server threads in Git repositories.", pullRequest: "This thread's branch has no pull request yet.", agents: "Agents are only available from a thread.", + device: "Devices are only available from a server thread.", } as const; /** Overlays that must win over the launcher's letter shortcuts. */ @@ -162,6 +166,7 @@ const SURFACE_UNAVAILABLE_HINTS = { diff: "Available for Git repositories.", pullRequest: "No pull request on this branch yet.", agents: "Available from a thread.", + device: "Available from a server thread.", } as const; type TabContextMenuAction = @@ -299,12 +304,14 @@ function RightPanelEmptyState(props: { onAddFiles: () => void; onAddPullRequest: () => void; onAddAgents: () => void; + onAddDevice: () => void; browserAvailable: boolean; terminalAvailable: boolean; diffAvailable: boolean; filesAvailable: boolean; pullRequestAvailable: boolean; agentsAvailable: boolean; + deviceAvailable: boolean; liveAgentCount: number; }) { // -1 means no highlight: it only appears on hover or arrow use. @@ -371,6 +378,16 @@ function RightPanelEmptyState(props: { onClick: props.onAddAgents, badgeCount: props.liveAgentCount, }, + { + label: "Device", + description: "Watch an iOS Simulator or Android Emulator.", + icon: Smartphone, + shortcut: "M", + available: props.deviceAvailable, + disabledReason: SURFACE_UNAVAILABLE_HINTS.device, + onClick: props.onAddDevice, + badgeCount: 0, + }, ] as const; type SurfaceAction = (typeof actions)[number]; @@ -604,6 +621,8 @@ function surfaceTitle( return `#${surface.number}`; case "agents": return "Agents"; + case "device": + return "Device"; case "preview": { const snapshot = surface.resourceId ? sessions[surface.resourceId] : null; if (!snapshot || snapshot.navStatus._tag === "Idle") return "Browser"; @@ -685,6 +704,8 @@ function SurfaceIcon({ ); case "agents": return ; + case "device": + return ; } } @@ -814,6 +835,14 @@ export function RightPanelTabs(props: RightPanelTabsProps) { disabledReason: SURFACE_DISABLED_REASONS.agents, onClick: props.onAddAgents, }, + { + label: "Device", + icon: Smartphone, + shortcut: "M", + available: props.deviceAvailable, + disabledReason: SURFACE_DISABLED_REASONS.device, + onClick: props.onAddDevice, + }, ] as const; const handleAddSurfaceMenuKeyDown = (event: ReactKeyboardEvent) => { @@ -1251,12 +1280,14 @@ export function RightPanelTabs(props: RightPanelTabsProps) { onAddFiles={props.onAddFiles} onAddPullRequest={props.onAddPullRequest} onAddAgents={props.onAddAgents} + onAddDevice={props.onAddDevice} browserAvailable={props.browserAvailable} terminalAvailable={props.terminalAvailable} diffAvailable={props.diffAvailable} filesAvailable={props.filesAvailable} pullRequestAvailable={props.pullRequestAvailable} agentsAvailable={props.agentsAvailable} + deviceAvailable={props.deviceAvailable} liveAgentCount={props.liveAgentCount} /> ) : ( diff --git a/apps/web/src/components/chat/MessagesTimeline.logic.ts b/apps/web/src/components/chat/MessagesTimeline.logic.ts index a87531673506..badafd9c739d 100644 --- a/apps/web/src/components/chat/MessagesTimeline.logic.ts +++ b/apps/web/src/components/chat/MessagesTimeline.logic.ts @@ -337,7 +337,7 @@ export type MessagesTimelineRow = summaryKind: ToolGroupSummaryKind; toolSurface?: WorkLogEntry["toolSurface"]; toolIcon?: WorkLogEntry["toolIcon"]; - summaryToolIcon?: "browser" | "t3-code"; + summaryToolIcon?: "browser" | "device" | "t3-code"; hasFailure: boolean; } | { diff --git a/apps/web/src/components/chat/MessagesTimeline.tsx b/apps/web/src/components/chat/MessagesTimeline.tsx index 1d4abe39bf39..c05d7d0b8d0c 100644 --- a/apps/web/src/components/chat/MessagesTimeline.tsx +++ b/apps/web/src/components/chat/MessagesTimeline.tsx @@ -96,6 +96,7 @@ import { MousePointerClickIcon, PaintbrushIcon, SearchIcon, + SmartphoneIcon, SquarePenIcon, TerminalIcon, Undo2Icon, @@ -2794,6 +2795,7 @@ type WorkEntryIconName = | "check" | "circle-alert" | "computer" + | "device" | "eye" | "globe" | "hammer" @@ -3013,6 +3015,8 @@ function WorkEntryIcon({ name, className }: { name: WorkEntryIconName; className return ; case "computer": return ; + case "device": + return ; case "t3-code": return ; case "check": diff --git a/apps/web/src/components/device/DevicePanel.tsx b/apps/web/src/components/device/DevicePanel.tsx new file mode 100644 index 0000000000000000000000000000000000000000..669636607a918c8789c668268eedaa46a70a0c5f GIT binary patch literal 11038 zcmcgy>vG%174C06#hTMeNe=EG~51{HTA(#9q#x`<~0j>rIxIs=VK5wLMVkg}#YHeU=2}GRxP~dgNT^ zG;v-ArFKTE^*YG!Y2ZA}HadE{%5@OE(aY(Ul5rVow91O|G#aU&aYQsaA{^Am-*07)FSEPf)dO*~`-oF!4<8<= zhwAN08yXbsVpVz@Vz4Tzd>MrLP{B4898#(ToEoz;DzU3%7xjn2%n-0b7ex%VPSkB& zu2i5lz!!#6W~#tQxKdfJF0-suL7D+Z?yKkciqk6$M5-uBS@uGdI@+lY`xasSq9BD&1K5X#~Sy(VUXOE))~(*EA4dyN#`Qs*)^- zv@C{28}%ZOFmAz}G+UDxrn4Z=@3;q}rH;q(Y28$9!59+CUxGNPa*(kMk^+Y60Av;W z41l!G*%l@l0DZ%FHEyTjgrcmDUtTT8apxIl_TjDtylaO41D|TiKM*Lc9 zvt=qNxN&k24EKCmaUCUC=gKUeH%_=Ll-pf2pU0Ae523B)2W4EBFI}) z<}n4A?_iZi!#LSJj8hxT#K1X(%p$#v;RGk@sRK1A$KK-}YNBdjRtr;xg}r3fz8qSB zSwPF2qUnnECs z8{q3Wghz-Y@Y~@iJWXpHEaNj-ilm2ZY67-<z z9;e7eTw;fqQ@d#1#AF$GT^B^x3S75q9DNS}0OFANGI_>+U`W%(8%nr-SR!07_jOOt z9<=PFD`c%qb-{)r&=(kiqV?x z3pg%FTu{fFRiqhj5Nc5rE9|9_b1=Fp*v>{2h}|R9w#=pZ=T)(PLyWK;!pG%}koV_3 zP_ZPZxY7Y!rNIpXkN|JR9M< za}SH+f+QI&whv7B3SF?e(=2Rguc^1mb`k#*94*!(m%+$Byj!r980|eCsUVL74?3fh z#Yhly*hxqkmNMn;VKmoeYjf4oEd84o2<_bSS-^ubEpucvxJ)|d8#qlx$Av~DdS5{y zth$WUtc&Cz8qHxX8vGPmxSP)i>(7E?&sC+qE4_79>c~SqOw*-~a=}p>I&wWqANL+G zd@!lyXK;2829Ev;D%SVkdSOWL(vAu9U6KB!WY&&7>$khX53|!n&K@wl3 zFfhU%hD2Xfm-`Ww0kd_n)BA+ z8@h{Y`%ZLvh0-DAl8+$st}?ksVq%wGNTO_P&_0r=+nQJb%Je!j4zf^7zKp0I`x2Ef zM2+NLyQ*P{=@U;E5JfTW4!fFNVS|SqZbn)+V29v&vy8Ro1;=O7Lg@(BgecVT>E0w+ zdYqsPi-3$>kJPE2`lyW;g}@r4gp7SX`B^?9#*$qwg-W%tgKg01-2h`gbFpr3Y0+Lg zSV;?zJ5yzx3g~dOmU9Qod zPjQatUB-6Qf#wa(Z!{$b#cw4$H`{y4K}Y;n%BAP{zY{(0_~oI{&$LHOqtP;fd!M6y zg3ft*JlPFU)EC{n?xgl9_>|B(EF46Xd$lO~E_=8-MTapi9X;uVGIQD7;nAls8bslk zYU-ano};#b^MC5{WB=K%B)k4|K10@P+1|~7bB!Dw1z1Ia;;PBi?D}rjg$iY>g3<9O zwwcDACu%vi42c6sxz>BhaN8HQsxymi>$APm#+L1*yY_zV#d7v3vRpS2M=7nn%oOEq z3Bi1}*JP>m9Tr=Z%H*fJz|oqnRdq$cIVuNo97l4h9A=$>^6ynq#>;!RGVzt#cz=|* z&D1M6$|=_Svcr}v>a2{4KHct3N2=tT=C(`kP)6fj>SS&GAi~W>n8t5VK@GyVy!XC& z)JvbcCcSaDuY)_(5>Q}hbdwH)h9W0I6~3E6E>WG*QQuMH2JCKlp-U{LX26IVxS}Qs zi6L5R1cW=Tlz90g+eMJ0=`X5&ojzBK(XtSbZEIGbBPK$P5j`lBwN)8hQ-%o}L`v`X zQKm!wFIhBodp$wWBl4+@%pEN==%b?soL9jHG#Bvy;V`NNGx$J6!+fkN?FOH2!TMP`ma8kCmE z904#rFo0ld(9|KckV&a-1M1?Vt$^k|9t~0du(-w<&<5@!VhOi$@od2FgwXTZrfmq@ zp^B_!z!AcK_-;}nsUHu-fmJeOZVE|nsJ(mdvG^5aQPzULeblo>SmPiXTxYPpdVkKI z#FDGqm5m>7(<%(cVZDVC?mBNo+=n5f9a^XL7L>{hl*b0tVm?da{(;>ji(<+uX|L;E zzJzG&0*1%5CpzQv{+k+ti*onw;GpyN!Xz17-tIiU;PA|L|jAi(@g0(Jk`sI+6DcRg}|mF}fnvmXc`Oe>x>+(-L4VSs{of)ftO44a{eIjb)Jy z3+y*t8&Y_y>B8FsOFHW8X1EzKcvz;|1@l>(O?HFjo+Q65BI+$pVqA~W{*DH*TYSNJ zuj&5UCs*!6O7}4g@4zFUXL%mn&$snkEW7W_^nRY$^xsaL%Y&--7ixEezR>#%QYaLw z)mXZI)(q)M#(RF$HB}><89&PV( zzs%0@o<`$MK`>R9yzh0jzQn_aOH(hKC{LN8pdLcx6mSjSPqHvbG~V@X@c0v(sAx3- o?@7C;1#R$jyzxY?sKVb=#R~5{EKXC)NBh77FpO=9D99!J7g8gz+W-In literal 0 HcmV?d00001 diff --git a/apps/web/src/components/device/DeviceStreamView.tsx b/apps/web/src/components/device/DeviceStreamView.tsx new file mode 100644 index 000000000000..6e9e5849a5a8 --- /dev/null +++ b/apps/web/src/components/device/DeviceStreamView.tsx @@ -0,0 +1,205 @@ +import type { DevicePlatform, EnvironmentId } from "@t3tools/contracts"; +import { useEffect, useMemo, useRef, useState } from "react"; + +import { cn } from "~/lib/utils"; +import { refreshDeviceHubAccess, useDeviceHubAccess } from "~/state/device"; +import { Spinner } from "~/components/ui/spinner"; +import { + createDeviceStreamClient, + type DeviceHardwareButton, + type DeviceScreenSize, + type DeviceStreamClient, + type DeviceStreamStatus, +} from "./deviceStream"; + +export interface DeviceStreamHandle { + readonly pressButton: (button: DeviceHardwareButton) => void; + readonly rotate: () => void; +} + +/** + * The live device screen. Pointer events map onto normalized coordinates in + * the displayed frame and go to the device; keyboard input is forwarded while + * the surface is focused. `visible=false` tears the stream down so a hidden + * panel decodes nothing. + */ +export function DeviceStreamView(props: { + readonly environmentId: EnvironmentId; + readonly platform: DevicePlatform; + readonly deviceId: string; + readonly visible: boolean; + readonly onHandle?: (handle: DeviceStreamHandle | null) => void; + readonly onScreen?: (screen: DeviceScreenSize | null) => void; +}) { + const access = useDeviceHubAccess(props.environmentId); + const canvasRef = useRef(null); + const clientRef = useRef(null); + const [status, setStatus] = useState("connecting"); + const [detail, setDetail] = useState(undefined); + const [screen, setScreen] = useState(null); + const [mjpegUrl, setMjpegUrl] = useState(null); + const [mjpegGeneration, setMjpegGeneration] = useState(0); + const { onHandle, onScreen } = props; + + useEffect(() => { + const canvas = canvasRef.current; + if (!access || !canvas || !props.visible) { + setStatus("connecting"); + onHandle?.(null); + return; + } + const client = createDeviceStreamClient( + { platform: props.platform, deviceId: props.deviceId, access }, + canvas, + { + onStatus: (next, nextDetail) => { + setStatus(next); + setDetail(nextDetail); + }, + onScreen: (next) => { + setScreen(next); + onScreen?.(next); + }, + onUnauthorized: () => { + // A fresh ticket re-runs this effect through the access dependency. + refreshDeviceHubAccess(props.environmentId); + }, + }, + ); + clientRef.current = client; + setMjpegUrl(client.mjpegUrl); + setMjpegGeneration((generation) => generation + 1); + client.start(); + onHandle?.({ pressButton: client.pressButton, rotate: client.rotate }); + return () => { + client.stop(); + clientRef.current = null; + onHandle?.(null); + onScreen?.(null); + setScreen(null); + }; + }, [ + access, + onHandle, + onScreen, + props.deviceId, + props.environmentId, + props.platform, + props.visible, + ]); + + const aspect = useMemo(() => { + if (!screen) return props.platform === "ios" ? "9 / 19.5" : "9 / 20"; + const landscape = + screen.orientation === "landscape_left" || screen.orientation === "landscape_right"; + const w = landscape + ? Math.max(screen.width, screen.height) + : Math.min(screen.width, screen.height); + const h = landscape + ? Math.min(screen.width, screen.height) + : Math.max(screen.width, screen.height); + return `${w} / ${h}`; + }, [props.platform, screen]); + + // serve-sim streams the raw framebuffer; rotate the display for a device + // that reports landscape while its frames stay portrait. + const rotation = useMemo(() => { + if (props.platform !== "ios" || !screen || screen.width > screen.height) return 0; + switch (screen.orientation) { + case "landscape_left": + return -90; + case "landscape_right": + return 90; + case "portrait_upside_down": + return 180; + default: + return 0; + } + }, [props.platform, screen]); + + const pointerActive = useRef(false); + const normalizedPoint = (event: React.PointerEvent) => { + const rect = event.currentTarget.getBoundingClientRect(); + let x = (event.clientX - rect.left) / rect.width; + let y = (event.clientY - rect.top) / rect.height; + if (rotation === -90) [x, y] = [1 - y, x]; + else if (rotation === 90) [x, y] = [y, 1 - x]; + else if (rotation === 180) [x, y] = [1 - x, 1 - y]; + return { x: Math.min(1, Math.max(0, x)), y: Math.min(1, Math.max(0, y)) }; + }; + + return ( +
{ + if (event.metaKey && !["r", "R"].includes(event.key)) return; + event.preventDefault(); + clientRef.current?.sendKey(event.nativeEvent, "down"); + }} + onKeyUp={(event) => { + clientRef.current?.sendKey(event.nativeEvent, "up"); + }} + > +
{ + event.currentTarget.setPointerCapture(event.pointerId); + (event.currentTarget.parentElement as HTMLElement | null)?.focus(); + pointerActive.current = true; + const { x, y } = normalizedPoint(event); + clientRef.current?.sendTouch("begin", x, y); + }} + onPointerMove={(event) => { + if (!pointerActive.current) return; + const { x, y } = normalizedPoint(event); + clientRef.current?.sendTouch("move", x, y); + }} + onPointerUp={(event) => { + if (!pointerActive.current) return; + pointerActive.current = false; + const { x, y } = normalizedPoint(event); + clientRef.current?.sendTouch("end", x, y); + }} + onPointerCancel={(event) => { + if (!pointerActive.current) return; + pointerActive.current = false; + const { x, y } = normalizedPoint(event); + clientRef.current?.sendTouch("end", x, y); + }} + > + + {mjpegUrl ? ( + + ) : null} +
+ {status !== "streaming" ? ( +
+ {status === "connecting" ? : null} + {status === "error" ? (detail ?? "Stream failed.") : "Connecting to device…"} + {status === "connecting" && detail ? ( + {detail} + ) : null} +
+ ) : null} +
+ ); +} diff --git a/apps/web/src/components/device/deviceStream.test.ts b/apps/web/src/components/device/deviceStream.test.ts new file mode 100644 index 000000000000..25b7a53ba6f2 --- /dev/null +++ b/apps/web/src/components/device/deviceStream.test.ts @@ -0,0 +1,68 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { AvccDemuxer, avcCodecString, parseSemuPacket, scanAccessUnit } from "./deviceStream"; + +const envelope = (tag: number, payload: number[]) => { + const length = 1 + payload.length; + return [ + (length >>> 24) & 0xff, + (length >>> 16) & 0xff, + (length >>> 8) & 0xff, + length & 0xff, + tag, + ...payload, + ]; +}; + +describe("AvccDemuxer", () => { + it("reassembles envelopes split across reads", () => { + const demuxer = new AvccDemuxer(); + const bytes = new Uint8Array([ + ...envelope(1, [1, 0x64, 0x00, 0x1f]), + ...envelope(2, [9, 9, 9]), + ...envelope(0x7f, [0]), + ...envelope(3, [4]), + ]); + const first = demuxer.push(bytes.subarray(0, 7)); + const rest = demuxer.push(bytes.subarray(7)); + const chunks = [...first, ...rest]; + expect(chunks.map((chunk) => chunk.type)).toEqual(["description", "keyframe", "delta"]); + expect(Array.from(chunks[0]!.payload)).toEqual([1, 0x64, 0x00, 0x1f]); + expect(Array.from(chunks[1]!.payload)).toEqual([9, 9, 9]); + }); + + it("derives the WebCodecs codec string from the avcC record", () => { + expect(avcCodecString(new Uint8Array([1, 0x64, 0x00, 0x1f]))).toBe("avc1.64001f"); + expect(avcCodecString(new Uint8Array([1]))).toBe("avc1.42E01E"); + }); +}); + +describe("serve-emu frames", () => { + it("strips the SEMU header and reads the keyframe flag and timestamp", () => { + const buffer = new ArrayBuffer(16 + 3); + const view = new DataView(buffer); + view.setUint32(0, 0x53454d55); + view.setUint8(4, 1); + view.setUint8(5, 1); + view.setBigUint64(8, 123456n); + new Uint8Array(buffer).set([7, 8, 9], 16); + const packet = parseSemuPacket(buffer); + expect(packet.isKey).toBe(true); + expect(packet.timestamp).toBe(123456); + expect(Array.from(packet.data)).toEqual([7, 8, 9]); + }); + + it("treats a frame without the header as raw data", () => { + const packet = parseSemuPacket(new Uint8Array([0, 0, 1, 0x65]).buffer); + expect(packet.isKey).toBeNull(); + expect(packet.data.length).toBe(4); + }); + + it("finds the SPS and IDR NAL units in an Annex-B access unit", () => { + const unit = new Uint8Array([0, 0, 0, 1, 0x67, 0x64, 0x00, 0x1f, 0, 0, 1, 0x65, 0xaa]); + const scanned = scanAccessUnit(unit); + expect(scanned.isKey).toBe(true); + expect(scanned.sps && avcCodecString(scanned.sps)).toBe("avc1.64001f"); + expect(scanAccessUnit(new Uint8Array([0, 0, 1, 0x41, 0x00])).isKey).toBe(false); + }); +}); diff --git a/apps/web/src/components/device/deviceStream.ts b/apps/web/src/components/device/deviceStream.ts new file mode 100644 index 000000000000..a7f5573d297d --- /dev/null +++ b/apps/web/src/components/device/deviceStream.ts @@ -0,0 +1,599 @@ +/** + * Framework-free client for expo-device-hub's per-device streams, reached + * through the T3 proxy. One class handles both platforms because the hub + * vendors two servers with different wire formats: + * + * - iOS (serve-sim): video is an HTTP `stream.avcc` body of length-prefixed + * envelopes (`u32be length, u8 tag, payload`; tag 1 avcC description, + * 2 keyframe, 3 delta, 4 JPEG seed) decoded with WebCodecs; input goes over + * `helper/ws?device=` as `[tag][json]` packets. When WebCodecs is + * unavailable (plain-http remote origins) the MJPEG endpoint is used as an + * `` source instead. + * - Android (serve-emu): one WebSocket at `ws?device=&frame-meta=1` + * carries H.264 access units prefixed with a 16-byte "SEMU" header + * (magic, version, key flag, pts) and accepts JSON gestures upstream. + * + * The decoder only runs while frames arrive and the viewer is attached; a + * hidden panel calls `stop()` so an idle device costs nothing on the GPU. + */ +import type { DeviceHubAccess } from "@t3tools/client-runtime/state/deviceHubAccess"; +import { withDeviceHubQuery } from "@t3tools/client-runtime/state/deviceHubAccess"; +import type { DevicePlatform } from "@t3tools/contracts"; + +export type DeviceStreamStatus = "connecting" | "streaming" | "error"; + +export interface DeviceScreenSize { + readonly width: number; + readonly height: number; + readonly orientation: "portrait" | "portrait_upside_down" | "landscape_left" | "landscape_right"; +} + +export interface DeviceStreamEvents { + readonly onStatus: (status: DeviceStreamStatus, detail?: string) => void; + readonly onScreen: (screen: DeviceScreenSize) => void; + /** The proxy rejected the credential; the owner should refresh access and reconnect. */ + readonly onUnauthorized: () => void; +} + +export interface DeviceStreamTarget { + readonly platform: DevicePlatform; + readonly deviceId: string; + readonly access: DeviceHubAccess; +} + +export type DeviceHardwareButton = "home" | "back" | "recents" | "power" | "appSwitcher"; + +const RETRY_DELAY_MS = 1_000; +const FRAME_DURATION_US = 16_667; +const SEMU_MAGIC = 0x53454d55; +const SEMU_HEADER_BYTES = 16; +const SEMU_FLAG_KEY = 1; +const SOFT_DECODE_QUEUE = 8; + +// serve-sim binary WS message tags (browser -> helper). +const IOS_MSG_TOUCH = 0x03; +const IOS_MSG_BUTTON = 0x04; +const IOS_MSG_KEY = 0x06; +const IOS_MSG_ORIENTATION = 0x07; +const IOS_MSG_HARDWARE_KEYBOARD = 0x0d; +// helper -> browser. +const IOS_TAG_SCREEN_CONFIG = 0x82; + +const encoder = new TextEncoder(); +const decoder = new TextDecoder(); + +export const isWebCodecsSupported = (): boolean => + typeof globalThis !== "undefined" && + "VideoDecoder" in globalThis && + "EncodedVideoChunk" in globalThis; + +function taggedJson(tag: number, payload: unknown): Uint8Array { + const json = encoder.encode(JSON.stringify(payload)); + const out = new Uint8Array(1 + json.length); + out[0] = tag; + out.set(json, 1); + return out; +} + +/** Build the WebCodecs `avc1.PPCCLL` string from an avcC record or an SPS NAL. */ +export function avcCodecString(bytes: Uint8Array): string { + if (bytes.length < 4) return "avc1.42E01E"; + const hex = (byte: number) => byte.toString(16).padStart(2, "0"); + return `avc1.${hex(bytes[1]!)}${hex(bytes[2]!)}${hex(bytes[3]!)}`; +} + +/** Split serve-emu's SEMU-framed message into metadata and the Annex-B payload. */ +export function parseSemuPacket(raw: ArrayBuffer): { + readonly data: Uint8Array; + readonly isKey: boolean | null; + readonly timestamp: number | null; +} { + const bytes = new Uint8Array(raw); + if (bytes.byteLength > SEMU_HEADER_BYTES) { + const view = new DataView(raw, 0, SEMU_HEADER_BYTES); + if (view.getUint32(0, false) === SEMU_MAGIC && view.getUint8(4) === 1) { + const pts = view.getBigUint64(8, false); + return { + data: bytes.subarray(SEMU_HEADER_BYTES), + isKey: (view.getUint8(5) & SEMU_FLAG_KEY) !== 0, + timestamp: pts <= BigInt(Number.MAX_SAFE_INTEGER) ? Number(pts) : null, + }; + } + } + return { data: bytes, isKey: null, timestamp: null }; +} + +/** Walk an Annex-B access unit for its keyframe flag and SPS bytes. */ +export function scanAccessUnit(buf: Uint8Array): { isKey: boolean; sps: Uint8Array | null } { + let isKey = false; + let sps: Uint8Array | null = null; + const len = buf.length; + let i = 0; + while (i + 2 < len) { + if (buf[i] === 0 && buf[i + 1] === 0) { + let codeLen = 0; + if (buf[i + 2] === 1) codeLen = 3; + else if (i + 3 < len && buf[i + 2] === 0 && buf[i + 3] === 1) codeLen = 4; + if (codeLen) { + const nalType = buf[i + codeLen]! & 0x1f; + if (nalType === 7 && !sps) sps = buf.subarray(i + codeLen); + if (nalType === 5) isKey = true; + i += codeLen + 1; + continue; + } + } + i++; + } + return { isKey, sps }; +} + +export type AvccChunk = { + readonly type: "description" | "keyframe" | "delta" | "seed"; + readonly payload: Uint8Array; +}; + +const AVCC_TAGS: Record = { + 1: "description", + 2: "keyframe", + 3: "delta", + 4: "seed", +}; + +/** Turns a fragmented AVCC byte stream into complete envelopes. */ +export class AvccDemuxer { + private buffer = new Uint8Array(64 * 1024); + private length = 0; + + push(bytes: Uint8Array): AvccChunk[] { + if (this.length + bytes.length > this.buffer.length) { + let capacity = this.buffer.length; + while (capacity < this.length + bytes.length) capacity *= 2; + const grown = new Uint8Array(capacity); + grown.set(this.buffer.subarray(0, this.length)); + this.buffer = grown; + } + this.buffer.set(bytes, this.length); + this.length += bytes.length; + + const chunks: AvccChunk[] = []; + let offset = 0; + while (this.length - offset >= 4) { + const view = new DataView(this.buffer.buffer, this.buffer.byteOffset + offset, 4); + const frameLength = view.getUint32(0, false); + if (this.length - offset - 4 < frameLength) break; + if (frameLength >= 1) { + const type = AVCC_TAGS[this.buffer[offset + 4]!]; + if (type) { + chunks.push({ type, payload: this.buffer.slice(offset + 5, offset + 4 + frameLength) }); + } + } + offset += 4 + frameLength; + } + if (offset > 0) { + this.buffer.copyWithin(0, offset, this.length); + this.length -= offset; + } + return chunks; + } + + reset(): void { + this.length = 0; + } +} + +export interface DeviceStreamClient { + readonly start: () => void; + readonly stop: () => void; + /** Normalized 0..1 coordinates in the displayed frame. */ + readonly sendTouch: (phase: "begin" | "move" | "end", x: number, y: number) => void; + readonly sendKey: (event: KeyboardEvent, phase: "down" | "up") => void; + readonly pressButton: (button: DeviceHardwareButton) => void; + readonly rotate: () => void; + /** MJPEG fallback source when WebCodecs is unavailable on iOS, else null. */ + readonly mjpegUrl: string | null; +} + +const HID_USAGE_BY_CODE: Readonly> = { + Enter: 0x28, + Escape: 0x29, + Backspace: 0x2a, + Tab: 0x2b, + Space: 0x2c, + Minus: 0x2d, + Equal: 0x2e, + BracketLeft: 0x2f, + BracketRight: 0x30, + Backslash: 0x31, + Semicolon: 0x33, + Quote: 0x34, + Backquote: 0x35, + Comma: 0x36, + Period: 0x37, + Slash: 0x38, + Delete: 0x4c, + ArrowRight: 0x4f, + ArrowLeft: 0x50, + ArrowDown: 0x51, + ArrowUp: 0x52, + ControlLeft: 0xe0, + ShiftLeft: 0xe1, + AltLeft: 0xe2, + MetaLeft: 0xe3, + ControlRight: 0xe4, + ShiftRight: 0xe5, + AltRight: 0xe6, + MetaRight: 0xe7, +}; + +function hidUsageForCode(code: string): number | null { + if (/^Key[A-Z]$/.test(code)) return 0x04 + (code.charCodeAt(3) - 65); + if (/^Digit[1-9]$/.test(code)) return 0x1e + (code.charCodeAt(5) - 49); + if (code === "Digit0") return 0x27; + return HID_USAGE_BY_CODE[code] ?? null; +} + +const ANDROID_KEYCODE_BY_KEY: Readonly> = { + ArrowUp: 19, + ArrowDown: 20, + ArrowLeft: 21, + ArrowRight: 22, + Tab: 61, + Enter: 66, + Backspace: 67, + Delete: 112, + Home: 122, + End: 123, + PageUp: 92, + PageDown: 93, +}; + +const IOS_ORIENTATIONS: ReadonlyArray = [ + "portrait", + "landscape_left", + "portrait_upside_down", + "landscape_right", +]; + +export function createDeviceStreamClient( + target: DeviceStreamTarget, + canvas: HTMLCanvasElement, + events: DeviceStreamEvents, +): DeviceStreamClient { + const { access, platform, deviceId } = target; + const vendor = platform === "ios" ? "/vendor/serve-sim" : "/vendor/serve-emu"; + const device = encodeURIComponent(deviceId); + const httpUrl = (path: string) => + withDeviceHubQuery(`${access.httpBase}${vendor}${path}`, access); + const wsUrl = (path: string) => withDeviceHubQuery(`${access.wsBase}${vendor}${path}`, access); + const useWebCodecs = isWebCodecsSupported(); + + let stopped = true; + let socket: WebSocket | null = null; + let controller: AbortController | null = null; + let retryTimer: ReturnType | null = null; + let videoDecoder: VideoDecoder | null = null; + let timestamp = 0; + let awaitingKeyframe = true; + let screen: DeviceScreenSize | null = null; + let firstFrame = false; + + const setStatus = (status: DeviceStreamStatus, detail?: string) => { + if (!stopped) events.onStatus(status, detail); + }; + + const paint = (source: CanvasImageSource, width: number, height: number) => { + if (stopped) return; + if (canvas.width !== width || canvas.height !== height) { + canvas.width = width; + canvas.height = height; + if (platform === "android") { + screen = { width, height, orientation: width > height ? "landscape_left" : "portrait" }; + events.onScreen(screen); + } + } + canvas.getContext("2d")?.drawImage(source, 0, 0, width, height); + if (!firstFrame) { + firstFrame = true; + setStatus("streaming"); + } + }; + + const closeDecoder = () => { + try { + videoDecoder?.close(); + } catch { + // Already closed. + } + videoDecoder = null; + awaitingKeyframe = true; + }; + + const makeDecoder = () => + new VideoDecoder({ + output: (frame) => { + try { + paint(frame, frame.displayWidth, frame.displayHeight); + } finally { + frame.close(); + } + }, + error: () => { + closeDecoder(); + requestKeyframe(); + }, + }); + + const configureDecoder = (config: VideoDecoderConfig) => { + if (!videoDecoder || videoDecoder.state === "closed") videoDecoder = makeDecoder(); + try { + videoDecoder.configure({ + ...config, + optimizeForLatency: true, + hardwareAcceleration: "prefer-hardware", + }); + return true; + } catch (cause) { + setStatus("error", `Video decoder: ${(cause as Error).message}`); + return false; + } + }; + + const decode = (isKey: boolean, data: Uint8Array, pts?: number | null) => { + if (!videoDecoder || videoDecoder.state !== "configured") return; + if (awaitingKeyframe) { + if (!isKey) return; + awaitingKeyframe = false; + } + if (videoDecoder.decodeQueueSize > SOFT_DECODE_QUEUE) { + closeDecoder(); + requestKeyframe(); + return; + } + try { + videoDecoder.decode( + new EncodedVideoChunk({ + type: isKey ? "key" : "delta", + timestamp: pts ?? timestamp, + data, + }), + ); + timestamp += FRAME_DURATION_US; + } catch { + closeDecoder(); + requestKeyframe(); + } + }; + + const requestKeyframe = () => { + if (platform === "android" && socket?.readyState === WebSocket.OPEN) { + socket.send(JSON.stringify({ type: "reset-video", ack: false })); + } + }; + + const scheduleRetry = (run: () => void) => { + if (stopped || retryTimer) return; + retryTimer = setTimeout(() => { + retryTimer = null; + run(); + }, RETRY_DELAY_MS); + }; + + const handleUnauthorized = () => { + stop(); + events.onUnauthorized(); + }; + + // iOS video: fetch the AVCC body and demux into the decoder. + const readIosVideo = async () => { + const demuxer = new AvccDemuxer(); + controller = new AbortController(); + try { + const response = await fetch(httpUrl(`/helper/${device}/stream.avcc`), { + signal: controller.signal, + credentials: access.credentials ? "include" : "same-origin", + }); + if (response.status === 401 || response.status === 403) return handleUnauthorized(); + if (!response.ok || !response.body) throw new Error(`stream ${response.status}`); + const reader = response.body.getReader(); + for (;;) { + const { done, value } = await reader.read(); + if (done || stopped) break; + for (const chunk of demuxer.push(value)) { + switch (chunk.type) { + case "seed": + void createImageBitmap(new Blob([chunk.payload as BlobPart], { type: "image/jpeg" })) + .then((bitmap) => { + paint(bitmap, bitmap.width, bitmap.height); + bitmap.close(); + }) + .catch(() => {}); + break; + case "description": + awaitingKeyframe = true; + configureDecoder({ + codec: avcCodecString(chunk.payload), + description: chunk.payload, + }); + break; + case "keyframe": + case "delta": + decode(chunk.type === "keyframe", chunk.payload); + break; + } + } + } + } catch (cause) { + if (stopped) return; + setStatus("connecting", (cause as Error).message); + } + if (!stopped) scheduleRetry(() => void readIosVideo()); + }; + + // iOS input socket; also carries the screen config the helper pushes. + const connectIosInput = () => { + if (stopped) return; + const ws = new WebSocket(wsUrl(`/helper/ws?device=${device}`)); + ws.binaryType = "arraybuffer"; + socket = ws; + ws.onopen = () => ws.send(taggedJson(IOS_MSG_HARDWARE_KEYBOARD, { enabled: false })); + ws.onmessage = (event) => { + if (!(event.data instanceof ArrayBuffer)) return; + const bytes = new Uint8Array(event.data); + if (bytes.length < 1 || bytes[0] !== IOS_TAG_SCREEN_CONFIG) return; + try { + const config = JSON.parse(decoder.decode(bytes.subarray(1))) as DeviceScreenSize; + if (config.width > 0 && config.height > 0) { + screen = config; + events.onScreen(config); + } + } catch { + // Ignore malformed config frames. + } + }; + ws.onclose = (event) => { + if (socket === ws) socket = null; + if (event.code === 1008 || event.code === 4401) return handleUnauthorized(); + scheduleRetry(connectIosInput); + }; + ws.onerror = () => ws.close(); + }; + + // Android: one socket for video and input. + const connectAndroid = () => { + if (stopped) return; + const ws = new WebSocket(wsUrl(`/ws?device=${device}&frame-meta=1`)); + ws.binaryType = "arraybuffer"; + socket = ws; + ws.onopen = () => setStatus("connecting"); + ws.onmessage = (event) => { + if (!(event.data instanceof ArrayBuffer)) return; + const packet = parseSemuPacket(event.data); + const needsScan = + packet.isKey === null || + (packet.isKey && (!videoDecoder || videoDecoder.state !== "configured")); + const scanned = needsScan ? scanAccessUnit(packet.data) : null; + const isKey = packet.isKey ?? scanned?.isKey ?? false; + if (scanned?.sps && (!videoDecoder || videoDecoder.state !== "configured")) { + if (!configureDecoder({ codec: avcCodecString(scanned.sps) })) return; + awaitingKeyframe = true; + } + if (!videoDecoder || videoDecoder.state !== "configured") { + if (!isKey) requestKeyframe(); + return; + } + decode(isKey, packet.data, packet.timestamp); + }; + ws.onclose = (event) => { + if (socket === ws) socket = null; + closeDecoder(); + if (event.code === 1008 || event.code === 4401) return handleUnauthorized(); + if (!stopped) { + setStatus("connecting", event.reason || undefined); + scheduleRetry(connectAndroid); + } + }; + ws.onerror = () => ws.close(); + }; + + const start = () => { + if (!stopped) return; + stopped = false; + firstFrame = false; + events.onStatus("connecting"); + if (platform === "ios") { + connectIosInput(); + if (useWebCodecs) void readIosVideo(); + else setStatus("streaming"); + } else if (useWebCodecs) { + connectAndroid(); + } else { + setStatus("error", "This browser cannot decode the Android stream (WebCodecs unavailable)."); + } + }; + + const stop = () => { + if (stopped) return; + stopped = true; + if (retryTimer) clearTimeout(retryTimer); + retryTimer = null; + controller?.abort(); + controller = null; + socket?.close(); + socket = null; + closeDecoder(); + }; + + const send = (payload: Uint8Array | string) => { + if (socket?.readyState === WebSocket.OPEN) socket.send(payload); + }; + + const rawPoint = (x: number, y: number) => { + // serve-sim streams the raw framebuffer; rotated devices need input + // remapped into that raw space. + if (platform !== "ios" || !screen || screen.width > screen.height) return { x, y }; + switch (screen.orientation) { + case "landscape_left": + return { x: y, y: 1 - x }; + case "landscape_right": + return { x: 1 - y, y: x }; + case "portrait_upside_down": + return { x: 1 - x, y: 1 - y }; + default: + return { x, y }; + } + }; + + return { + start, + stop, + mjpegUrl: + platform === "ios" && !useWebCodecs ? httpUrl(`/helper/${device}/stream.mjpeg`) : null, + sendTouch: (phase, x, y) => { + if (platform === "ios") { + send(taggedJson(IOS_MSG_TOUCH, { type: phase, ...rawPoint(x, y) })); + return; + } + const action = phase === "begin" ? "down" : phase === "move" ? "move" : "up"; + send(JSON.stringify({ type: "touch", action, x, y })); + }, + sendKey: (event, phase) => { + if (platform === "ios") { + const usage = hidUsageForCode(event.code); + if (usage !== null) send(taggedJson(IOS_MSG_KEY, { type: phase, usage })); + return; + } + if (phase !== "down") return; + if (event.key === "Escape") return send(JSON.stringify({ type: "back" })); + const keycode = ANDROID_KEYCODE_BY_KEY[event.key]; + if (keycode !== undefined) return send(JSON.stringify({ type: "key", keycode })); + if (event.key.length === 1 && !event.metaKey && !event.ctrlKey) { + send(JSON.stringify({ type: "text", text: event.key })); + } + }, + pressButton: (button) => { + if (platform === "ios") { + const name = + button === "home" + ? "home" + : button === "appSwitcher" + ? "app_switcher" + : button === "power" + ? "lock" + : null; + if (name) send(taggedJson(IOS_MSG_BUTTON, { button: name })); + return; + } + const type = button === "appSwitcher" ? "recents" : button; + if (type === "home" || type === "back" || type === "recents" || type === "power") { + send(JSON.stringify({ type })); + } + }, + rotate: () => { + if (platform !== "ios") return; + const current = screen?.orientation ?? "portrait"; + const next = + IOS_ORIENTATIONS[(IOS_ORIENTATIONS.indexOf(current) + 1) % IOS_ORIENTATIONS.length]!; + send(taggedJson(IOS_MSG_ORIENTATION, { orientation: next })); + }, + }; +} diff --git a/apps/web/src/components/settings/ProjectDefaultsSettings.tsx b/apps/web/src/components/settings/ProjectDefaultsSettings.tsx index 938000e01002..8de434d837e8 100644 --- a/apps/web/src/components/settings/ProjectDefaultsSettings.tsx +++ b/apps/web/src/components/settings/ProjectDefaultsSettings.tsx @@ -96,6 +96,11 @@ export function ProjectDefaultsSettings({ target.serverConfig?.settings.enableAgentBrowserAccess !== serverSettings.enableAgentBrowserAccess, ); + const mixedDevice = targets.some( + (target) => + target.serverConfig?.settings.enableAgentDeviceAccess !== + serverSettings.enableAgentDeviceAccess, + ); const disabled = (key: keyof ServerSettingsPatch) => targets.length === 0 || saving.has(key); const mixedAutoPull = targets.some( (target) => target.serverConfig?.settings.defaultAutoPull !== serverSettings.defaultAutoPull, @@ -388,6 +393,58 @@ export function ProjectDefaultsSettings({ } /> + + void save({ + enableAgentDeviceAccess: DEFAULT_SERVER_SETTINGS.enableAgentDeviceAccess, + }) + } + /> + ) : null + } + control={ + + } + /> void) { ...(settings.enableAgentBrowserAccess !== DEFAULT_UNIFIED_SETTINGS.enableAgentBrowserAccess ? ["Agent browser access"] : []), + ...(settings.enableAgentDeviceAccess !== DEFAULT_UNIFIED_SETTINGS.enableAgentDeviceAccess + ? ["Agent device access"] + : []), ], [ isTextGenerationModelDirty, @@ -598,6 +601,7 @@ export function useSettingsRestore(onRestored?: () => void) { settings.browserAutoShowFloatingPreview, settings.appearanceContrast, settings.enableAgentBrowserAccess, + settings.enableAgentDeviceAccess, settings.confirmQuit, settings.confirmThreadArchive, settings.confirmThreadDelete, @@ -749,6 +753,7 @@ export function useSettingsRestore(onRestored?: () => void) { // name, so a user restoring defaults is told the agent regains access // rather than discovering it later. enableAgentBrowserAccess: DEFAULT_UNIFIED_SETTINGS.enableAgentBrowserAccess, + enableAgentDeviceAccess: DEFAULT_UNIFIED_SETTINGS.enableAgentDeviceAccess, }); onRestored?.(); }, [ diff --git a/apps/web/src/components/settings/settingsSearch.ts b/apps/web/src/components/settings/settingsSearch.ts index e32baf6f3987..0fe040460832 100644 --- a/apps/web/src/components/settings/settingsSearch.ts +++ b/apps/web/src/components/settings/settingsSearch.ts @@ -396,6 +396,12 @@ export const SETTINGS_SEARCH_ITEMS = [ to: "/settings/projects", searchTerms: ["allow open drive preview tools sessions"], }, + { + id: "agent-device-access", + title: "Agent device access", + to: "/settings/projects", + searchTerms: ["allow simulator emulator ios android drive tools sessions"], + }, { id: "browser-profiles", title: "Browser profiles", diff --git a/apps/web/src/rightPanelStore.ts b/apps/web/src/rightPanelStore.ts index d167afeddb7f..bb50d200f377 100644 --- a/apps/web/src/rightPanelStore.ts +++ b/apps/web/src/rightPanelStore.ts @@ -24,6 +24,7 @@ const RIGHT_PANEL_KINDS = [ "files", "file", "preview", + "device", "terminal", "pull-request", "agents", @@ -33,6 +34,12 @@ export type RightPanelKind = (typeof RIGHT_PANEL_KINDS)[number]; export type RightPanelSurface = | { id: `browser:${string}`; kind: "preview"; resourceId: string } | { id: "browser:new"; kind: "preview"; resourceId: null } + /** + * One Device tab per thread. The tab is the surface; which device it shows + * comes from the thread's server-side device sessions, so an agent opening a + * device from another client lands in the same tab. + */ + | { id: "device"; kind: "device" } | { id: `terminal:${string}`; kind: "terminal"; @@ -78,7 +85,8 @@ const RIGHT_PANEL_STORAGE_KEY = "t3code:right-panel-state:v2"; // v9 removed the "plan" surface kind (plans render inline in the transcript). // v10 keys pull-request surfaces by reference instead of a singleton tab. // v11 stops persisting the pull-request list's shared panel, so a restart opens the page fresh. -const RIGHT_PANEL_STORAGE_VERSION = 11; +// v12 adds the device surface. +const RIGHT_PANEL_STORAGE_VERSION = 12; /** A fixed workspace-level ref: each PR surface carries its own real environment. */ export const PULL_REQUESTS_PANEL_REF = scopeThreadRef( @@ -171,6 +179,8 @@ const singletonSurface = ( return { id: "files", kind }; case "agents": return { id: "agents", kind }; + case "device": + return { id: "device", kind }; } }; diff --git a/apps/web/src/routes/_chat.pull-requests.tsx b/apps/web/src/routes/_chat.pull-requests.tsx index 99986567cbfa..428a2f6dcfc6 100644 --- a/apps/web/src/routes/_chat.pull-requests.tsx +++ b/apps/web/src/routes/_chat.pull-requests.tsx @@ -1924,12 +1924,14 @@ function PullRequestsRouteView() { onAddFiles={() => undefined} onAddPullRequest={() => undefined} onAddAgents={() => undefined} + onAddDevice={() => undefined} browserAvailable={false} terminalAvailable={false} diffAvailable={false} filesAvailable={false} pullRequestAvailable={false} agentsAvailable={false} + deviceAvailable={false} liveAgentCount={0} pullRequestStatusSeeds={listedPullRequestTabStatuses} > diff --git a/apps/web/src/state/device.ts b/apps/web/src/state/device.ts new file mode 100644 index 000000000000..1c9bcd1aa993 --- /dev/null +++ b/apps/web/src/state/device.ts @@ -0,0 +1,68 @@ +import { useAtomValue } from "@effect/atom-react"; +import { createDeviceEnvironmentAtoms } from "@t3tools/client-runtime/state/device"; +import { + type DeviceHubAccess, + resolveDeviceHubAccess, +} from "@t3tools/client-runtime/state/deviceHubAccess"; +import type { DeviceServiceState, EnvironmentId } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; +import { AsyncResult, Atom } from "effect/unstable/reactivity"; + +import { connectionAtomRuntime } from "../connection/runtime"; +import { appAtomRegistry } from "../rpc/atomRegistry"; +import { environmentSession } from "./session"; +import { useEnvironmentQuery } from "./query"; + +export const deviceEnvironment = createDeviceEnvironmentAtoms(connectionAtomRuntime); + +const EMPTY_DEVICE_STATE: DeviceServiceState = { + hosts: [], + hostStatus: "idle", + devices: [], + sessions: [], + hubBasePath: "/api/device-hub", + revision: 0, +}; + +export function useDeviceState(environmentId: EnvironmentId | null): { + readonly state: DeviceServiceState; + readonly loaded: boolean; +} { + const query = useEnvironmentQuery( + environmentId === null ? null : deviceEnvironment.state({ environmentId, input: {} }), + ); + return { state: query.data ?? EMPTY_DEVICE_STATE, loaded: query.data !== undefined }; +} + +/** + * Hub access for one environment. Bearer and DPoP connections mint a ticket + * here; a stream that gets a 401 back refreshes this atom and reconnects. + * Keyed on the prepared connection so a re-pair produces new credentials. + */ +const deviceHubAccessAtom = Atom.family((environmentId: EnvironmentId) => + connectionAtomRuntime + .atom((get) => { + const prepared = Option.getOrNull( + get(environmentSession.preparedConnectionValueAtom(environmentId)), + ); + if (prepared === null) return Effect.never; + return resolveDeviceHubAccess({ prepared, hubBasePath: EMPTY_DEVICE_STATE.hubBasePath }); + }) + .pipe(Atom.setIdleTTL(60_000), Atom.withLabel(`device-hub-access:${environmentId}`)), +); + +export function useDeviceHubAccess(environmentId: EnvironmentId | null): DeviceHubAccess | null { + const result = useAtomValue( + environmentId === null ? EMPTY_ACCESS_ATOM : deviceHubAccessAtom(environmentId), + ); + return AsyncResult.isSuccess(result) ? result.value : null; +} + +const EMPTY_ACCESS_ATOM = Atom.make(AsyncResult.initial()).pipe( + Atom.withLabel("device-hub-access:empty"), +); + +export function refreshDeviceHubAccess(environmentId: EnvironmentId): void { + appAtomRegistry.refresh(deviceHubAccessAtom(environmentId)); +} diff --git a/packages/client-runtime/package.json b/packages/client-runtime/package.json index 368585556cfe..6eb62eaa44e2 100644 --- a/packages/client-runtime/package.json +++ b/packages/client-runtime/package.json @@ -131,6 +131,14 @@ "types": "./src/state/presentation.ts", "default": "./src/state/presentation.ts" }, + "./state/device": { + "types": "./src/state/device.ts", + "default": "./src/state/device.ts" + }, + "./state/deviceHubAccess": { + "types": "./src/state/deviceHubAccess.ts", + "default": "./src/state/deviceHubAccess.ts" + }, "./state/preview": { "types": "./src/state/preview.ts", "default": "./src/state/preview.ts" diff --git a/packages/client-runtime/src/rpc/client.ts b/packages/client-runtime/src/rpc/client.ts index 4ddfb9c4160e..cfabaa00c0b7 100644 --- a/packages/client-runtime/src/rpc/client.ts +++ b/packages/client-runtime/src/rpc/client.ts @@ -51,6 +51,7 @@ export type EnvironmentSubscriptionRpcTag = | typeof WS_METHODS.subscribeTerminalMetadata | typeof WS_METHODS.subscribePreviewEvents | typeof WS_METHODS.subscribeDiscoveredLocalServers + | typeof WS_METHODS.subscribeDeviceState | typeof WS_METHODS.subscribeResourceTelemetry | typeof WS_METHODS.pullRequestsSubscribeRefreshes | typeof WS_METHODS.previewAutomationConnect diff --git a/packages/client-runtime/src/state/device.ts b/packages/client-runtime/src/state/device.ts new file mode 100644 index 000000000000..1b983ab671fe --- /dev/null +++ b/packages/client-runtime/src/state/device.ts @@ -0,0 +1,50 @@ +import { WS_METHODS } from "@t3tools/contracts"; +import { Atom } from "effect/unstable/reactivity"; + +import type { EnvironmentRegistry } from "../connection/registry.ts"; +import { + createAtomCommandScheduler, + createEnvironmentRpcCommand, + createEnvironmentRpcSubscriptionAtomFamily, +} from "./runtime.ts"; + +export function createDeviceEnvironmentAtoms( + runtime: Atom.AtomRuntime, +) { + const scheduler = createAtomCommandScheduler(); + const concurrency = { + mode: "serial" as const, + key: ({ environmentId }: { environmentId: string }) => environmentId, + }; + return { + /** Server-pushed device hosts, devices, and open sessions for one environment. */ + state: createEnvironmentRpcSubscriptionAtomFamily(runtime, { + label: "environment-data:device:state", + tag: WS_METHODS.subscribeDeviceState, + }), + list: createEnvironmentRpcCommand(runtime, { + label: "environment-data:device:list", + tag: WS_METHODS.deviceList, + scheduler, + concurrency, + }), + open: createEnvironmentRpcCommand(runtime, { + label: "environment-data:device:open", + tag: WS_METHODS.deviceOpen, + scheduler, + concurrency, + }), + close: createEnvironmentRpcCommand(runtime, { + label: "environment-data:device:close", + tag: WS_METHODS.deviceClose, + scheduler, + concurrency, + }), + shutdown: createEnvironmentRpcCommand(runtime, { + label: "environment-data:device:shutdown", + tag: WS_METHODS.deviceShutdown, + scheduler, + concurrency, + }), + }; +} diff --git a/packages/client-runtime/src/state/deviceHubAccess.ts b/packages/client-runtime/src/state/deviceHubAccess.ts new file mode 100644 index 000000000000..561c56423761 --- /dev/null +++ b/packages/client-runtime/src/state/deviceHubAccess.ts @@ -0,0 +1,72 @@ +/** + * Credentials for the Device panel's media requests. + * + * The panel reaches simulator streams through `/api/device-hub/*` on the + * environment origin, using ``, `fetch`, and `WebSocket`. None of those + * can carry a bearer or DPoP header, so bearer and DPoP connections mint a + * short-lived WebSocket ticket and pass it as `wsTicket`, the same way the + * app's own `/ws` upgrade authenticates. Cookie sessions send the cookie. + * + * A ticket lives five minutes server-side and is bound to the session, not + * to one request, so one ticket covers everything a panel opens at once. + * Callers fetch a fresh one each time they (re)connect a stream. + */ +import * as Effect from "effect/Effect"; +import type { HttpClient } from "effect/unstable/http"; + +import { RemoteEnvironmentAuthorization } from "../authorization/service.ts"; +import type { PreparedConnection } from "../connection/model.ts"; +import { environmentEndpointUrl } from "../environment/endpoint.ts"; +import { ManagedRelayDpopSigner } from "../relay/managedRelay.ts"; +import type { RemoteEnvironmentRequestError } from "../rpc/http.ts"; +import { executeAuthenticatedEnvironmentHttpRequest } from "./environmentHttpAuth.ts"; + +const TICKET_TIMEOUT_MS = 8_000; + +export interface DeviceHubAccess { + /** Absolute origin-relative base, e.g. `https://env.example/api/device-hub`. */ + readonly httpBase: string; + /** Same base with the `ws(s)` scheme. */ + readonly wsBase: string; + /** Query parameters to append to every hub request; empty for cookie sessions. */ + readonly query: Readonly>; + /** Whether requests must include cookies (same-origin session). */ + readonly credentials: boolean; +} + +export const resolveDeviceHubAccess = Effect.fn("clientRuntime.state.resolveDeviceHubAccess")( + function* (input: { + readonly prepared: PreparedConnection; + readonly hubBasePath: string; + }): Effect.fn.Return { + const httpBase = environmentEndpointUrl(input.prepared.httpBaseUrl, input.hubBasePath); + const wsBase = httpBase.replace(/^http/, "ws"); + if (input.prepared.httpAuthorization === null) { + return { httpBase, wsBase, query: {}, credentials: true }; + } + const signer = yield* Effect.serviceOption(ManagedRelayDpopSigner); + const remoteAuthorization = yield* Effect.serviceOption(RemoteEnvironmentAuthorization); + const ticket = yield* executeAuthenticatedEnvironmentHttpRequest({ + prepared: input.prepared, + signer, + remoteAuthorization, + method: "POST", + url: (httpBaseUrl) => environmentEndpointUrl(httpBaseUrl, "/api/auth/websocket-ticket"), + timeoutMs: TICKET_TIMEOUT_MS, + request: ({ client, headers }) => client.auth.webSocketTicket({ headers }), + }); + return { + httpBase, + wsBase, + query: { wsTicket: ticket.ticket }, + credentials: false, + }; + }, +); + +export const withDeviceHubQuery = (url: string, access: DeviceHubAccess): string => { + const entries = Object.entries(access.query); + if (entries.length === 0) return url; + const separator = url.includes("?") ? "&" : "?"; + return `${url}${separator}${new URLSearchParams(entries).toString()}`; +}; diff --git a/packages/client-runtime/src/work-log/presentation.test.ts b/packages/client-runtime/src/work-log/presentation.test.ts index e056e92afe5c..2a2334ace676 100644 --- a/packages/client-runtime/src/work-log/presentation.test.ts +++ b/packages/client-runtime/src/work-log/presentation.test.ts @@ -221,6 +221,19 @@ describe("resolveWorkEntryToolPresentation", () => { }); }); + it("labels device tools with the device icon", () => { + expect( + resolveWorkEntryToolPresentation({ + label: "mcp__t3-code__device_open", + toolLifecycleStatus: "completed", + }), + ).toEqual({ displayName: "Opened a device in the Device panel", icon: "device" }); + expect(resolveWorkEntryToolPresentation({ label: "t3-code · device_screenshot" })).toEqual({ + displayName: "Taking a screenshot of the device", + icon: "device", + }); + }); + it("uses structured MCP identity when the provider supplies a custom title", () => { expect( resolveWorkEntryToolPresentation({ diff --git a/packages/client-runtime/src/work-log/presentation.ts b/packages/client-runtime/src/work-log/presentation.ts index 2e1ef3bbf003..9b2e3988752c 100644 --- a/packages/client-runtime/src/work-log/presentation.ts +++ b/packages/client-runtime/src/work-log/presentation.ts @@ -96,6 +96,15 @@ const T3_MCP_TOOL_LABELS: Record< preview_set_appearance: ["Set", "Setting", "Set", "preview browser appearance"], preview_recording_start: ["Start", "Starting", "Started", "recording the preview browser"], preview_recording_stop: ["Stop", "Stopping", "Stopped", "recording the preview browser"], + device_list: ["List", "Listing", "Listed", "simulators and emulators"], + device_open: ["Open", "Opening", "Opened", "a device in the Device panel"], + device_screenshot: [ + "Take a screenshot of", + "Taking a screenshot of", + "Took a screenshot of", + "the device", + ], + device_close: ["Close", "Closing", "Closed", "a device"], }; function resolveT3McpToolPresentation(value: string | undefined, status: string | undefined) { @@ -122,7 +131,11 @@ function resolveT3McpToolPresentation(value: string | undefined, status: string return { displayName: `${verb} ${detail}`, - icon: name.startsWith("preview_") ? ("browser" as const) : ("t3-code" as const), + icon: name.startsWith("preview_") + ? ("browser" as const) + : name.startsWith("device_") + ? ("device" as const) + : ("t3-code" as const), }; } diff --git a/packages/contracts/src/device.ts b/packages/contracts/src/device.ts new file mode 100644 index 000000000000..39ce80df2968 --- /dev/null +++ b/packages/contracts/src/device.ts @@ -0,0 +1,300 @@ +/** + * Device - Schemas for first-class iOS Simulator and Android Emulator support. + * + * The server owns device discovery, the streaming helper (expo-device-hub), + * and the agent driver (agent-device). Clients render the live screen from the + * server-proxied stream, and agents reach devices through the `device_*` MCP + * tools plus the `agent-device` CLI the server preconfigures for them. + * + * Devices live on a *host*. Only the local host (the machine the server runs + * on) exists today; the host id is carried everywhere so SSH and cloud hosts + * can be added without changing the client contract. + * + * @module Device + */ +import { Schema } from "effect"; + +import { ThreadId, TrimmedNonEmptyString } from "./baseSchemas.ts"; + +export const DevicePlatform = Schema.Literals(["ios", "android"]); +export type DevicePlatform = typeof DevicePlatform.Type; + +export const DeviceHostId = TrimmedNonEmptyString.check(Schema.isMaxLength(128)); +export type DeviceHostId = typeof DeviceHostId.Type; + +/** The server machine. Always present; other host kinds are future work. */ +export const LOCAL_DEVICE_HOST_ID = "local" as DeviceHostId; + +/** Simulator udid or adb serial (an AVD name while it is not running). */ +export const DeviceId = TrimmedNonEmptyString.check(Schema.isMaxLength(256)); +export type DeviceId = typeof DeviceId.Type; + +export const DeviceSummary = Schema.Struct({ + hostId: DeviceHostId, + id: DeviceId, + platform: DevicePlatform, + name: TrimmedNonEmptyString, + /** OS label such as "iOS 18.0" or "Android 15.0". */ + version: Schema.String, + booted: Schema.Boolean, + physical: Schema.Boolean, +}); +export type DeviceSummary = typeof DeviceSummary.Type; + +/** + * What the host can do right now. Platforms missing their toolchain are + * reported rather than hidden so the picker and the agent can explain why a + * platform is absent instead of showing an empty list. + */ +export const DevicePlatformAvailability = Schema.Struct({ + platform: DevicePlatform, + available: Schema.Boolean, + reason: Schema.optional(Schema.String), +}); +export type DevicePlatformAvailability = typeof DevicePlatformAvailability.Type; + +export const DeviceHostSummary = Schema.Struct({ + id: DeviceHostId, + kind: Schema.Literals(["local"]), + label: TrimmedNonEmptyString, + platforms: Schema.Array(DevicePlatformAvailability), +}); +export type DeviceHostSummary = typeof DeviceHostSummary.Type; + +/** + * Lifecycle of the helper processes on a host. Tools are installed on first + * use, so a fresh install spends a while in `installing` before any device can + * stream; the UI shows that instead of an empty picker. + */ +export const DeviceHostStatus = Schema.Literals([ + "idle", + "installing", + "starting", + "ready", + "failed", +]); +export type DeviceHostStatus = typeof DeviceHostStatus.Type; + +/** + * A device a thread is looking at. One session per (thread, device); the same + * device may be open in several threads, since the stream is shared. + */ +export const DeviceSession = Schema.Struct({ + threadId: ThreadId, + hostId: DeviceHostId, + deviceId: DeviceId, + platform: DevicePlatform, + openedAt: Schema.String, +}); +export type DeviceSession = typeof DeviceSession.Type; + +export const DeviceServiceState = Schema.Struct({ + hosts: Schema.Array(DeviceHostSummary), + hostStatus: DeviceHostStatus, + hostStatusDetail: Schema.optional(Schema.String), + devices: Schema.Array(DeviceSummary), + sessions: Schema.Array(DeviceSession), + /** Origin-relative path the client prefixes to hub routes. */ + hubBasePath: Schema.String, + revision: Schema.Int, +}); +export type DeviceServiceState = typeof DeviceServiceState.Type; + +export const DeviceListInput = Schema.Struct({}); +export type DeviceListInput = typeof DeviceListInput.Type; + +export const DeviceOpenInput = Schema.Struct({ + threadId: ThreadId, + hostId: Schema.optional(DeviceHostId), + deviceId: DeviceId, + platform: DevicePlatform, + /** Boot the simulator or emulator when it is not running. Defaults to true. */ + boot: Schema.optional(Schema.Boolean), +}); +export type DeviceOpenInput = typeof DeviceOpenInput.Type; + +export const DeviceCloseInput = Schema.Struct({ + threadId: ThreadId, + /** Omit to close every device session for the thread. */ + deviceId: Schema.optional(DeviceId), + /** Also shut the simulator or emulator down. Defaults to false. */ + shutdown: Schema.optional(Schema.Boolean), +}); +export type DeviceCloseInput = typeof DeviceCloseInput.Type; + +export const DeviceShutdownInput = Schema.Struct({ + hostId: Schema.optional(DeviceHostId), + deviceId: DeviceId, + platform: DevicePlatform, +}); +export type DeviceShutdownInput = typeof DeviceShutdownInput.Type; + +export class DeviceHostUnavailableError extends Schema.TaggedError()( + "DeviceHostUnavailableError", + { + hostId: DeviceHostId, + reason: Schema.String, + }, +) { + override get message(): string { + return `Device host ${this.hostId} is unavailable: ${this.reason}`; + } +} + +export class DevicePlatformUnavailableError extends Schema.TaggedError()( + "DevicePlatformUnavailableError", + { + hostId: DeviceHostId, + platform: DevicePlatform, + reason: Schema.String, + }, +) { + override get message(): string { + return `${this.platform} devices are unavailable on host ${this.hostId}: ${this.reason}`; + } +} + +export class DeviceNotFoundError extends Schema.TaggedError()( + "DeviceNotFoundError", + { + hostId: DeviceHostId, + deviceId: DeviceId, + }, +) { + override get message(): string { + return `Device ${this.deviceId} was not found on host ${this.hostId}.`; + } +} + +export class DeviceBootError extends Schema.TaggedError()("DeviceBootError", { + hostId: DeviceHostId, + deviceId: DeviceId, + detail: Schema.String, +}) { + override get message(): string { + return `Device ${this.deviceId} failed to boot: ${this.detail}`; + } +} + +export class DeviceOperationError extends Schema.TaggedError()( + "DeviceOperationError", + { + operation: Schema.String, + detail: Schema.String, + }, +) { + override get message(): string { + return `Device ${this.operation} failed: ${this.detail}`; + } +} + +export const DeviceError = Schema.Union([ + DeviceHostUnavailableError, + DevicePlatformUnavailableError, + DeviceNotFoundError, + DeviceBootError, + DeviceOperationError, +]); +export type DeviceError = typeof DeviceError.Type; + +// MCP tool shapes. Kept next to the RPC shapes so the tool surface and the +// panel describe devices the same way. + +export const DeviceToolListResult = Schema.Struct({ + hosts: Schema.Array(DeviceHostSummary), + devices: Schema.Array(DeviceSummary), + /** Devices already open in this thread's Device panel. */ + open: Schema.Array(Schema.Struct({ hostId: DeviceHostId, deviceId: DeviceId })), +}); +export type DeviceToolListResult = typeof DeviceToolListResult.Type; + +export const DeviceToolOpenInput = Schema.Struct({ + deviceId: Schema.optional( + DeviceId.annotate({ + description: + "Simulator udid or emulator serial from device_list. Omit to use the booted device for the platform, or the most recently used one.", + }), + ), + platform: Schema.optional( + DevicePlatform.annotate({ + description: "Required when deviceId is omitted and both platforms are available.", + }), + ), + hostId: Schema.optional( + DeviceHostId.annotate({ description: "Device host from device_list. Defaults to local." }), + ), +}).annotate({ + description: + "Boots the device if needed, starts its live stream, and opens the Device panel so the user can watch. Returns how to drive it with the agent-device CLI.", +}); +export type DeviceToolOpenInput = typeof DeviceToolOpenInput.Type; + +export const DeviceToolOpenResult = Schema.Struct({ + device: DeviceSummary, + /** Ready-to-run agent-device invocation pinned to this device. */ + agentDevice: Schema.Struct({ + command: Schema.String, + /** Flags that pin every command to this device, e.g. `--udid `. */ + targetArgs: Schema.Array(Schema.String), + }), + quickStart: Schema.String, +}); +export type DeviceToolOpenResult = typeof DeviceToolOpenResult.Type; + +export const DeviceToolTargetInput = Schema.Struct({ + deviceId: Schema.optional( + DeviceId.annotate({ + description: + "Device from device_list. Omit to use the device most recently opened in this thread.", + }), + ), + hostId: Schema.optional(DeviceHostId), +}); +export type DeviceToolTargetInput = typeof DeviceToolTargetInput.Type; + +export const DeviceToolScreenshotResult = Schema.Struct({ + device: DeviceSummary, + screenshot: Schema.Struct({ + mimeType: Schema.Literal("image/png"), + data: Schema.String, + width: Schema.Int, + height: Schema.Int, + }), +}); +export type DeviceToolScreenshotResult = typeof DeviceToolScreenshotResult.Type; + +export const DeviceToolCloseInput = Schema.Struct({ + deviceId: Schema.optional( + DeviceId.annotate({ + description: "Device to close. Omit to close every device in this thread.", + }), + ), + hostId: Schema.optional(DeviceHostId), + shutdown: Schema.optional( + Schema.Boolean.annotate({ + description: "Also power the simulator or emulator off. Defaults to false.", + }), + ), +}); +export type DeviceToolCloseInput = typeof DeviceToolCloseInput.Type; + +export class DeviceToolUnavailableError extends Schema.TaggedError()( + "DeviceToolUnavailableError", + { + reason: Schema.String, + }, +) { + override get message(): string { + return this.reason; + } +} + +export const DeviceToolError = Schema.Union([ + DeviceToolUnavailableError, + DeviceHostUnavailableError, + DevicePlatformUnavailableError, + DeviceNotFoundError, + DeviceBootError, + DeviceOperationError, +]); +export type DeviceToolError = typeof DeviceToolError.Type; diff --git a/packages/contracts/src/index.ts b/packages/contracts/src/index.ts index 74a1b4939f1a..32ac53dae9ba 100644 --- a/packages/contracts/src/index.ts +++ b/packages/contracts/src/index.ts @@ -34,6 +34,7 @@ export * from "./assets.ts"; export * from "./review.ts"; export * from "./browserImport.ts"; export * from "./browserProfile.ts"; +export * from "./device.ts"; export * from "./preview.ts"; export * from "./previewAutomation.ts"; export * from "./resourceTelemetry.ts"; diff --git a/packages/contracts/src/rpc.ts b/packages/contracts/src/rpc.ts index 9dbcaa9f4164..7d65551a8454 100644 --- a/packages/contracts/src/rpc.ts +++ b/packages/contracts/src/rpc.ts @@ -179,6 +179,15 @@ import { PreviewResizeInput, PreviewSessionSnapshot, } from "./preview.ts"; +import { + DeviceCloseInput, + DeviceError, + DeviceListInput, + DeviceOpenInput, + DeviceServiceState, + DeviceSession, + DeviceShutdownInput, +} from "./device.ts"; import { PreviewAutomationError, PreviewAutomationHost, @@ -310,6 +319,12 @@ export const WS_METHODS = { previewAutomationRespond: "previewAutomation.respond", previewAutomationFocusHost: "previewAutomation.focusHost", + // Device methods + deviceList: "device.list", + deviceOpen: "device.open", + deviceClose: "device.close", + deviceShutdown: "device.shutdown", + // Server meta serverProbe: "server.probe", serverGetConfig: "server.getConfig", @@ -374,6 +389,7 @@ export const WS_METHODS = { subscribeTerminalMetadata: "subscribeTerminalMetadata", subscribePreviewEvents: "subscribePreviewEvents", subscribeDiscoveredLocalServers: "subscribeDiscoveredLocalServers", + subscribeDeviceState: "subscribeDeviceState", subscribeServerConfig: "subscribeServerConfig", subscribeServerLifecycle: "subscribeServerLifecycle", subscribeAuthAccess: "subscribeAuthAccess", @@ -1062,6 +1078,35 @@ const WsSubscribeDiscoveredLocalServersRpc = Rpc.make(WS_METHODS.subscribeDiscov stream: true, }); +const WsDeviceListRpc = Rpc.make(WS_METHODS.deviceList, { + payload: DeviceListInput, + success: DeviceServiceState, + error: Schema.Union([DeviceError, EnvironmentAuthorizationError]), +}); + +const WsDeviceOpenRpc = Rpc.make(WS_METHODS.deviceOpen, { + payload: DeviceOpenInput, + success: DeviceSession, + error: Schema.Union([DeviceError, EnvironmentAuthorizationError]), +}); + +const WsDeviceCloseRpc = Rpc.make(WS_METHODS.deviceClose, { + payload: DeviceCloseInput, + error: Schema.Union([DeviceError, EnvironmentAuthorizationError]), +}); + +const WsDeviceShutdownRpc = Rpc.make(WS_METHODS.deviceShutdown, { + payload: DeviceShutdownInput, + error: Schema.Union([DeviceError, EnvironmentAuthorizationError]), +}); + +const WsSubscribeDeviceStateRpc = Rpc.make(WS_METHODS.subscribeDeviceState, { + payload: Schema.Struct({}), + success: DeviceServiceState, + error: EnvironmentAuthorizationError, + stream: true, +}); + const WsOrchestrationDispatchCommandRpc = Rpc.make(ORCHESTRATION_WS_METHODS.dispatchCommand, { payload: ClientOrchestrationCommand, success: OrchestrationRpcSchemas.dispatchCommand.output, @@ -1290,6 +1335,11 @@ export const WsRpcGroup = RpcGroup.make( WsPreviewAutomationFocusHostRpc, WsSubscribePreviewEventsRpc, WsSubscribeDiscoveredLocalServersRpc, + WsDeviceListRpc, + WsDeviceOpenRpc, + WsDeviceCloseRpc, + WsDeviceShutdownRpc, + WsSubscribeDeviceStateRpc, WsSubscribeServerConfigRpc, WsSubscribeServerLifecycleRpc, WsSubscribeAuthAccessRpc, diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index 3491103da94f..680d7fe19683 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -962,6 +962,14 @@ export const ServerSettings = Schema.Struct({ defaultModelSelection: Schema.NullOr(ModelSelection).pipe( Schema.withDecodingDefault(Effect.succeed(null)), ), + /** + * Whether agents may drive simulators and emulators. Gates the `device_*` + * MCP tools and the preconfigured `agent-device` CLI the same way + * `enableAgentBrowserAccess` gates the browser: server-authoritative, applied + * when the provider session is prepared. The user's own Device panel is + * unaffected. + */ + enableAgentDeviceAccess: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(true))), sidebarAutoSettleAfterDays: Schema.NullOr(SidebarAutoSettleAfterDays).pipe( Schema.withDecodingDefault(Effect.succeed(DEFAULT_SIDEBAR_AUTO_SETTLE_AFTER_DAYS)), ), @@ -1234,6 +1242,7 @@ export const ServerSettingsPatch = Schema.Struct({ Schema.Record(ProjectId, Schema.NullOr(Schema.Boolean)), ), defaultModelSelection: Schema.optionalKey(Schema.NullOr(ModelSelection)), + enableAgentDeviceAccess: Schema.optionalKey(Schema.Boolean), sidebarAutoSettleAfterDays: Schema.optionalKey(Schema.NullOr(SidebarAutoSettleAfterDays)), sidebarAutoSettleOnMerge: Schema.optionalKey(Schema.Boolean), backgroundActivity: Schema.optionalKey( From 173b8c856442a777711c6471008a13e1a4d4fd4f Mon Sep 17 00:00:00 2001 From: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com> Date: Mon, 7 Sep 2026 20:10:22 -0700 Subject: [PATCH 02/28] fix(devices): proxy hub WebSocket upgrades in dev and scope the state stream Co-Authored-By: Claude Fable 5 --- apps/server/src/device/DeviceService.test.ts | 45 ++++++++++++++++++++ apps/server/src/device/DeviceService.ts | 6 ++- apps/web/vite.config.ts | 9 ++-- 3 files changed, 54 insertions(+), 6 deletions(-) create mode 100644 apps/server/src/device/DeviceService.test.ts diff --git a/apps/server/src/device/DeviceService.test.ts b/apps/server/src/device/DeviceService.test.ts new file mode 100644 index 000000000000..a41f1242c337 --- /dev/null +++ b/apps/server/src/device/DeviceService.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, it } from "@effect/vitest"; +import type { DeviceServiceState } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import * as PubSub from "effect/PubSub"; +import * as Ref from "effect/Ref"; +import * as Stream from "effect/Stream"; + +import { type DeviceService, stateStream } from "./DeviceService.ts"; + +const baseState: DeviceServiceState = { + hosts: [], + hostStatus: "idle", + devices: [], + sessions: [], + hubBasePath: "/api/device-hub", + revision: 0, +}; + +describe("DeviceService.stateStream", () => { + it.effect("emits the current snapshot and then every published change", () => + Effect.gen(function* () { + const pubsub = yield* PubSub.unbounded(); + const current = yield* Ref.make(baseState); + const service: Pick = { + state: Ref.get(current), + subscribe: PubSub.subscribe(pubsub), + }; + + const collected = yield* stateStream(service as DeviceService["Service"]).pipe( + Stream.take(3), + Stream.runCollect, + Effect.forkChild, + ); + yield* Effect.yieldNow; + for (const revision of [1, 2]) { + const next = { ...baseState, revision, hostStatus: "ready" as const }; + yield* Ref.set(current, next); + yield* PubSub.publish(pubsub, next); + } + const seen = yield* Fiber.join(collected); + expect(seen.map((state) => state.revision)).toEqual([0, 1, 2]); + }), + ); +}); diff --git a/apps/server/src/device/DeviceService.ts b/apps/server/src/device/DeviceService.ts index a0d254cd9994..a4d6f4462b90 100644 --- a/apps/server/src/device/DeviceService.ts +++ b/apps/server/src/device/DeviceService.ts @@ -468,8 +468,10 @@ export const layer = Layer.effect(DeviceService, make); export const stateStream = (service: DeviceService["Service"]): Stream.Stream => Stream.unwrap( Effect.gen(function* () { - const initial = yield* service.state; + // Subscribe before reading the snapshot so no change between the two + // is lost; the scope lives as long as the stream does. const subscription = yield* service.subscribe; + const initial = yield* service.state; return Stream.concat(Stream.make(initial), Stream.fromSubscription(subscription)); }), - ); + ).pipe(Stream.scoped); diff --git a/apps/web/vite.config.ts b/apps/web/vite.config.ts index 702c18840983..ec4f15fc7eeb 100644 --- a/apps/web/vite.config.ts +++ b/apps/web/vite.config.ts @@ -230,16 +230,17 @@ export default defineConfig(() => { ? { // One entry per shared prefix; the server's dev catch-all 404s the // same list, so the two sides cannot drift. `/ws` is the app's own - // socket — Vite's HMR socket is matched separately and exactly - // (path "/" plus a vite-hmr subprotocol), so the two upgrade - // handlers don't collide. + // socket and `/api` carries the device hub's stream sockets — + // Vite's HMR socket is matched separately and exactly (path "/" + // plus a vite-hmr subprotocol), so the upgrade handlers don't + // collide. proxy: Object.fromEntries( DEV_PROXIED_PATH_PREFIXES.map((prefix) => [ prefix, { target: devProxyTarget, changeOrigin: true, - ...(prefix === "/ws" ? { ws: true } : {}), + ...(prefix === "/ws" || prefix === "/api" ? { ws: true } : {}), }, ]), ), From 9b1e2e91382a9286335d96b8e61968b396003e03 Mon Sep 17 00:00:00 2001 From: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com> Date: Mon, 7 Sep 2026 20:16:56 -0700 Subject: [PATCH 03/28] fix(devices): probe decoder support and fall back to MJPEG on iOS Co-Authored-By: Claude Fable 5 --- .../components/device/DeviceStreamView.tsx | 7 +- .../web/src/components/device/deviceStream.ts | 65 ++++++++++++++----- 2 files changed, 55 insertions(+), 17 deletions(-) diff --git a/apps/web/src/components/device/DeviceStreamView.tsx b/apps/web/src/components/device/DeviceStreamView.tsx index 6e9e5849a5a8..0f14c191b887 100644 --- a/apps/web/src/components/device/DeviceStreamView.tsx +++ b/apps/web/src/components/device/DeviceStreamView.tsx @@ -64,11 +64,14 @@ export function DeviceStreamView(props: { // A fresh ticket re-runs this effect through the access dependency. refreshDeviceHubAccess(props.environmentId); }, + onMjpegFallback: (url) => { + setMjpegUrl(url); + setMjpegGeneration((generation) => generation + 1); + }, }, ); clientRef.current = client; - setMjpegUrl(client.mjpegUrl); - setMjpegGeneration((generation) => generation + 1); + setMjpegUrl(null); client.start(); onHandle?.({ pressButton: client.pressButton, rotate: client.rotate }); return () => { diff --git a/apps/web/src/components/device/deviceStream.ts b/apps/web/src/components/device/deviceStream.ts index a7f5573d297d..94c2074982bd 100644 --- a/apps/web/src/components/device/deviceStream.ts +++ b/apps/web/src/components/device/deviceStream.ts @@ -33,6 +33,12 @@ export interface DeviceStreamEvents { readonly onScreen: (screen: DeviceScreenSize) => void; /** The proxy rejected the credential; the owner should refresh access and reconnect. */ readonly onUnauthorized: () => void; + /** + * H.264 cannot be decoded here (no WebCodecs, or the simulator's profile is + * unsupported); the owner should show this MJPEG URL in an `` instead of + * the canvas. + */ + readonly onMjpegFallback: (url: string) => void; } export interface DeviceStreamTarget { @@ -189,8 +195,6 @@ export interface DeviceStreamClient { readonly sendKey: (event: KeyboardEvent, phase: "down" | "up") => void; readonly pressButton: (button: DeviceHardwareButton) => void; readonly rotate: () => void; - /** MJPEG fallback source when WebCodecs is unavailable on iOS, else null. */ - readonly mjpegUrl: string | null; } const HID_USAGE_BY_CODE: Readonly> = { @@ -276,6 +280,18 @@ export function createDeviceStreamClient( let awaitingKeyframe = true; let screen: DeviceScreenSize | null = null; let firstFrame = false; + let configuring = false; + let mjpeg = false; + + const mjpegUrl = () => httpUrl(`/helper/${device}/stream.mjpeg`); + + const fallBackToMjpeg = () => { + if (stopped || mjpeg) return; + mjpeg = true; + closeDecoder(); + events.onMjpegFallback(mjpegUrl()); + setStatus("streaming"); + }; const setStatus = (status: DeviceStreamStatus, detail?: string) => { if (!stopped) events.onStatus(status, detail); @@ -323,14 +339,22 @@ export function createDeviceStreamClient( }, }); - const configureDecoder = (config: VideoDecoderConfig) => { + /** + * Resolves false when this browser cannot decode the stream's profile + * (simulators encode High 5.1, which headless and some hardware decoders + * reject). iOS then falls back to MJPEG; Android has no MJPEG. + */ + const configureDecoder = async (config: VideoDecoderConfig): Promise => { + const full: VideoDecoderConfig = { ...config, optimizeForLatency: true }; + const support = await VideoDecoder.isConfigSupported(full).catch(() => ({ supported: false })); + if (stopped) return false; + if (!support.supported) { + setStatus("error", `This browser cannot decode ${config.codec}.`); + return false; + } if (!videoDecoder || videoDecoder.state === "closed") videoDecoder = makeDecoder(); try { - videoDecoder.configure({ - ...config, - optimizeForLatency: true, - hardwareAcceleration: "prefer-hardware", - }); + videoDecoder.configure(full); return true; } catch (cause) { setStatus("error", `Video decoder: ${(cause as Error).message}`); @@ -408,13 +432,19 @@ export function createDeviceStreamClient( }) .catch(() => {}); break; - case "description": + case "description": { awaitingKeyframe = true; - configureDecoder({ + const configured = await configureDecoder({ codec: avcCodecString(chunk.payload), description: chunk.payload, }); + if (!configured) { + await reader.cancel().catch(() => {}); + fallBackToMjpeg(); + return; + } break; + } case "keyframe": case "delta": decode(chunk.type === "keyframe", chunk.payload); @@ -474,8 +504,14 @@ export function createDeviceStreamClient( const scanned = needsScan ? scanAccessUnit(packet.data) : null; const isKey = packet.isKey ?? scanned?.isKey ?? false; if (scanned?.sps && (!videoDecoder || videoDecoder.state !== "configured")) { - if (!configureDecoder({ codec: avcCodecString(scanned.sps) })) return; - awaitingKeyframe = true; + if (configuring) return; + configuring = true; + void configureDecoder({ codec: avcCodecString(scanned.sps) }).then((configured) => { + configuring = false; + awaitingKeyframe = true; + if (configured) requestKeyframe(); + }); + return; } if (!videoDecoder || videoDecoder.state !== "configured") { if (!isKey) requestKeyframe(); @@ -503,7 +539,7 @@ export function createDeviceStreamClient( if (platform === "ios") { connectIosInput(); if (useWebCodecs) void readIosVideo(); - else setStatus("streaming"); + else fallBackToMjpeg(); } else if (useWebCodecs) { connectAndroid(); } else { @@ -514,6 +550,7 @@ export function createDeviceStreamClient( const stop = () => { if (stopped) return; stopped = true; + mjpeg = false; if (retryTimer) clearTimeout(retryTimer); retryTimer = null; controller?.abort(); @@ -546,8 +583,6 @@ export function createDeviceStreamClient( return { start, stop, - mjpegUrl: - platform === "ios" && !useWebCodecs ? httpUrl(`/helper/${device}/stream.mjpeg`) : null, sendTouch: (phase, x, y) => { if (platform === "ios") { send(taggedJson(IOS_MSG_TOUCH, { type: phase, ...rawPoint(x, y) })); From a6a204c563e7531d2fe44ac837b28f214487d231 Mon Sep 17 00:00:00 2001 From: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com> Date: Mon, 7 Sep 2026 20:27:16 -0700 Subject: [PATCH 04/28] fix(devices): give device_list an object schema so providers keep the toolkit Co-Authored-By: Claude Fable 5 --- apps/server/src/mcp/McpDeviceToolkit.test.ts | 123 ++++++++++++++++++ .../src/mcp/toolkits/device/handlers.ts | 11 +- apps/server/src/mcp/toolkits/device/tools.ts | 8 +- 3 files changed, 139 insertions(+), 3 deletions(-) create mode 100644 apps/server/src/mcp/McpDeviceToolkit.test.ts diff --git a/apps/server/src/mcp/McpDeviceToolkit.test.ts b/apps/server/src/mcp/McpDeviceToolkit.test.ts new file mode 100644 index 000000000000..a4e99b639a63 --- /dev/null +++ b/apps/server/src/mcp/McpDeviceToolkit.test.ts @@ -0,0 +1,123 @@ +import { expect, it } from "@effect/vitest"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { EnvironmentId, ProviderInstanceId, ThreadId } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import { McpSchema, McpServer } from "effect/unstable/ai"; + +import * as DeviceService from "../device/DeviceService.ts"; +import * as McpHttpServer from "./McpHttpServer.ts"; +import * as McpInvocationContext from "./McpInvocationContext.ts"; + +const environmentId = EnvironmentId.make("environment-device-test"); +const threadId = ThreadId.make("thread-device-test"); +const invocation = (capabilities: ReadonlyArray) => ({ + environmentId, + threadId, + providerSessionId: "provider-session-device-test", + providerInstanceId: ProviderInstanceId.make("codex"), + capabilities: new Set(capabilities), + issuedAt: 1, +}); +const client = McpSchema.McpServerClient.of({ + clientId: 1, + clientCapabilities: {}, + clientInfo: { name: "mcp-test", version: "1.0.0" }, + protocolVersion: "2025-06-18", + initializePayload: { + protocolVersion: "2025-06-18", + capabilities: {}, + clientInfo: { name: "mcp-test", version: "1.0.0" }, + }, + getClient: Effect.die("unused"), +}); + +const device = { + hostId: "local", + id: "UDID-1", + platform: "ios" as const, + name: "iPhone 17 Pro", + version: "iOS 27.0", + booted: true, + physical: false, +}; +const state = { + hosts: [ + { + id: "local", + kind: "local" as const, + label: "This machine", + platforms: [ + { platform: "ios" as const, available: true }, + { platform: "android" as const, available: false, reason: "No SDK" }, + ], + }, + ], + hostStatus: "ready" as const, + devices: [device], + sessions: [], + hubBasePath: "/api/device-hub", + revision: 1, +}; +const png = new Uint8Array(24); +new DataView(png.buffer).setUint32(0, 0x89504e47); +new DataView(png.buffer).setUint32(4, 0x0d0a1a0a); +new DataView(png.buffer).setUint32(12, 0x49484452); +new DataView(png.buffer).setUint32(16, 1206); +new DataView(png.buffer).setUint32(20, 2622); + +const DeviceServiceMock = Layer.mock(DeviceService.DeviceService)({ + state: Effect.succeed(state), + list: Effect.succeed(state), + open: (input) => + Effect.succeed({ + threadId: input.threadId, + hostId: "local", + deviceId: input.deviceId, + platform: input.platform, + openedAt: "2026-09-08T00:00:00.000Z", + }), + sessionsForThread: () => Effect.succeed([]), + screenshot: () => Effect.succeed({ device, png }), + close: () => Effect.void, +}); + +const TestLayer = McpHttpServer.DeviceToolkitRegistrationLive.pipe( + Layer.provideMerge(McpServer.McpServer.layer), + Layer.provideMerge(DeviceServiceMock), + Layer.provide(NodeServices.layer), +); + +it.effect("registers the device tools and returns the screenshot as image content", () => + Effect.scoped( + Effect.gen(function* () { + const server = yield* McpServer.McpServer; + const names = server.tools.map(({ tool }) => tool.name).toSorted(); + expect(names).toEqual(["device_close", "device_list", "device_open", "device_screenshot"]); + + const callWith = (capabilities: ReadonlyArray) => + Effect.provideService(McpInvocationContext.McpInvocationContext, invocation(capabilities)); + + const opened = yield* server + .callTool({ name: "device_open", arguments: { platform: "ios" } }) + .pipe(callWith(["device"]), Effect.provideService(McpSchema.McpServerClient, client)); + expect(opened.isError).toBe(false); + const openedContent = opened.structuredContent as { quickStart: string }; + expect(openedContent.quickStart).toContain("--udid UDID-1"); + + const shot = yield* server + .callTool({ name: "device_screenshot", arguments: { deviceId: "UDID-1" } }) + .pipe(callWith(["device"]), Effect.provideService(McpSchema.McpServerClient, client)); + expect(shot.isError).toBe(false); + expect(shot.content.map((entry) => entry.type)).toEqual(["text", "image"]); + expect(shot.structuredContent).toMatchObject({ + screenshot: { mimeType: "image/png", width: 1206, height: 2622 }, + }); + + const denied = yield* server + .callTool({ name: "device_list", arguments: {} }) + .pipe(callWith(["preview"]), Effect.provideService(McpSchema.McpServerClient, client)); + expect(denied.isError).toBe(true); + }), + ).pipe(Effect.provide(TestLayer)), +); diff --git a/apps/server/src/mcp/toolkits/device/handlers.ts b/apps/server/src/mcp/toolkits/device/handlers.ts index baf55201152e..dad91114d5c9 100644 --- a/apps/server/src/mcp/toolkits/device/handlers.ts +++ b/apps/server/src/mcp/toolkits/device/handlers.ts @@ -101,15 +101,22 @@ const pickDevice = ( const toolError = (error: DeviceError | DeviceToolUnavailableError) => error; const handlers = { - device_list: () => + device_list: (input) => Effect.gen(function* () { const scope = yield* requireDeviceAccess; const devices = yield* DeviceService.DeviceService; const state = yield* devices.list; + const hostId = input?.hostId; const open = state.sessions .filter((session) => session.threadId === scope.threadId) .map((session) => ({ hostId: session.hostId, deviceId: session.deviceId })); - return { hosts: state.hosts, devices: state.devices, open }; + return { + hosts: hostId ? state.hosts.filter((host) => host.id === hostId) : state.hosts, + devices: hostId + ? state.devices.filter((device) => device.hostId === hostId) + : state.devices, + open, + }; }).pipe(Effect.mapError(toolError)), device_open: (input) => Effect.gen(function* () { diff --git a/apps/server/src/mcp/toolkits/device/tools.ts b/apps/server/src/mcp/toolkits/device/tools.ts index 5e157f1f5821..68b083174870 100644 --- a/apps/server/src/mcp/toolkits/device/tools.ts +++ b/apps/server/src/mcp/toolkits/device/tools.ts @@ -25,7 +25,13 @@ const dependencies = [McpInvocationContext.McpInvocationContext, DeviceService.D export const DeviceListTool = Tool.make("device_list", { description: "List iOS Simulators and Android Emulators on this environment's device hosts, which platforms each host can run, and which devices are already open in this thread's Device panel. Call this before device_open when you do not know a device id.", - parameters: Schema.Struct({}), + // An empty struct serializes as `anyOf [object, array]`, which some + // providers reject and then drop every tool on the server with it. + parameters: Schema.Struct({ + hostId: Schema.optional( + Schema.String.annotate({ description: "Limit to one device host. Defaults to all hosts." }), + ), + }), success: DeviceToolListResult, failure: DeviceToolError, dependencies, From 89f132a81035370571a25a88f45eb90a0df24019 Mon Sep 17 00:00:00 2001 From: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com> Date: Mon, 7 Sep 2026 20:33:39 -0700 Subject: [PATCH 05/28] fix(devices): start the device host when a device-capable session begins The provider environment is fixed at spawn, so agent-device has to be on PATH before the first turn rather than after device_open. Co-Authored-By: Claude Fable 5 --- apps/server/src/device/DeviceService.ts | 17 ++++++++++++++ .../src/provider/Layers/ProviderService.ts | 23 ++++++++++++------- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/apps/server/src/device/DeviceService.ts b/apps/server/src/device/DeviceService.ts index a4d6f4462b90..48c27ee7268f 100644 --- a/apps/server/src/device/DeviceService.ts +++ b/apps/server/src/device/DeviceService.ts @@ -95,6 +95,13 @@ export class DeviceService extends Context.Service< }) => Effect.Effect; /** Host endpoints for the proxy and the provider environment. */ readonly readiness: (hostId?: DeviceHostId) => Effect.Effect; + /** + * `readiness` only when the host can run at least one platform; a machine + * with no simulator toolchain never installs or starts anything. + */ + readonly readinessIfSupported: ( + hostId?: DeviceHostId, + ) => Effect.Effect; readonly currentReadiness: (hostId?: DeviceHostId) => Effect.Effect; readonly sessionsForThread: (threadId: ThreadId) => Effect.Effect>; } @@ -176,6 +183,15 @@ export const make = Effect.gen(function* () { }, ); + const readinessIfSupported: DeviceService["Service"]["readinessIfSupported"] = Effect.fn( + "DeviceService.readinessIfSupported", + )(function* (hostId) { + const host = yield* resolveHost(hostId); + const summary = yield* host.summary; + if (!summary.platforms.some((platform) => platform.available)) return null; + return yield* readiness(host.id); + }); + const currentReadiness: DeviceService["Service"]["currentReadiness"] = (hostId) => resolveHost(hostId).pipe( Effect.flatMap((host) => @@ -457,6 +473,7 @@ export const make = Effect.gen(function* () { shutdown, screenshot, readiness, + readinessIfSupported, currentReadiness, sessionsForThread, }); diff --git a/apps/server/src/provider/Layers/ProviderService.ts b/apps/server/src/provider/Layers/ProviderService.ts index 85a5b8cb9785..58fa02cfbbc4 100644 --- a/apps/server/src/provider/Layers/ProviderService.ts +++ b/apps/server/src/provider/Layers/ProviderService.ts @@ -257,7 +257,7 @@ export interface ProviderServiceLiveOptions { /** Same seam as `issueMcpCredential`, for observing the deny path's revoke. */ readonly revokeMcpCredential?: typeof McpSessionRegistry.revokeActiveMcpThread; /** Overrides the device host lookup used to build the agent-device environment. */ - readonly deviceReadiness?: () => Effect.Effect; + readonly deviceReadiness?: () => Effect.Effect; } interface TurnAnalyticsMetadata { @@ -493,7 +493,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( (() => Effect.serviceOption(DeviceService.DeviceService).pipe( Effect.flatMap((service) => - Option.isSome(service) ? service.value.currentReadiness() : Effect.succeed(null), + Option.isSome(service) ? service.value.readinessIfSupported() : Effect.succeed(null), ), )); const fileSystem = yield* FileSystem.FileSystem; @@ -925,15 +925,22 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( ); /** - * The device host only starts when a device is first opened, so a session - * prepared before that gets the tools without the CLI environment; the - * `device_open` result tells the agent the CLI is ready, and by then the - * next session restart picks the environment up. Sessions prepared after - * the host is running get it immediately. + * Starting a session with device access also brings the device host up, so + * the `agent-device` CLI is on the provider's PATH from its first turn. The + * environment is fixed at spawn time, so a host started later by + * `device_open` could not reach an already-running agent. Tools install once + * and the host is idempotent, so this is cheap after the first session; + * a host that fails to start withholds only the CLI, not the MCP tools. */ const hostPlatform = yield* HostProcessPlatform; const agentDeviceEnvironment = Effect.gen(function* () { - const readiness = yield* deviceReadiness(); + const readiness = yield* deviceReadiness().pipe( + Effect.catch((cause) => + Effect.logWarning("Device host unavailable; starting session without agent-device", { + cause, + }).pipe(Effect.as(null)), + ), + ); if (!readiness) return undefined; const shimDir = yield* ensureAgentDeviceShim({ entryPath: readiness.agentDevice.entryPath, From afaf7be6bb2da8276a55185de4eef35012e81057 Mon Sep 17 00:00:00 2001 From: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com> Date: Mon, 7 Sep 2026 20:49:01 -0700 Subject: [PATCH 06/28] fix(devices): reap a device hub left behind by a killed server Co-Authored-By: Claude Fable 5 docs(devices): describe the Device panel and its architecture --- apps/server/src/device/LocalDeviceHost.ts | 82 +++++++++++++++++++++++ docs/README.md | 2 + docs/internals/devices.md | 72 ++++++++++++++++++++ docs/user/devices.md | 42 ++++++++++++ 4 files changed, 198 insertions(+) create mode 100644 docs/internals/devices.md create mode 100644 docs/user/devices.md diff --git a/apps/server/src/device/LocalDeviceHost.ts b/apps/server/src/device/LocalDeviceHost.ts index 6a6a7c610473..24853ea29dd8 100644 --- a/apps/server/src/device/LocalDeviceHost.ts +++ b/apps/server/src/device/LocalDeviceHost.ts @@ -58,6 +58,20 @@ const DAEMON_POLL_MS = 100; const HUB_RESTART_STABLE_UPTIME_MS = 60_000; const HUB_RESTART_MAX_DELAY_MS = 30_000; +/** + * Written beside the agent-device state so a server that dies without running + * its finalizers (SIGKILL, dev-runner restarts) does not leave a hub bound to + * a loopback port forever. The next start reads it, kills only a process that + * is still that hub, and replaces the file. + */ +const HubStateFile = Schema.Struct({ + pid: Schema.Int, + port: Schema.Int, + entryPath: Schema.String, +}); +const decodeHubStateFile = Schema.decodeUnknownEffect(Schema.fromJsonString(HubStateFile)); +const encodeHubStateFile = Schema.encodeUnknownEffect(Schema.fromJsonString(HubStateFile)); + const AgentDeviceDaemonFile = Schema.Struct({ httpPort: Schema.Int, token: Schema.String, @@ -135,9 +149,75 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () { const stopHub = (hub: HubProcess | undefined) => hub ? Scope.close(hub.scope, Exit.void).pipe(Effect.ignore) : Effect.void; + const hubStatePath = () => path.join(agentDeviceStateDir(path, config.stateDir), "hub.json"); + + const isProcessAlive = (pid: number) => + Effect.sync(() => { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } + }); + + /** + * A hub left behind by a previous server is identified by pid plus the + * command line's entry path, so a recycled pid belonging to something else + * is never touched. + */ + const reapStaleHub = Effect.gen(function* () { + const previous = yield* fs + .readFileString(hubStatePath()) + .pipe(Effect.flatMap(decodeHubStateFile), Effect.option); + if (previous._tag === "None") return; + const alive = yield* isProcessAlive(previous.value.pid); + if (alive) { + const commandLine = yield* runner + .run({ + command: "ps", + args: ["-o", "command=", "-p", String(previous.value.pid)], + timeout: Duration.seconds(5), + timeoutBehavior: "timedOutResult", + }) + .pipe( + Effect.map((result) => result.stdout), + Effect.orElseSucceed(() => ""), + ); + if (commandLine.includes(previous.value.entryPath)) { + yield* Effect.logWarning("Stopping a device hub left behind by a previous server", { + pid: previous.value.pid, + port: previous.value.port, + }); + yield* Effect.sync(() => { + try { + process.kill(previous.value.pid, "SIGTERM"); + } catch { + // Already gone. + } + }); + } + } + yield* fs.remove(hubStatePath(), { force: true }).pipe(Effect.ignore); + }).pipe(Effect.catchCause(() => Effect.void)); + + const recordHub = (hub: HubProcess, tools: DeviceToolchainPaths) => + encodeHubStateFile({ + pid: Number(hub.child.pid), + port: Number(new URL(hub.origin).port), + entryPath: tools.hub.entryPath, + }).pipe( + Effect.flatMap((json) => fs.writeFileString(hubStatePath(), json)), + Effect.ignore, + ); + const spawnHub = Effect.fn("LocalDeviceHost.spawnHub")(function* ( tools: DeviceToolchainPaths, ): Effect.fn.Return { + yield* reapStaleHub; + yield* fs + .makeDirectory(agentDeviceStateDir(path, config.stateDir), { recursive: true }) + .pipe(Effect.ignore); const port = yield* net.reserveLoopbackPort("127.0.0.1").pipe( Effect.mapError( (cause) => @@ -203,6 +283,7 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () { Effect.provideService(HttpClient.HttpClient, httpClient), Effect.tapError(() => stopHub(hub)), ); + yield* recordHub(hub, tools); yield* Effect.logInfo("Device hub started", { pid: Number(child.pid), port }); return hub; }); @@ -399,6 +480,7 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () { Effect.gen(function* () { const running = yield* Ref.getAndSet(runningRef, null); yield* stopHub(running?.hub); + yield* fs.remove(hubStatePath(), { force: true }).pipe(Effect.ignore); yield* stopAgentDeviceDaemon(toolsRef); }), ); diff --git a/docs/README.md b/docs/README.md index 2d4809b7003a..4691e6f83c8e 100644 --- a/docs/README.md +++ b/docs/README.md @@ -13,6 +13,7 @@ - [Keyboard shortcuts](./user/keybindings.md) - [SnapShots](./user/snap-shot.md) - [Import browser sessions](./user/browser-import.md) +- [Devices](./user/devices.md) - [Usage and limits](./user/usage.md) - [Product usage data](./user/telemetry.md) - [Remote access](./user/remote-access.md) @@ -46,6 +47,7 @@ source alone does not explain. Most code changes do not need an internal documen - [Mobile navigation](./internals/mobile-navigation.md) - [Mobile development lifecycle](./internals/mobile-development.md) - [Terminal runtime](./internals/terminal-runtime.md) +- [Devices](./internals/devices.md) - [Voice input](./internals/voice-input.md) ### Runbooks diff --git a/docs/internals/devices.md b/docs/internals/devices.md new file mode 100644 index 000000000000..7eb01ac2e589 --- /dev/null +++ b/docs/internals/devices.md @@ -0,0 +1,72 @@ +# Devices + +The environment server owns simulators and emulators the way it owns +terminals: discovery, streaming, and agent access all run there, and every +client reaches them through the environment connection. This is what makes the +Device panel work over Tailscale and T3 Connect, and what will let a device +host on another machine slot in later. + +## Two external tools, one seam + +[expo-device-hub](../../apps/server/src/device/LocalDeviceHost.ts) streams and +[agent-device](../../apps/server/src/device/AgentDeviceShim.ts) drives. Both +are npm-installed at pinned versions into the T3 home on first use and run with +the server's Node; `npx` would make the first `device_open` after a reboot +depend on the registry. The hub is a supervised child rather than an imported +middleware because serve-sim loads private CoreSimulator frameworks through a +native addon, and a crash there must not take the server down. + +Everything platform-specific sits behind +[`DeviceHost`](../../apps/server/src/device/DeviceHost.ts). The service, the +proxy, and the MCP tools only see a hub origin and an agent-device endpoint. +An SSH or cloud host would forward those two things to the server and change +nothing above it. + +## The hub is never exposed + +serve-sim has a shell-exec route whose token is readable from its own +unauthenticated `/api`, and serve-emu's action routes have no auth at all. The +hub binds loopback and the only way in is the +[proxy](../../apps/server/src/device/DeviceHubProxy.ts), which allowlists the +stream, config, and screenshot routes and authenticates every request as an +environment session. `` and `WebSocket` cannot carry headers, so the proxy +authenticates like the `/ws` upgrade: cookie, or a short-lived `wsTicket` that +bearer and DPoP clients mint over authenticated HTTP. The ticket is stripped +before the request reaches the hub. + +Stream responses carry `Cache-Control: no-transform`; the compression +middleware would otherwise buffer an MJPEG body that never ends. In browser dev, +the Vite proxy must forward WebSocket upgrades for `/api`, not only `/ws`. + +## Agents drive through the CLI + +The `device_*` toolkit is deliberately four tools: list, open, screenshot, and +close. Driving happens through the `agent-device` CLI, which has the semantic +snapshot model agents need and stays current with its own releases. T3 prepends +a shim directory to the provider's PATH and sets +`AGENT_DEVICE_DAEMON_BASE_URL` and `AGENT_DEVICE_DAEMON_AUTH_TOKEN` so the +agent never handles the endpoint or token. + +That environment is fixed when the provider subprocess spawns, so +[`prepareMcpSession`](../../apps/server/src/provider/Layers/ProviderService.ts) +starts the device host whenever the session has the `device` capability and +the machine can run at least one platform. Starting it later from +`device_open` would leave the already-running agent without the CLI. + +How to drive a device is returned from `device_open`, not kept in an +always-loaded prompt or skill: it costs nothing in threads that never open a +device and cannot drift from the pinned CLI version. The always-on prompt block +is a few lines that point at the tools and forbid raw `simctl` and `adb`. + +## The viewer decodes both vendored protocols + +The hub vendors two streaming servers with different wire formats. iOS video is +an HTTP body of AVCC envelopes decoded with WebCodecs, with input on a separate +binary WebSocket; Android multiplexes SEMU-framed H.264 and JSON gestures over +one WebSocket. [`deviceStream.ts`](../../apps/web/src/components/device/deviceStream.ts) +speaks both so one panel covers both platforms. + +Simulators encode H.264 High 5.1. Hardware decoders on some machines and all +headless browsers reject that profile, and WebCodecs is secure-context only, so +the viewer probes `isConfigSupported` and falls back to the MJPEG endpoint on +iOS. Android has no MJPEG; there the panel reports that it cannot decode. diff --git a/docs/user/devices.md b/docs/user/devices.md new file mode 100644 index 000000000000..3ff87d992107 --- /dev/null +++ b/docs/user/devices.md @@ -0,0 +1,42 @@ +# Devices + +The Device panel shows a live iOS Simulator or Android Emulator next to a +thread, so you can watch an agent verify mobile work and tap the device +yourself. Agents get the same device through `device_*` tools and the +`agent-device` command line, which T3 Code sets up for them. + +## Open a device + +Open the right panel in a project thread and choose **Device**, then pick a +simulator or emulator. A device that is not running boots when you pick it. +The first time, T3 Code installs its device tools on the server; that takes a +minute and happens once. + +Simulators run on the machine that hosts the environment server. iOS needs +macOS with Xcode. Android needs the Android SDK with `ANDROID_HOME` set or +`adb` on the path. The panel says which platforms the server can run and why one +cannot. + +The screen is interactive: click and drag to touch, type while the screen is +focused, and use the toolbar for Home, Back, and Recents on Android, rotate on +iOS, and power off. Close the tab to stop watching; the device keeps running +unless you power it off. + +## Agents and devices + +When an agent opens a device, the panel opens in every client connected to the +thread. Agents drive the device through the `agent-device` command line, which +T3 Code preinstalls and connects for them. iOS taps through `agent-device` build +a small test runner on first use, which takes a couple of minutes once per +server. + +To keep agents away from simulators, turn off **Agent device access** in +Settings → Projects → Project defaults. This hides the device tools from agents +started from then on; your own Device panel is unaffected. + +## Remote connections + +The device stream goes through the environment server, so it works over the +local network, Tailscale, and T3 Connect. Live video needs a secure page +(HTTPS or localhost); on a plain-HTTP remote origin iOS falls back to a slower +still-image stream and Android cannot show video. From a87ece2610f213fb8f5c3d8b60e236345f0190d2 Mon Sep 17 00:00:00 2001 From: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com> Date: Mon, 7 Sep 2026 22:24:51 -0700 Subject: [PATCH 07/28] fix(devices): show input connectivity and disable controls while the socket is down Co-Authored-By: Claude Fable 5 --- .../web/src/components/device/DevicePanel.tsx | Bin 11038 -> 11165 bytes .../components/device/DeviceStreamView.tsx | 23 +++++++++++++++++- .../web/src/components/device/deviceStream.ts | 19 +++++++++++++-- 3 files changed, 39 insertions(+), 3 deletions(-) diff --git a/apps/web/src/components/device/DevicePanel.tsx b/apps/web/src/components/device/DevicePanel.tsx index 669636607a918c8789c668268eedaa46a70a0c5f..c3e0358b0e867a11c562e60a537a04aa453a5ea8 100644 GIT binary patch delta 140 zcmbOiHaC33Qz;>Pz0ACV(h}$Vyu8%plGK!1uE{*IGMkyDIT;1e+c|KwUJPL81blEji!C9cV9rPU|zmE~mQn*3T?5t~H?Dos8kEisu_ fj#F46C9^m&DJM0>wpuYGF)t-2wRUraoGUK?;2;@s diff --git a/apps/web/src/components/device/DeviceStreamView.tsx b/apps/web/src/components/device/DeviceStreamView.tsx index 0f14c191b887..83e97f083601 100644 --- a/apps/web/src/components/device/DeviceStreamView.tsx +++ b/apps/web/src/components/device/DeviceStreamView.tsx @@ -15,6 +15,8 @@ import { export interface DeviceStreamHandle { readonly pressButton: (button: DeviceHardwareButton) => void; readonly rotate: () => void; + /** False while the input socket is down; controls should disable. */ + readonly inputConnected: boolean; } /** @@ -39,6 +41,9 @@ export function DeviceStreamView(props: { const [screen, setScreen] = useState(null); const [mjpegUrl, setMjpegUrl] = useState(null); const [mjpegGeneration, setMjpegGeneration] = useState(0); + const [inputState, setInputState] = useState<{ connected: boolean; detail?: string }>({ + connected: false, + }); const { onHandle, onScreen } = props; useEffect(() => { @@ -68,12 +73,21 @@ export function DeviceStreamView(props: { setMjpegUrl(url); setMjpegGeneration((generation) => generation + 1); }, + onInputConnected: (connected, detail) => { + setInputState({ connected, ...(detail ? { detail } : {}) }); + onHandle?.({ + pressButton: client.pressButton, + rotate: client.rotate, + inputConnected: connected, + }); + }, }, ); clientRef.current = client; setMjpegUrl(null); + setInputState({ connected: false }); client.start(); - onHandle?.({ pressButton: client.pressButton, rotate: client.rotate }); + onHandle?.({ pressButton: client.pressButton, rotate: client.rotate, inputConnected: false }); return () => { client.stop(); clientRef.current = null; @@ -194,6 +208,13 @@ export function DeviceStreamView(props: { /> ) : null} + {status === "streaming" && !inputState.connected ? ( +
+ + Input disconnected{inputState.detail ? ` (${inputState.detail})` : ""}, reconnecting… + +
+ ) : null} {status !== "streaming" ? (
{status === "connecting" ? : null} diff --git a/apps/web/src/components/device/deviceStream.ts b/apps/web/src/components/device/deviceStream.ts index 94c2074982bd..03dce09d2bb1 100644 --- a/apps/web/src/components/device/deviceStream.ts +++ b/apps/web/src/components/device/deviceStream.ts @@ -39,6 +39,8 @@ export interface DeviceStreamEvents { * the canvas. */ readonly onMjpegFallback: (url: string) => void; + /** Whether touches and keys can currently reach the device. */ + readonly onInputConnected: (connected: boolean, detail?: string) => void; } export interface DeviceStreamTarget { @@ -465,7 +467,10 @@ export function createDeviceStreamClient( const ws = new WebSocket(wsUrl(`/helper/ws?device=${device}`)); ws.binaryType = "arraybuffer"; socket = ws; - ws.onopen = () => ws.send(taggedJson(IOS_MSG_HARDWARE_KEYBOARD, { enabled: false })); + ws.onopen = () => { + ws.send(taggedJson(IOS_MSG_HARDWARE_KEYBOARD, { enabled: false })); + events.onInputConnected(true); + }; ws.onmessage = (event) => { if (!(event.data instanceof ArrayBuffer)) return; const bytes = new Uint8Array(event.data); @@ -482,6 +487,12 @@ export function createDeviceStreamClient( }; ws.onclose = (event) => { if (socket === ws) socket = null; + if (!stopped) { + events.onInputConnected( + false, + event.reason || (event.code === 1006 ? "input socket refused" : `closed ${event.code}`), + ); + } if (event.code === 1008 || event.code === 4401) return handleUnauthorized(); scheduleRetry(connectIosInput); }; @@ -494,7 +505,10 @@ export function createDeviceStreamClient( const ws = new WebSocket(wsUrl(`/ws?device=${device}&frame-meta=1`)); ws.binaryType = "arraybuffer"; socket = ws; - ws.onopen = () => setStatus("connecting"); + ws.onopen = () => { + setStatus("connecting"); + events.onInputConnected(true); + }; ws.onmessage = (event) => { if (!(event.data instanceof ArrayBuffer)) return; const packet = parseSemuPacket(event.data); @@ -522,6 +536,7 @@ export function createDeviceStreamClient( ws.onclose = (event) => { if (socket === ws) socket = null; closeDecoder(); + if (!stopped) events.onInputConnected(false, event.reason || `closed ${event.code}`); if (event.code === 1008 || event.code === 4401) return handleUnauthorized(); if (!stopped) { setStatus("connecting", event.reason || undefined); From 8b045a91e274e11f1f9e36ab378cb2129fe151f1 Mon Sep 17 00:00:00 2001 From: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com> Date: Mon, 7 Sep 2026 22:36:37 -0700 Subject: [PATCH 08/28] fix(devices): prime the iOS helper before opening the input socket and allow the Device tab on draft threads serve-sim only accepts HID after screen capture is running, which the AVCC stream alone does not start; taps from the panel were silently dropped. Draft threads pre-allocate the ref the server thread inherits, so there is no reason to withhold the surface until the first message is sent. Co-Authored-By: Claude Fable 5 --- apps/web/src/components/ChatView.tsx | 8 ++--- apps/web/src/components/RightPanelTabs.tsx | 4 +-- .../web/src/components/device/deviceStream.ts | 29 +++++++++++++++++-- 3 files changed, 32 insertions(+), 9 deletions(-) diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index 62ede3749c58..f56e5af3d3d2 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -4132,9 +4132,9 @@ export default function ChatView(props: ChatViewProps) { useRightPanelStore.getState().open(activeThreadRef, "agents"); }, [activeThreadRef]); const addDeviceSurface = useCallback(() => { - if (!activeThreadRef || !isServerThread) return; + if (!activeThreadRef) return; useRightPanelStore.getState().open(activeThreadRef, "device"); - }, [activeThreadRef, isServerThread]); + }, [activeThreadRef]); // An agent's `device_open` surfaces in every client the same way a // `preview_open` does: the thread gains a device session and the panel // opens on it. Closing the last session leaves the tab in place so the @@ -8661,7 +8661,7 @@ export default function ChatView(props: ChatViewProps) { filesAvailable={activeProject !== null} pullRequestAvailable={pullRequestSurfaceAvailable} agentsAvailable - deviceAvailable={isServerThread} + deviceAvailable={activeThreadRef !== null} liveAgentCount={agentPanelModel.liveCount} > {rightPanelContent} @@ -8713,7 +8713,7 @@ export default function ChatView(props: ChatViewProps) { filesAvailable={activeProject !== null} pullRequestAvailable={pullRequestSurfaceAvailable} agentsAvailable - deviceAvailable={isServerThread} + deviceAvailable={activeThreadRef !== null} liveAgentCount={agentPanelModel.liveCount} > {rightPanelContent} diff --git a/apps/web/src/components/RightPanelTabs.tsx b/apps/web/src/components/RightPanelTabs.tsx index b6b897b23a1e..2d6db2ec8435 100644 --- a/apps/web/src/components/RightPanelTabs.tsx +++ b/apps/web/src/components/RightPanelTabs.tsx @@ -143,7 +143,7 @@ const SURFACE_DISABLED_REASONS = { diff: "Diff is only available for server threads in Git repositories.", pullRequest: "This thread's branch has no pull request yet.", agents: "Agents are only available from a thread.", - device: "Devices are only available from a server thread.", + device: "Devices are only available from a thread.", } as const; /** Overlays that must win over the launcher's letter shortcuts. */ @@ -166,7 +166,7 @@ const SURFACE_UNAVAILABLE_HINTS = { diff: "Available for Git repositories.", pullRequest: "No pull request on this branch yet.", agents: "Available from a thread.", - device: "Available from a server thread.", + device: "Available from a thread.", } as const; type TabContextMenuAction = diff --git a/apps/web/src/components/device/deviceStream.ts b/apps/web/src/components/device/deviceStream.ts index 03dce09d2bb1..99f247bbfce1 100644 --- a/apps/web/src/components/device/deviceStream.ts +++ b/apps/web/src/components/device/deviceStream.ts @@ -461,8 +461,31 @@ export function createDeviceStreamClient( if (!stopped) scheduleRetry(() => void readIosVideo()); }; + /** + * serve-sim's helper only accepts HID and pushes its screen config once + * screen capture is running, and the AVCC stream does not reliably start + * it. Touching the MJPEG endpoint does; one aborted request is enough. + */ + const primeIosHelper = async () => { + const controller = new AbortController(); + try { + const response = await fetch(httpUrl(`/helper/${device}/stream.mjpeg`), { + signal: controller.signal, + credentials: access.credentials ? "include" : "same-origin", + }); + if (response.status === 401 || response.status === 403) return handleUnauthorized(); + await response.body?.getReader().read(); + } catch { + // A failed prime just means the socket may take a retry to come up. + } finally { + controller.abort(); + } + }; + // iOS input socket; also carries the screen config the helper pushes. - const connectIosInput = () => { + const connectIosInput = async () => { + if (stopped) return; + await primeIosHelper(); if (stopped) return; const ws = new WebSocket(wsUrl(`/helper/ws?device=${device}`)); ws.binaryType = "arraybuffer"; @@ -494,7 +517,7 @@ export function createDeviceStreamClient( ); } if (event.code === 1008 || event.code === 4401) return handleUnauthorized(); - scheduleRetry(connectIosInput); + scheduleRetry(() => void connectIosInput()); }; ws.onerror = () => ws.close(); }; @@ -552,7 +575,7 @@ export function createDeviceStreamClient( firstFrame = false; events.onStatus("connecting"); if (platform === "ios") { - connectIosInput(); + void connectIosInput(); if (useWebCodecs) void readIosVideo(); else fallBackToMjpeg(); } else if (useWebCodecs) { From 28669ac11d0e6bea3d4f5225bd838f4956db4dec Mon Sep 17 00:00:00 2001 From: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com> Date: Mon, 7 Sep 2026 22:49:24 -0700 Subject: [PATCH 09/28] fix(devices): keep the device aspect ratio and shrink height when the panel is narrow Co-Authored-By: Claude Fable 5 --- .../components/device/DeviceStreamView.tsx | 72 +++++++++++++++---- 1 file changed, 60 insertions(+), 12 deletions(-) diff --git a/apps/web/src/components/device/DeviceStreamView.tsx b/apps/web/src/components/device/DeviceStreamView.tsx index 83e97f083601..fd4a4d06299e 100644 --- a/apps/web/src/components/device/DeviceStreamView.tsx +++ b/apps/web/src/components/device/DeviceStreamView.tsx @@ -105,8 +105,9 @@ export function DeviceStreamView(props: { props.visible, ]); + // Displayed aspect ratio (width / height) of the device as the user sees it. const aspect = useMemo(() => { - if (!screen) return props.platform === "ios" ? "9 / 19.5" : "9 / 20"; + if (!screen) return props.platform === "ios" ? 9 / 19.5 : 9 / 20; const landscape = screen.orientation === "landscape_left" || screen.orientation === "landscape_right"; const w = landscape @@ -115,9 +116,39 @@ export function DeviceStreamView(props: { const h = landscape ? Math.min(screen.width, screen.height) : Math.max(screen.width, screen.height); - return `${w} / ${h}`; + return w / h; }, [props.platform, screen]); + // The frame is the largest box at `aspect` that fits the container, so a + // narrow panel shows a shorter phone rather than a squeezed one. CSS + // `aspect-ratio` alone cannot do this: with the height pinned to 100% the + // width clamp wins and distorts the drawn frame. + const hostRef = useRef(null); + const [host, setHost] = useState({ width: 0, height: 0 }); + useEffect(() => { + const element = hostRef.current; + if (!element) return; + const update = () => { + const rect = element.getBoundingClientRect(); + setHost((current) => + current.width === rect.width && current.height === rect.height + ? current + : { width: rect.width, height: rect.height }, + ); + }; + update(); + const observer = new ResizeObserver(update); + observer.observe(element); + return () => observer.disconnect(); + }, []); + const frame = useMemo(() => { + if (host.width === 0 || host.height === 0) return { width: 0, height: 0 }; + const byHeight = { width: host.height * aspect, height: host.height }; + return byHeight.width <= host.width + ? byHeight + : { width: host.width, height: host.width / aspect }; + }, [aspect, host]); + // serve-sim streams the raw framebuffer; rotate the display for a device // that reports landscape while its frames stay portrait. const rotation = useMemo(() => { @@ -134,6 +165,26 @@ export function DeviceStreamView(props: { } }, [props.platform, screen]); + // A sideways rotation draws the raw portrait frame into a landscape box: + // the media element takes the transposed size and is rotated about the + // box's center. + const sideways = rotation === 90 || rotation === -90; + const mediaStyle: React.CSSProperties = sideways + ? { + width: frame.height, + height: frame.width, + left: (frame.width - frame.height) / 2, + top: (frame.height - frame.width) / 2, + transform: `rotate(${rotation}deg)`, + } + : { + width: frame.width, + height: frame.height, + left: 0, + top: 0, + ...(rotation ? { transform: `rotate(${rotation}deg)` } : {}), + }; + const pointerActive = useRef(false); const normalizedPoint = (event: React.PointerEvent) => { const rect = event.currentTarget.getBoundingClientRect(); @@ -147,6 +198,7 @@ export function DeviceStreamView(props: { return (
{ event.currentTarget.setPointerCapture(event.pointerId); (event.currentTarget.parentElement as HTMLElement | null)?.focus(); @@ -194,8 +242,8 @@ export function DeviceStreamView(props: { > {mjpegUrl ? ( ) : null}
From db49a5e7be3eb581ca22f4e84fa4d3f45f154378 Mon Sep 17 00:00:00 2001 From: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com> Date: Mon, 7 Sep 2026 23:35:31 -0700 Subject: [PATCH 10/28] feat(devices): add a Tools drawer for simulator and emulator settings The Device panel only streamed and forwarded input; serve-sim's own preview has a Tools panel for appearance, text size, accessibility toggles, location, permissions and more. Porting it means exposing device state changes without proxying serve-sim's shell-exec channel, so the server now owns them: a device.action RPC runs simctl, adb and serve-sim's bundled accessibility helper directly per typed action, and device.detail reads the settings back. The web panel gets a Tools drawer with per-platform controls (Liquid Glass, color filters, VoiceOver and push on iOS; orientation and network on Android), an accessibility frame overlay, and the serve-sim event log. The proxy allowlists the read-only routes those need and rejects non-GET methods outside screenshot capture and stream tuning. Made with Claude Fable 5 via Claude Code. Co-Authored-By: Claude Fable 5 --- apps/server/src/auth/RpcAuthorization.ts | 2 + apps/server/src/device/DeviceActions.test.ts | 246 ++++++ apps/server/src/device/DeviceActions.ts | 501 ++++++++++++ apps/server/src/device/DeviceHost.ts | 15 + apps/server/src/device/DeviceHubProxy.ts | 13 +- apps/server/src/device/DeviceService.ts | 44 ++ apps/server/src/device/LocalDeviceHost.ts | 51 +- apps/server/src/ws.ts | 8 + .../web/src/components/device/DevicePanel.tsx | Bin 11165 -> 12531 bytes .../components/device/DeviceStreamView.tsx | 55 ++ .../components/device/DeviceToolsPanel.tsx | 722 ++++++++++++++++++ .../web/src/components/device/deviceHubApi.ts | 214 ++++++ docs/internals/devices.md | 12 + docs/user/devices.md | 11 + packages/client-runtime/src/state/device.ts | 12 + packages/contracts/src/device.ts | 162 ++++ packages/contracts/src/rpc.ts | 19 + 17 files changed, 2085 insertions(+), 2 deletions(-) create mode 100644 apps/server/src/device/DeviceActions.test.ts create mode 100644 apps/server/src/device/DeviceActions.ts create mode 100644 apps/web/src/components/device/DeviceToolsPanel.tsx create mode 100644 apps/web/src/components/device/deviceHubApi.ts diff --git a/apps/server/src/auth/RpcAuthorization.ts b/apps/server/src/auth/RpcAuthorization.ts index e07ddfb71517..24b6f5cb58d0 100644 --- a/apps/server/src/auth/RpcAuthorization.ts +++ b/apps/server/src/auth/RpcAuthorization.ts @@ -146,6 +146,8 @@ export const RPC_REQUIRED_SCOPES = { [WS_METHODS.deviceOpen]: AuthOrchestrationOperateScope, [WS_METHODS.deviceClose]: AuthOrchestrationOperateScope, [WS_METHODS.deviceShutdown]: AuthOrchestrationOperateScope, + [WS_METHODS.deviceDetail]: AuthOrchestrationReadScope, + [WS_METHODS.deviceAction]: AuthOrchestrationOperateScope, [WS_METHODS.subscribeDeviceState]: AuthOrchestrationReadScope, [WS_METHODS.subscribeServerConfig]: AuthOrchestrationReadScope, [WS_METHODS.subscribeServerLifecycle]: AuthOrchestrationReadScope, diff --git a/apps/server/src/device/DeviceActions.test.ts b/apps/server/src/device/DeviceActions.test.ts new file mode 100644 index 000000000000..d1ecd9fa670e --- /dev/null +++ b/apps/server/src/device/DeviceActions.test.ts @@ -0,0 +1,246 @@ +import { describe, expect, it } from "@effect/vitest"; +import type { DeviceActionInput } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; + +import { readDeviceDetail, runDeviceAction, supportsAction } from "./DeviceActions.ts"; +import type { DeviceHostReady } from "./DeviceHost.ts"; + +type Call = { command: string; args: ReadonlyArray; stdin?: string }; + +const makeReady = ( + respond: (call: Call) => { stdout?: string; stderr?: string; code?: number } = () => ({}), + helpers: DeviceHostReady["helpers"] = { + serveSimAxSettings: "/hub/simax/serve-sim-ax-settings", + serveSimCli: "/hub/serve-sim.js", + }, +) => { + const calls: Call[] = []; + const ready: DeviceHostReady = { + hub: { origin: "http://127.0.0.1:1" }, + agentDevice: { baseUrl: "http://127.0.0.1:2", token: "t", entryPath: "/x" }, + helpers, + run: (command, args, options) => { + const call = { command, args, ...(options?.stdin ? { stdin: options.stdin } : {}) }; + calls.push(call); + const result = respond(call); + return Effect.succeed({ + stdout: result.stdout ?? "", + stderr: result.stderr ?? "", + code: result.code ?? 0, + }); + }, + }; + return { ready, calls }; +}; + +const udid = "SIM-1"; + +describe("supportsAction", () => { + it("advertises platform-specific toggles", () => { + const toggle = (setting: Extract["setting"]) => + ({ type: "setToggle", deviceId: udid, setting, value: true }) as const; + expect(supportsAction("ios", toggle("voiceOver"))).toBe(true); + expect(supportsAction("android", toggle("voiceOver"))).toBe(false); + expect(supportsAction("android", toggle("networkEnabled"))).toBe(true); + expect(supportsAction("ios", toggle("networkEnabled"))).toBe(false); + expect(supportsAction("ios", { type: "setLiquidGlass", deviceId: udid, value: "clear" })).toBe( + true, + ); + expect( + supportsAction("android", { type: "setLiquidGlass", deviceId: udid, value: "clear" }), + ).toBe(false); + expect( + supportsAction("android", { type: "setOrientation", deviceId: udid, value: "portrait" }), + ).toBe(true); + expect( + supportsAction("ios", { type: "setOrientation", deviceId: udid, value: "portrait" }), + ).toBe(false); + }); +}); + +describe("runDeviceAction", () => { + it.effect("maps shared text sizes onto simctl content-size categories", () => + Effect.gen(function* () { + const { ready, calls } = makeReady(); + yield* runDeviceAction(ready, "ios", { + type: "setTextSize", + deviceId: udid, + value: "extra-large", + }); + expect(calls).toEqual([ + { command: "xcrun", args: ["simctl", "ui", udid, "content_size", "accessibility-large"] }, + ]); + }), + ); + + it.effect("maps shared text sizes onto Android font_scale", () => + Effect.gen(function* () { + const { ready, calls } = makeReady(); + yield* runDeviceAction(ready, "android", { + type: "setTextSize", + deviceId: "emulator-5554", + value: "large", + }); + expect(calls[0]?.args).toEqual([ + "-s", + "emulator-5554", + "shell", + "settings", + "put", + "system", + "font_scale", + "1.15", + ]); + }), + ); + + it.effect("runs accessibility toggles through the bundled helper via simctl spawn", () => + Effect.gen(function* () { + const { ready, calls } = makeReady(); + yield* runDeviceAction(ready, "ios", { + type: "setToggle", + deviceId: udid, + setting: "voiceOver", + value: true, + }); + expect(calls).toEqual([ + { + command: "xcrun", + args: [ + "simctl", + "spawn", + udid, + "/hub/simax/serve-sim-ax-settings", + "set", + "voiceover", + "on", + ], + }, + ]); + }), + ); + + it.effect("fails clearly when the helper is missing", () => + Effect.gen(function* () { + const { ready } = makeReady(() => ({}), { serveSimAxSettings: null, serveSimCli: null }); + const error = yield* Effect.flip( + runDeviceAction(ready, "ios", { + type: "setColorFilter", + deviceId: udid, + value: "grayscale", + }), + ); + expect(error._tag).toBe("DeviceOperationError"); + expect(error.detail).toContain("accessibility helper"); + }), + ); + + it.effect("rejects actions the platform does not support without running anything", () => + Effect.gen(function* () { + const { ready, calls } = makeReady(); + const error = yield* Effect.flip( + runDeviceAction(ready, "android", { + type: "sendPush", + deviceId: "emulator-5554", + appId: "com.example", + payload: "hi", + }), + ); + expect(error.detail).toContain("not supported on android"); + expect(calls).toEqual([]); + }), + ); + + it.effect("surfaces non-zero exit codes as operation errors", () => + Effect.gen(function* () { + const { ready } = makeReady(() => ({ code: 1, stderr: "Invalid device: SIM-1" })); + const error = yield* Effect.flip( + runDeviceAction(ready, "ios", { type: "setAppearance", deviceId: udid, value: "dark" }), + ); + expect(error.operation).toBe("appearance"); + expect(error.detail).toBe("Invalid device: SIM-1"); + }), + ); + + it.effect("wraps a bare push string in an APNs alert and feeds it on stdin", () => + Effect.gen(function* () { + const { ready, calls } = makeReady(); + yield* runDeviceAction(ready, "ios", { + type: "sendPush", + deviceId: udid, + appId: "com.example.app", + payload: "Hello", + }); + expect(calls[0]?.args).toEqual(["simctl", "push", udid, "com.example.app", "-"]); + expect(calls[0]?.stdin).toBe('{"aps":{"alert":"Hello"}}'); + }), + ); +}); + +describe("readDeviceDetail", () => { + it.effect("reads iOS settings from simctl and the accessibility helper", () => + Effect.gen(function* () { + const { ready } = makeReady((call) => { + const key = call.args.join(" "); + if (key.endsWith("ui SIM-1 appearance")) return { stdout: "dark\n" }; + if (key.endsWith("ui SIM-1 content_size")) return { stdout: "extra-extra-large\n" }; + if (key.endsWith("ui SIM-1 increase_contrast")) return { stdout: "enabled\n" }; + if (key.includes("serve-sim-ax-settings status")) { + return { + stdout: + '{"reduce-motion":"on","reduce-transparency":"off","show-borders":"off","voiceover":"off","liquid-glass":"tinted","color-filter":"grayscale"}', + }; + } + return { code: 1 }; + }); + const detail = yield* readDeviceDetail(ready, "ios", udid); + expect(detail.settings).toEqual({ + appearance: "dark", + textSize: "large", + increaseContrast: true, + reduceMotion: true, + reduceTransparency: false, + showBorders: false, + voiceOver: false, + liquidGlass: "tinted", + colorFilter: "grayscale", + }); + }), + ); + + it.effect("degrades unreadable values to unknown instead of failing", () => + Effect.gen(function* () { + const { ready } = makeReady(() => ({ code: 1, stderr: "boom" })); + const detail = yield* readDeviceDetail(ready, "ios", udid); + expect(detail.settings).toEqual({}); + expect(detail.foregroundApp).toBeNull(); + }), + ); + + it.effect("reads Android settings and the focused package", () => + Effect.gen(function* () { + const { ready } = makeReady((call) => { + const key = call.args.join(" "); + if (key.endsWith("cmd uimode night")) return { stdout: "Night mode: yes\n" }; + if (key.endsWith("font_scale")) return { stdout: "0.85\n" }; + if (key.endsWith("animator_duration_scale")) return { stdout: "0\n" }; + if (key.endsWith("wifi_on")) return { stdout: "1\n" }; + if (key.endsWith("dumpsys window windows")) { + return { + stdout: + " mCurrentFocus=Window{1a2b u0 com.example.app/com.example.app.MainActivity}\n", + }; + } + return { code: 1 }; + }); + const detail = yield* readDeviceDetail(ready, "android", "emulator-5554"); + expect(detail.settings).toEqual({ + appearance: "dark", + textSize: "small", + reduceMotion: true, + networkEnabled: true, + }); + expect(detail.foregroundApp).toEqual({ id: "com.example.app" }); + }), + ); +}); diff --git a/apps/server/src/device/DeviceActions.ts b/apps/server/src/device/DeviceActions.ts new file mode 100644 index 000000000000..39300c000fe4 --- /dev/null +++ b/apps/server/src/device/DeviceActions.ts @@ -0,0 +1,501 @@ +/** + * Device settings and one-shot actions, run by the server against the host's + * toolchain instead of through serve-sim's shell-exec channel. + * + * serve-sim's preview drives its "Simulator" panel by sending shell commands + * over a token-gated socket. Proxying that would hand any environment + * session arbitrary command execution on the host, so T3 runs the same + * underlying commands itself, typed per action: `xcrun simctl ui` and + * `simctl privacy` for iOS, the `serve-sim-ax-settings` helper that serve-sim + * bundles for the accessibility toggles, and `adb shell` for Android. + * + * Each platform advertises which actions it supports; the panel hides the + * rest rather than showing controls that cannot work. + */ +import { + type DeviceActionInput, + type DeviceActionType, + type DeviceForegroundApp, + DeviceOperationError, + type DevicePlatform, + type DeviceSettings, + type DeviceTextSize, +} from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Schema from "effect/Schema"; + +import type { DeviceHostReady } from "./DeviceHost.ts"; + +type Runner = DeviceHostReady["run"]; + +const decodeAxStatus = Schema.decodeUnknownEffect( + Schema.fromJsonString(Schema.Record(Schema.String, Schema.String)), +); +const encodePushPayload = Schema.encodeUnknownEffect( + Schema.fromJsonString(Schema.Record(Schema.String, Schema.Unknown)), +); + +export const IOS_ACTIONS: ReadonlySet = new Set([ + "setAppearance", + "setTextSize", + "setToggle", + "setLiquidGlass", + "setColorFilter", + "setLocation", + "clearLocation", + "setPermission", + "openUrl", + "launchApp", + "terminateApp", + "sendPush", +]); + +export const ANDROID_ACTIONS: ReadonlySet = new Set([ + "setAppearance", + "setTextSize", + "setToggle", + "setOrientation", + "setLocation", + "clearLocation", + "setPermission", + "openUrl", + "launchApp", + "terminateApp", +]); + +/** Toggle settings each platform can actually flip. */ +const IOS_TOGGLES = new Set([ + "reduceMotion", + "increaseContrast", + "reduceTransparency", + "showBorders", + "voiceOver", +]); +const ANDROID_TOGGLES = new Set(["reduceMotion", "networkEnabled"]); + +export const supportsAction = (platform: DevicePlatform, input: DeviceActionInput): boolean => { + const actions = platform === "ios" ? IOS_ACTIONS : ANDROID_ACTIONS; + if (!actions.has(input.type)) return false; + if (input.type === "setToggle") { + return (platform === "ios" ? IOS_TOGGLES : ANDROID_TOGGLES).has(input.setting); + } + return true; +}; + +const fail = (operation: string, detail: string) => + new DeviceOperationError({ operation, detail: detail.trim() || "command failed" }); + +const ok = (operation: string) => (result: { code: number; stderr: string; stdout: string }) => + result.code === 0 + ? Effect.succeed(result.stdout) + : Effect.fail(fail(operation, result.stderr || result.stdout)); + +// iOS text-size categories in ascending order; the four shared steps index +// into it. `default` is what a fresh simulator reports ("large"). +const IOS_TEXT_SIZES: Record = { + small: "small", + default: "large", + large: "extra-extra-large", + "extra-large": "accessibility-large", +}; +const ANDROID_TEXT_SIZES: Record = { + small: "0.85", + default: "1.0", + large: "1.15", + "extra-large": "1.3", +}; + +const textSizeFromIos = (category: string): DeviceTextSize | undefined => { + const entry = (Object.entries(IOS_TEXT_SIZES) as Array<[DeviceTextSize, string]>).find( + ([, value]) => value === category, + ); + if (entry) return entry[0]; + if (category.startsWith("accessibility")) return "extra-large"; + if (category.includes("extra")) return "large"; + return category === "extra-small" || category === "small" || category === "medium" + ? "small" + : "default"; +}; + +const textSizeFromAndroid = (scale: number): DeviceTextSize => { + if (scale <= 0.9) return "small"; + if (scale >= 1.25) return "extra-large"; + if (scale >= 1.1) return "large"; + return "default"; +}; + +const IOS_TOGGLE_OPTIONS: Record = { + reduceMotion: "reduce-motion", + increaseContrast: "increase-contrast", + reduceTransparency: "reduce-transparency", + showBorders: "show-borders", + voiceOver: "voiceover", +}; + +// serve-sim permission names -> the TCC service or simctl privacy service. +const IOS_TCC_SERVICES: Record = { + camera: "camera", + microphone: "microphone", + photos: "photos", + contacts: "contacts", + calendar: "calendar", + reminders: "reminders", + motion: "motion", + "media-library": "media-library", + faceid: "faceid", +}; + +const ANDROID_PERMISSIONS: Record> = { + camera: ["android.permission.CAMERA"], + microphone: ["android.permission.RECORD_AUDIO"], + photos: ["android.permission.READ_MEDIA_IMAGES", "android.permission.READ_EXTERNAL_STORAGE"], + contacts: ["android.permission.READ_CONTACTS", "android.permission.WRITE_CONTACTS"], + calendar: ["android.permission.READ_CALENDAR", "android.permission.WRITE_CALENDAR"], + location: [ + "android.permission.ACCESS_FINE_LOCATION", + "android.permission.ACCESS_COARSE_LOCATION", + ], + notifications: ["android.permission.POST_NOTIFICATIONS"], + motion: ["android.permission.ACTIVITY_RECOGNITION"], +}; + +export const runDeviceAction = Effect.fn("DeviceActions.run")(function* ( + ready: DeviceHostReady, + platform: DevicePlatform, + input: DeviceActionInput, +) { + if (!supportsAction(platform, input)) { + return yield* fail(input.type, `${input.type} is not supported on ${platform}.`); + } + if (platform === "ios") return yield* runIos(ready, input); + return yield* runAndroid(ready.run, input); +}); + +const simctl = (run: Runner, udid: string, args: ReadonlyArray, operation: string) => + run("xcrun", ["simctl", ...args.slice(0, 1), udid, ...args.slice(1)]).pipe( + Effect.flatMap(ok(operation)), + ); + +const axSettings = (ready: DeviceHostReady, udid: string, args: ReadonlyArray) => + Effect.gen(function* () { + const helper = ready.helpers.serveSimAxSettings; + if (!helper) { + return yield* fail( + "accessibility", + "The bundled accessibility helper is missing from this expo-device-hub install.", + ); + } + return yield* ready + .run("xcrun", ["simctl", "spawn", udid, helper, ...args]) + .pipe(Effect.flatMap(ok("accessibility"))); + }); + +const runIos = Effect.fn("DeviceActions.runIos")(function* ( + ready: DeviceHostReady, + input: DeviceActionInput, +) { + const udid = input.deviceId; + const { run } = ready; + switch (input.type) { + case "setAppearance": + yield* simctl(run, udid, ["ui", "appearance", input.value], "appearance"); + return; + case "setTextSize": + yield* simctl(run, udid, ["ui", "content_size", IOS_TEXT_SIZES[input.value]], "text size"); + return; + case "setToggle": + if (input.setting === "increaseContrast") { + yield* simctl( + run, + udid, + ["ui", "increase_contrast", input.value ? "enabled" : "disabled"], + "increase contrast", + ); + return; + } + yield* axSettings(ready, udid, [ + "set", + IOS_TOGGLE_OPTIONS[input.setting]!, + input.value ? "on" : "off", + ]); + return; + case "setLiquidGlass": + yield* axSettings(ready, udid, ["set", "liquid-glass", input.value]); + return; + case "setColorFilter": + yield* axSettings(ready, udid, ["set", "color-filter", input.value]); + return; + case "setLocation": + yield* simctl( + run, + udid, + ["location", "set", `${input.latitude},${input.longitude}`], + "location", + ); + return; + case "clearLocation": + yield* simctl(run, udid, ["location", "clear"], "location"); + return; + case "setPermission": { + if (input.permission === "notifications") { + // simctl has no notification permission verb; serve-sim's CLI edits + // the BulletinBoard plist for it. + yield* serveSimPermissions(ready, udid, input); + return; + } + const service = + input.permission === "location" ? "location" : IOS_TCC_SERVICES[input.permission]; + if (!service) return yield* fail("permission", `Unknown permission ${input.permission}.`); + yield* simctl(run, udid, ["privacy", input.decision, service, input.appId], "permission"); + return; + } + case "openUrl": + yield* simctl(run, udid, ["openurl", input.url], "open url"); + return; + case "launchApp": + yield* simctl(run, udid, ["launch", input.appId], "launch"); + return; + case "terminateApp": + yield* simctl(run, udid, ["terminate", input.appId], "terminate"); + return; + case "sendPush": { + const payload = + typeof input.payload === "string" ? { aps: { alert: input.payload } } : input.payload; + const encoded = yield* encodePushPayload(payload).pipe( + Effect.mapError((cause) => fail("push", String(cause))), + ); + yield* run("xcrun", ["simctl", "push", udid, input.appId, "-"], { stdin: encoded }).pipe( + Effect.flatMap(ok("push")), + ); + return; + } + case "shake": + case "setOrientation": + return yield* fail(input.type, `${input.type} is not supported on iOS.`); + } +}); + +const serveSimPermissions = ( + ready: DeviceHostReady, + udid: string, + input: Extract, +) => + Effect.gen(function* () { + const cli = ready.helpers.serveSimCli; + if (!cli) return yield* fail("permission", "serve-sim's CLI is missing from this install."); + yield* ready + .run(process.execPath, [ + cli, + "permissions", + input.decision, + input.permission, + input.appId, + "-d", + udid, + ]) + .pipe(Effect.flatMap(ok("permission"))); + }); + +const adb = (run: Runner, serial: string, args: ReadonlyArray, operation: string) => + run("adb", ["-s", serial, ...args]).pipe(Effect.flatMap(ok(operation))); + +const runAndroid = Effect.fn("DeviceActions.runAndroid")(function* ( + run: Runner, + input: DeviceActionInput, +) { + const serial = input.deviceId; + const shell = (args: ReadonlyArray, operation: string) => + adb(run, serial, ["shell", ...args], operation); + switch (input.type) { + case "setAppearance": + yield* shell(["cmd", "uimode", "night", input.value === "dark" ? "yes" : "no"], "appearance"); + return; + case "setTextSize": + yield* shell( + ["settings", "put", "system", "font_scale", ANDROID_TEXT_SIZES[input.value]], + "text size", + ); + return; + case "setToggle": + if (input.setting === "networkEnabled") { + const state = input.value ? "enable" : "disable"; + yield* shell(["svc", "wifi", state], "network"); + yield* shell(["svc", "data", state], "network"); + return; + } + if (input.setting === "reduceMotion") { + const scale = input.value ? "0" : "1"; + for (const key of [ + "animator_duration_scale", + "transition_animation_scale", + "window_animation_scale", + ]) { + yield* shell(["settings", "put", "global", key, scale], "reduce motion"); + } + return; + } + return yield* fail(input.type, `${input.setting} is not supported on Android.`); + case "setOrientation": { + const rotation = + input.value === "portrait" + ? "0" + : input.value === "landscape_left" + ? "1" + : input.value === "portrait_upside_down" + ? "2" + : "3"; + yield* shell(["cmd", "window", "user-rotation", "lock", rotation], "orientation"); + return; + } + case "setLocation": + yield* adb( + run, + serial, + ["emu", "geo", "fix", String(input.longitude), String(input.latitude)], + "location", + ); + return; + case "clearLocation": + // The emulator has no "clear"; leaving the fix in place is the closest + // behavior, so this is a no-op that still refreshes the reading. + return; + case "setPermission": { + const permissions = ANDROID_PERMISSIONS[input.permission]; + if (!permissions) { + return yield* fail("permission", `${input.permission} has no Android equivalent.`); + } + const verb = input.decision === "grant" ? "grant" : "revoke"; + for (const permission of permissions) { + // Not every app declares every permission in a group; ignore those. + yield* shell(["pm", verb, input.appId, permission], "permission").pipe(Effect.ignore); + } + return; + } + case "openUrl": + yield* shell( + ["am", "start", "-a", "android.intent.action.VIEW", "-d", input.url], + "open url", + ); + return; + case "launchApp": + yield* shell( + ["monkey", "-p", input.appId, "-c", "android.intent.category.LAUNCHER", "1"], + "launch", + ); + return; + case "terminateApp": + yield* shell(["am", "force-stop", input.appId], "terminate"); + return; + case "setLiquidGlass": + case "setColorFilter": + case "shake": + case "sendPush": + return yield* fail(input.type, `${input.type} is not supported on Android.`); + } +}); + +/** Read the current settings and foreground app. Errors degrade to unknowns. */ +export const readDeviceDetail = Effect.fn("DeviceActions.readDetail")(function* ( + ready: DeviceHostReady, + platform: DevicePlatform, + deviceId: string, +): Effect.fn.Return<{ settings: DeviceSettings; foregroundApp: DeviceForegroundApp | null }> { + return platform === "ios" + ? yield* readIos(ready, deviceId) + : yield* readAndroid(ready.run, deviceId); +}); + +const quiet = (effect: Effect.Effect) => + effect.pipe(Effect.orElseSucceed((): A | undefined => undefined)); + +const readIos = Effect.fn("DeviceActions.readIos")(function* ( + ready: DeviceHostReady, + udid: string, +) { + const { run } = ready; + const uiValue = (option: string) => + quiet( + simctl(run, udid, ["ui", option], option).pipe(Effect.map((out) => out.trim().toLowerCase())), + ); + const [appearance, contentSize, contrast, axStatus] = yield* Effect.all( + [ + uiValue("appearance"), + uiValue("content_size"), + uiValue("increase_contrast"), + quiet(axSettings(ready, udid, ["status"]).pipe(Effect.flatMap(decodeAxStatus))), + ], + { concurrency: 4 }, + ); + const onOff = (value: string | undefined) => + value === "on" ? true : value === "off" ? false : undefined; + const settings: DeviceSettings = { + ...(appearance === "light" || appearance === "dark" ? { appearance } : {}), + ...(contentSize ? { textSize: textSizeFromIos(contentSize) } : {}), + ...(contrast ? { increaseContrast: contrast === "enabled" } : {}), + ...(axStatus + ? { + ...(onOff(axStatus["reduce-motion"]) === undefined + ? {} + : { reduceMotion: onOff(axStatus["reduce-motion"]) }), + ...(onOff(axStatus["reduce-transparency"]) === undefined + ? {} + : { reduceTransparency: onOff(axStatus["reduce-transparency"]) }), + ...(onOff(axStatus["show-borders"]) === undefined + ? {} + : { showBorders: onOff(axStatus["show-borders"]) }), + ...(onOff(axStatus.voiceover) === undefined + ? {} + : { voiceOver: onOff(axStatus.voiceover) }), + ...(axStatus["liquid-glass"] === "clear" || axStatus["liquid-glass"] === "tinted" + ? { liquidGlass: axStatus["liquid-glass"] } + : {}), + ...(isColorFilter(axStatus["color-filter"]) + ? { colorFilter: axStatus["color-filter"] } + : {}), + } + : {}), + }; + return { settings, foregroundApp: null }; +}); + +const isColorFilter = (value: unknown): value is DeviceSettings["colorFilter"] & string => + value === "none" || + value === "grayscale" || + value === "red-green" || + value === "green-red" || + value === "blue-yellow"; + +const readAndroid = Effect.fn("DeviceActions.readAndroid")(function* (run: Runner, serial: string) { + const shell = (args: ReadonlyArray) => + quiet( + adb(run, serial, ["shell", ...args], args[0] ?? "shell").pipe(Effect.map((s) => s.trim())), + ); + const [night, fontScale, animator, wifi, focus] = yield* Effect.all( + [ + shell(["cmd", "uimode", "night"]), + shell(["settings", "get", "system", "font_scale"]), + shell(["settings", "get", "global", "animator_duration_scale"]), + shell(["settings", "get", "global", "wifi_on"]), + shell(["dumpsys", "window", "windows"]), + ], + { concurrency: 5 }, + ); + const scale = fontScale && fontScale !== "null" ? Number(fontScale) : Number.NaN; + const focused = focus?.match(/mCurrentFocus=Window\{[^ ]+ u\d+ ([^/ ]+)\/([^ }]+)\}/); + const settings: DeviceSettings = { + ...(night?.includes("yes") + ? { appearance: "dark" } + : night?.includes("no") + ? { appearance: "light" } + : {}), + ...(Number.isFinite(scale) ? { textSize: textSizeFromAndroid(scale) } : {}), + ...(animator !== undefined && animator !== "null" + ? { reduceMotion: Number(animator) === 0 } + : {}), + ...(wifi === "1" || wifi === "0" ? { networkEnabled: wifi === "1" } : {}), + }; + return { + settings, + foregroundApp: focused ? { id: focused[1]! } : null, + }; +}); diff --git a/apps/server/src/device/DeviceHost.ts b/apps/server/src/device/DeviceHost.ts index 0a2017255d15..bc162f9a109b 100644 --- a/apps/server/src/device/DeviceHost.ts +++ b/apps/server/src/device/DeviceHost.ts @@ -43,6 +43,21 @@ export interface AgentDeviceEndpoint { export interface DeviceHostReady { readonly hub: DeviceHubEndpoint; readonly agentDevice: AgentDeviceEndpoint; + /** + * Runs a host command (`xcrun`, `adb`, or a helper bundled with the hub) + * where the devices live. On the local host this is a plain spawn; a + * remote host would run it over its transport. + */ + readonly run: ( + command: string, + args: ReadonlyArray, + options?: { readonly timeoutMs?: number; readonly stdin?: string }, + ) => Effect.Effect<{ readonly stdout: string; readonly stderr: string; readonly code: number }>; + /** Absolute paths of helper binaries vendored with the hub, when present. */ + readonly helpers: { + readonly serveSimAxSettings: string | null; + readonly serveSimCli: string | null; + }; } export interface DeviceHost { diff --git a/apps/server/src/device/DeviceHubProxy.ts b/apps/server/src/device/DeviceHubProxy.ts index cce7f1176af4..0822cac5b5d4 100644 --- a/apps/server/src/device/DeviceHubProxy.ts +++ b/apps/server/src/device/DeviceHubProxy.ts @@ -38,12 +38,19 @@ const ALLOWED_PATHS: ReadonlyArray = [ /^\/api\/devices$/, /^\/vendor\/serve-sim\/api$/, /^\/vendor\/serve-sim\/api\/screenshot$/, + /^\/vendor\/serve-sim\/api\/event-log(\/events)?$/, /^\/vendor\/serve-sim\/helper\/[^/]+\/(stream\.mjpeg|stream\.avcc|config|health|ax|foreground)$/, /^\/vendor\/serve-sim\/appstate$/, - /^\/vendor\/serve-emu\/api\/(devices|screenshot|stream-mode|stream-settings)$/, + /^\/vendor\/serve-emu\/api\/(devices|screenshot|stream-mode|stream-settings|accessibility)$/, /^\/vendor\/serve-emu\/health$/, ]; +/** Read paths are GET-only; only these accept other methods (screenshot captures, stream tuning). */ +const MUTABLE_PATHS: ReadonlyArray = [ + /^\/vendor\/serve-sim\/api\/screenshot$/, + /^\/vendor\/serve-emu\/api\/(screenshot|stream-mode|stream-settings)$/, +]; + const ALLOWED_WS_PATHS: ReadonlyArray = [ /^\/api\/devices\/ws$/, /^\/vendor\/serve-sim\/helper\/ws$/, @@ -179,6 +186,10 @@ const handler = Effect.gen(function* () { if (!allowed) { return HttpServerResponse.text("Not Found", { status: 404 }); } + const readOnly = request.method === "GET" || request.method === "HEAD"; + if (!upgrade && !readOnly && !MUTABLE_PATHS.some((pattern) => pattern.test(hubPath))) { + return HttpServerResponse.text("Method Not Allowed", { status: 405 }); + } yield* authenticate; const devices = yield* DeviceService; const ready = yield* devices.currentReadiness(); diff --git a/apps/server/src/device/DeviceService.ts b/apps/server/src/device/DeviceService.ts index 48c27ee7268f..46267a72490d 100644 --- a/apps/server/src/device/DeviceService.ts +++ b/apps/server/src/device/DeviceService.ts @@ -11,7 +11,10 @@ * every connected client the way `preview_open` does. */ import { + type DeviceActionInput, type DeviceCloseInput, + type DeviceDetail, + type DeviceDetailInput, type DeviceError, type DeviceHostId, type DeviceId, @@ -41,6 +44,7 @@ import * as Stream from "effect/Stream"; import * as SynchronizedRef from "effect/SynchronizedRef"; import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"; +import { readDeviceDetail, runDeviceAction } from "./DeviceActions.ts"; import type { DeviceHost, DeviceHostReady } from "./DeviceHost.ts"; import * as LocalDeviceHost from "./LocalDeviceHost.ts"; @@ -89,6 +93,10 @@ export class DeviceService extends Context.Service< readonly open: (input: DeviceOpenInput) => Effect.Effect; readonly close: (input: DeviceCloseInput) => Effect.Effect; readonly shutdown: (input: DeviceShutdownInput) => Effect.Effect; + /** Current settings and foreground app for one device. */ + readonly detail: (input: DeviceDetailInput) => Effect.Effect; + /** Runs one action, then returns the refreshed detail. */ + readonly action: (input: DeviceActionInput) => Effect.Effect; readonly screenshot: (input: { readonly hostId?: DeviceHostId | undefined; readonly deviceId: DeviceId; @@ -459,6 +467,40 @@ export const make = Effect.gen(function* () { }, ); + const resolveDevice = Effect.fn("DeviceService.resolveDevice")(function* ( + hostId: DeviceHostId | undefined, + deviceId: DeviceId, + ) { + const host = yield* resolveHost(hostId); + const ready = yield* readiness(host.id); + const { state } = yield* SynchronizedRef.get(stateRef); + const device = findDevice(state, host.id, deviceId); + if (!device) return yield* new DeviceNotFoundError({ hostId: host.id, deviceId }); + return { ready, device }; + }); + + const detail: DeviceService["Service"]["detail"] = Effect.fn("DeviceService.detail")( + function* (input) { + const { ready, device } = yield* resolveDevice(input.hostId, input.deviceId); + const read = yield* readDeviceDetail(ready, device.platform, device.id); + return { + hostId: ready.hostId, + deviceId: device.id, + settings: read.settings, + foregroundApp: read.foregroundApp, + readAt: DateTime.formatIso(yield* DateTime.now), + }; + }, + ); + + const action: DeviceService["Service"]["action"] = Effect.fn("DeviceService.action")( + function* (input) { + const { ready, device } = yield* resolveDevice(input.hostId, input.deviceId); + yield* runDeviceAction(ready, device.platform, input); + return yield* detail({ hostId: ready.hostId, deviceId: device.id }); + }, + ); + const sessionsForThread: DeviceService["Service"]["sessionsForThread"] = (threadId) => SynchronizedRef.get(stateRef).pipe( Effect.map(({ state }) => state.sessions.filter((session) => session.threadId === threadId)), @@ -471,6 +513,8 @@ export const make = Effect.gen(function* () { open, close, shutdown, + detail, + action, screenshot, readiness, readinessIfSupported, diff --git a/apps/server/src/device/LocalDeviceHost.ts b/apps/server/src/device/LocalDeviceHost.ts index 24853ea29dd8..3066de791dc9 100644 --- a/apps/server/src/device/LocalDeviceHost.ts +++ b/apps/server/src/device/LocalDeviceHost.ts @@ -89,6 +89,7 @@ interface HubProcess { interface RunningHost { readonly hub: HubProcess; readonly agentDevice: AgentDeviceEndpoint; + readonly helpers: DeviceHostReady["helpers"]; } const platformReason = Effect.fn("LocalDeviceHost.platformReason")(function* ( @@ -459,7 +460,19 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () { const agentDevice = yield* startAgentDeviceDaemon(tools).pipe( Effect.tapError(() => stopHub(hub)), ); - const next: RunningHost = { hub, agentDevice }; + const candidate = helperPaths(tools); + const [axExists, cliExists] = yield* Effect.all([ + fs.exists(candidate.serveSimAxSettings).pipe(Effect.orElseSucceed(() => false)), + fs.exists(candidate.serveSimCli).pipe(Effect.orElseSucceed(() => false)), + ]); + const next: RunningHost = { + hub, + agentDevice, + helpers: { + serveSimAxSettings: axExists ? candidate.serveSimAxSettings : null, + serveSimCli: cliExists ? candidate.serveSimCli : null, + }, + }; yield* Ref.set(runningRef, next); yield* Ref.set(restartDelayRef, 0); yield* Effect.forkDetach(superviseHub(hub, tools)); @@ -467,9 +480,45 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () { }), ); + const helperPaths = (tools: DeviceToolchainPaths) => { + const serveSimDist = path.join( + tools.hub.installDir, + "node_modules", + "expo-device-hub", + "vendor", + "serve-sim", + "dist", + ); + return { + serveSimAxSettings: path.join(serveSimDist, "simax", "serve-sim-ax-settings"), + serveSimCli: path.join(serveSimDist, "serve-sim.js"), + }; + }; + + const run: DeviceHostReady["run"] = (command, args, options) => + runner + .run({ + command, + args, + env: hostEnvironment, + timeout: Duration.millis(options?.timeoutMs ?? 20_000), + timeoutBehavior: "timedOutResult", + ...(options?.stdin === undefined ? {} : { stdin: options.stdin }), + }) + .pipe( + Effect.map((result) => ({ + stdout: result.stdout, + stderr: result.stderr, + code: Number(result.code), + })), + Effect.catch((cause) => Effect.succeed({ stdout: "", stderr: String(cause), code: 127 })), + ); + const toReady = (running: RunningHost): DeviceHostReady => ({ hub: { origin: running.hub.origin } satisfies DeviceHubEndpoint, agentDevice: running.agentDevice, + run, + helpers: running.helpers, }); const current: DeviceHost["current"] = Ref.get(runningRef).pipe( diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index f415f4d9b06c..b39dda102ddd 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -2703,6 +2703,14 @@ const makeWsRpcLayer = ( observeRpcEffect(WS_METHODS.deviceShutdown, deviceService.shutdown(input), { "rpc.aggregate": "device", }), + [WS_METHODS.deviceDetail]: (input) => + observeRpcEffect(WS_METHODS.deviceDetail, deviceService.detail(input), { + "rpc.aggregate": "device", + }), + [WS_METHODS.deviceAction]: (input) => + observeRpcEffect(WS_METHODS.deviceAction, deviceService.action(input), { + "rpc.aggregate": "device", + }), [WS_METHODS.subscribeDeviceState]: (_input) => observeRpcStream( WS_METHODS.subscribeDeviceState, diff --git a/apps/web/src/components/device/DevicePanel.tsx b/apps/web/src/components/device/DevicePanel.tsx index c3e0358b0e867a11c562e60a537a04aa453a5ea8..d07c1973637893b369c53230b54f6ad44075afad 100644 GIT binary patch delta 1149 zcmaJ>&rcIU6b?vKRs$kRMJT-P#ck;F!$eG!ZNL&!qKS=QG!hQOcG_;7&eYlI4?t4y zCdOor#FNIG2h+2}vo~-42VOKDyqI`#W=adR#=XpYZ{~gT-uJ!P*AIW8SM|CDkSgE` zXY?-4iG+3o?h~x7G%K1l!>X9;z2iuGaL`<-VxF{7EHHoWF z%ZMB6QNr4V9A`zaN!eG4F2~3h_J^-4J$?B+!P4y+Jt>$mhkN(>)8)-2-^XDn-}LTJ zE#pey<2A-em_Za1Xn{@7bZ#{socP{X2k;pU1*{95bFYMH0zxbdg7!ARgK~&hbw5rH zkr2mks-6y#Q0-m?bAq-^(o)O&ZxS-qHer{?O}*_N984>LqPU2uSjje$r^QJBn(0;D zDL*<9%QltS+TC+5*R^r<3xjR!*|o4UUgC-&ob`6vZSHntGMZ_gNfi*yV{gYf$cclO zJVuFgOHqXpRR{VFYCOJc$%W{6VxdcFm`NVHXJMwt&z9WHk-}M3$ZErIWjEoTp1kDM-SPNvGc?7<=Ck~L$?_v(}zsM zeCsAPI{;ZDpn~>w-OU;6q*m2nG}ou}r2b>SH91{1>N{E=qwNXPBHh+1sab6YyKxy! z5f*AOyH#usvr_KIh?*m`88T897y_yw-7Q5k5Ap?})@^rg7OHNpOb8TFO-XS5Dr_>Y z9<@s#ETAl_x+D*a>(}OIOSO#~p?WNVf9RWp!`%W8=o5u|Byz@JQ1v&kE%_-Dm48M@ J8jGo3Y{Zpi=u delta 159 zcmeyII5&Jky^Df#Mrv76ex6TiT8WN=b7oO;PO6TAM}97l49G7}Ez(g4$}dSQNp(&x z*HH-0O)M%Y$jHwF$_5vfCKdtd2!)L=;}|DbF!fHBWZt})ndJe?=0?s!vB|&Wcs9!` zYBEkfrX)ByQ^|hvcO}`$63SYWU-9xzzN;cTxkOoa@=|5>$rqGWCd;YFGwa)NO};B9 Kv3Y} void; readonly rotate: () => void; @@ -30,6 +33,8 @@ export function DeviceStreamView(props: { readonly platform: DevicePlatform; readonly deviceId: string; readonly visible: boolean; + /** Draw accessibility element frames over the screen. */ + readonly axOverlay?: boolean; readonly onHandle?: (handle: DeviceStreamHandle | null) => void; readonly onScreen?: (screen: DeviceScreenSize | null) => void; }) { @@ -185,6 +190,34 @@ export function DeviceStreamView(props: { ...(rotation ? { transform: `rotate(${rotation}deg)` } : {}), }; + // The accessibility tree is polled while the overlay is on; each poll is + // one JSON fetch, so there is nothing to repaint between polls. + const [axElements, setAxElements] = useState>([]); + useEffect(() => { + if (!props.axOverlay || !access || !props.visible) return; + const target = { access, platform: props.platform, deviceId: props.deviceId }; + let controller: AbortController | null = null; + let timer: ReturnType | null = null; + let stopped = false; + const poll = async () => { + controller = new AbortController(); + try { + const tree = await fetchDeviceAxTree(target, controller.signal); + if (!stopped) setAxElements(tree.elements); + } catch { + // Keep the last good tree; the next poll retries. + } + if (!stopped) timer = setTimeout(() => void poll(), AX_POLL_INTERVAL_MS); + }; + void poll(); + return () => { + stopped = true; + controller?.abort(); + if (timer) clearTimeout(timer); + setAxElements([]); + }; + }, [access, props.axOverlay, props.deviceId, props.platform, props.visible]); + const pointerActive = useRef(false); const normalizedPoint = (event: React.PointerEvent) => { const rect = event.currentTarget.getBoundingClientRect(); @@ -255,6 +288,28 @@ export function DeviceStreamView(props: { style={mediaStyle} /> ) : null} + {axElements.length > 0 ? ( +
+ {axElements.map((element) => ( +
+ {element.label ? ( + + {element.label} + + ) : null} +
+ ))} +
+ ) : null}
{status === "streaming" && !inputState.connected ? (
diff --git a/apps/web/src/components/device/DeviceToolsPanel.tsx b/apps/web/src/components/device/DeviceToolsPanel.tsx new file mode 100644 index 000000000000..b3c7e68f467b --- /dev/null +++ b/apps/web/src/components/device/DeviceToolsPanel.tsx @@ -0,0 +1,722 @@ +import type { DeviceHubAccess } from "@t3tools/client-runtime/state/deviceHubAccess"; +import type { + DeviceActionInput, + DeviceDetail, + DevicePermission, + DeviceSummary, + DeviceTextSize, + EnvironmentId, +} from "@t3tools/contracts"; +import { ChevronDown, X } from "lucide-react"; +import { useCallback, useEffect, useMemo, useState } from "react"; + +import { Button } from "~/components/ui/button"; +import { Collapsible, CollapsiblePanel, CollapsibleTrigger } from "~/components/ui/collapsible"; +import { Input } from "~/components/ui/input"; +import { + Select, + SelectItem, + SelectPopup, + SelectTrigger, + SelectValue, +} from "~/components/ui/select"; +import { Spinner } from "~/components/ui/spinner"; +import { Switch } from "~/components/ui/switch"; +import { Toggle, ToggleGroup } from "~/components/ui/toggle-group"; +import { cn } from "~/lib/utils"; +import { deviceEnvironment } from "~/state/device"; +import { formatEnvironmentQueryError } from "~/state/query"; +import { useAtomCommand } from "~/state/use-atom-command"; +import { + type DeviceEventLogEntry, + type DeviceForegroundInfo, + subscribeDeviceEventLog, + subscribeDeviceForeground, +} from "./deviceHubApi"; + +type ActionBody = DeviceActionInput extends infer A + ? A extends { readonly type: string } + ? Omit + : never + : never; + +const TEXT_SIZES: ReadonlyArray<{ value: DeviceTextSize; label: string }> = [ + { value: "small", label: "Small" }, + { value: "default", label: "Default" }, + { value: "large", label: "Large" }, + { value: "extra-large", label: "Extra large" }, +]; + +const COLOR_FILTERS = [ + { value: "none", label: "None" }, + { value: "grayscale", label: "Grayscale" }, + { value: "red-green", label: "Red / green (protanopia)" }, + { value: "green-red", label: "Green / red (deuteranopia)" }, + { value: "blue-yellow", label: "Blue / yellow (tritanopia)" }, +] as const; + +const ORIENTATIONS = [ + { value: "portrait", label: "Portrait" }, + { value: "landscape_left", label: "Landscape left" }, + { value: "portrait_upside_down", label: "Upside down" }, + { value: "landscape_right", label: "Landscape right" }, +] as const; + +const IOS_PERMISSIONS: ReadonlyArray<{ value: DevicePermission; label: string }> = [ + { value: "camera", label: "Camera" }, + { value: "microphone", label: "Microphone" }, + { value: "photos", label: "Photos" }, + { value: "contacts", label: "Contacts" }, + { value: "calendar", label: "Calendar" }, + { value: "reminders", label: "Reminders" }, + { value: "location", label: "Location" }, + { value: "notifications", label: "Notifications" }, + { value: "motion", label: "Motion" }, + { value: "media-library", label: "Media library" }, + { value: "faceid", label: "Face ID" }, +]; + +const ANDROID_PERMISSIONS: ReadonlyArray<{ value: DevicePermission; label: string }> = [ + { value: "camera", label: "Camera" }, + { value: "microphone", label: "Microphone" }, + { value: "photos", label: "Photos" }, + { value: "contacts", label: "Contacts" }, + { value: "calendar", label: "Calendar" }, + { value: "location", label: "Location" }, + { value: "notifications", label: "Notifications" }, + { value: "motion", label: "Physical activity" }, +]; + +const LOCATION_PRESETS = [ + { label: "San Francisco", latitude: 37.7749, longitude: -122.4194 }, + { label: "New York", latitude: 40.7128, longitude: -74.006 }, + { label: "London", latitude: 51.5074, longitude: -0.1278 }, + { label: "Stockholm", latitude: 59.3293, longitude: 18.0686 }, + { label: "Tokyo", latitude: 35.6762, longitude: 139.6503 }, +] as const; + +/** + * The Tools drawer for one open device: current settings read from the device, + * one control per supported action, and the read-only feeds the hub exposes. + * Every change is a `device.action` round trip; the returned detail replaces + * local state so the controls never show a value the device did not confirm. + */ +export function DeviceToolsPanel(props: { + readonly environmentId: EnvironmentId; + readonly device: DeviceSummary; + readonly access: DeviceHubAccess | null; + readonly axOverlay: boolean; + readonly onAxOverlayChange: (enabled: boolean) => void; + readonly onClose: () => void; + readonly className?: string; +}) { + const { environmentId, device } = props; + const readDetail = useAtomCommand(deviceEnvironment.detail, { reportFailure: false }); + const runAction = useAtomCommand(deviceEnvironment.action, { reportFailure: false }); + const [detail, setDetail] = useState(null); + const [pending, setPending] = useState(false); + const [error, setError] = useState(null); + const [foreground, setForeground] = useState(null); + const isIos = device.platform === "ios"; + + const target = useMemo( + () => ({ hostId: device.hostId, deviceId: device.id }), + [device.hostId, device.id], + ); + + // The panel is keyed by device, so a mount is always a fresh device. + useEffect(() => { + let cancelled = false; + void readDetail({ environmentId, input: target }).then((result) => { + if (cancelled) return; + if (result._tag === "Success") setDetail(result.value); + else setError(formatEnvironmentQueryError(result.cause)); + }); + return () => { + cancelled = true; + }; + }, [environmentId, readDetail, target]); + + useEffect(() => { + if (!props.access) return; + return subscribeDeviceForeground( + { access: props.access, platform: device.platform, deviceId: device.id }, + setForeground, + ); + }, [device.id, device.platform, props.access]); + + const act = useCallback( + async (body: ActionBody) => { + setPending(true); + setError(null); + try { + const result = await runAction({ + environmentId, + input: { ...target, ...body } as DeviceActionInput, + }); + if (result._tag === "Success") setDetail(result.value); + else setError(formatEnvironmentQueryError(result.cause)); + } finally { + setPending(false); + } + }, + [environmentId, runAction, target], + ); + + const settings = detail?.settings; + const foregroundApp = foreground ?? detail?.foregroundApp ?? null; + const disabled = pending || detail === null; + + return ( +
+
+ Tools + {pending ? : null} + +
+
+ {error ? ( +

{error}

+ ) : null} + {detail === null && !error ? ( +
+ Reading device settings… +
+ ) : null} + +
+ + {foregroundApp?.id ?? "—"} + + {foregroundApp ? ( +
+ + +
+ ) : null} + act({ type: "openUrl", url })} + /> + act({ type: "launchApp", appId })} + /> +
+ +
+ + { + const next = value[0]; + if (next === "light" || next === "dark") + void act({ type: "setAppearance", value: next }); + }} + > + Light + Dark + + + + act({ type: "setTextSize", value })} + /> + + {isIos ? ( + <> + + { + const next = value[0]; + if (next === "clear" || next === "tinted") { + void act({ type: "setLiquidGlass", value: next }); + } + }} + > + Clear + Tinted + + + + act({ type: "setColorFilter", value })} + /> + + + ) : ( + + act({ type: "setOrientation", value })} + /> + + )} + act({ type: "setToggle", setting: "reduceMotion", value })} + /> + {isIos ? ( + <> + act({ type: "setToggle", setting: "increaseContrast", value })} + /> + + act({ type: "setToggle", setting: "reduceTransparency", value }) + } + /> + act({ type: "setToggle", setting: "showBorders", value })} + /> + act({ type: "setToggle", setting: "voiceOver", value })} + /> + + ) : ( + act({ type: "setToggle", setting: "networkEnabled", value })} + /> + )} +
+ +
+ { + props.onAxOverlayChange(value); + return Promise.resolve(); + }} + /> +
+ + act({ type: "setLocation", latitude, longitude })} + onClear={() => act({ type: "clearLocation" })} + /> + + + act({ type: "setPermission", appId, permission, decision }) + } + /> + + {isIos ? ( +
+ + foregroundApp + ? act({ type: "sendPush", appId: foregroundApp.id, payload }) + : Promise.resolve() + } + /> + {!foregroundApp ? ( +

Open an app first.

+ ) : null} +
+ ) : null} + + {isIos && props.access ? : null} +
+
+ ); +} + +function Section(props: { readonly title: string; readonly children: React.ReactNode }) { + return ( +
+

+ {props.title} +

+ {props.children} +
+ ); +} + +function Row(props: { readonly label: string; readonly children: React.ReactNode }) { + return ( +
+ {props.label} +
{props.children}
+
+ ); +} + +function SwitchRow(props: { + readonly label: string; + readonly checked: boolean | undefined; + readonly disabled: boolean; + readonly onChange: (value: boolean) => Promise; +}) { + return ( + + void props.onChange(checked)} + /> + + ); +} + +function ChoiceSelect(props: { + readonly ariaLabel: string; + readonly value: V | null; + readonly options: ReadonlyArray<{ readonly value: V; readonly label: string }>; + readonly disabled: boolean; + readonly placeholder?: string; + readonly onChange: (value: V) => Promise; +}) { + const current = props.options.find((option) => option.value === props.value); + return ( + + ); +} + +function SubmitRow(props: { + readonly placeholder: string; + readonly action: string; + readonly disabled: boolean; + readonly onSubmit: (value: string) => Promise; +}) { + const [value, setValue] = useState(""); + const submit = () => { + const trimmed = value.trim(); + if (!trimmed) return; + void props.onSubmit(trimmed).then(() => setValue("")); + }; + return ( +
{ + event.preventDefault(); + submit(); + }} + > + setValue(event.target.value)} + /> + +
+ ); +} + +function LocationSection(props: { + readonly disabled: boolean; + readonly canClear: boolean; + readonly onSet: (latitude: number, longitude: number) => Promise; + readonly onClear: () => Promise; +}) { + const [latitude, setLatitude] = useState(""); + const [longitude, setLongitude] = useState(""); + const parsed = { latitude: Number(latitude), longitude: Number(longitude) }; + const valid = + latitude.trim() !== "" && + longitude.trim() !== "" && + Math.abs(parsed.latitude) <= 90 && + Math.abs(parsed.longitude) <= 180; + return ( +
+
+ setLatitude(event.target.value)} + /> + setLongitude(event.target.value)} + /> +
+
+ + value={null} + disabled={props.disabled} + onValueChange={(value) => { + const preset = LOCATION_PRESETS.find((candidate) => candidate.label === value); + if (!preset) return; + setLatitude(String(preset.latitude)); + setLongitude(String(preset.longitude)); + void props.onSet(preset.latitude, preset.longitude); + }} + > + + + Preset… + + + + {LOCATION_PRESETS.map((preset) => ( + + {preset.label} + + ))} + + + + {props.canClear ? ( + + ) : null} +
+
+ ); +} + +function PermissionsSection(props: { + readonly permissions: ReadonlyArray<{ value: DevicePermission; label: string }>; + readonly canReset: boolean; + readonly defaultAppId: string; + readonly disabled: boolean; + readonly onDecide: ( + appId: string, + permission: DevicePermission, + decision: "grant" | "revoke" | "reset", + ) => Promise; +}) { + const [appId, setAppId] = useState(""); + const [permission, setPermission] = useState("camera"); + const resolvedAppId = appId.trim() || props.defaultAppId; + const decide = (decision: "grant" | "revoke" | "reset") => + void props.onDecide(resolvedAppId, permission, decision); + return ( +
+ setAppId(event.target.value)} + /> +
+ { + setPermission(value); + return Promise.resolve(); + }} + /> + + + {props.canReset ? ( + + ) : null} +
+
+ ); +} + +const EVENT_LOG_LIMIT = 100; + +function EventLogSection(props: { + readonly access: DeviceHubAccess; + readonly device: DeviceSummary; +}) { + const [open, setOpen] = useState(false); + const [entries, setEntries] = useState>([]); + + useEffect(() => { + if (!open) return; + const unsubscribe = subscribeDeviceEventLog( + { access: props.access, platform: props.device.platform, deviceId: props.device.id }, + (incoming, reset) => { + setEntries((current) => { + const merged = reset ? [...incoming] : [...current, ...incoming]; + return merged.length > EVENT_LOG_LIMIT ? merged.slice(-EVENT_LOG_LIMIT) : merged; + }); + }, + ); + return () => { + unsubscribe(); + setEntries([]); + }; + }, [open, props.access, props.device.id, props.device.platform]); + + return ( + + + Event log + + + +
    + {entries.length === 0 ? ( +
  1. No events yet.
    2. + ) : ( + entries.map((entry) => ( +
  2. + + {entry.timestamp.slice(11, 19)} + + {entry.summary} +
    3. + )) + )} +
+ + + ); +} diff --git a/apps/web/src/components/device/deviceHubApi.ts b/apps/web/src/components/device/deviceHubApi.ts new file mode 100644 index 000000000000..9479189c329f --- /dev/null +++ b/apps/web/src/components/device/deviceHubApi.ts @@ -0,0 +1,214 @@ +import { withDeviceHubQuery } from "@t3tools/client-runtime/state/deviceHubAccess"; +import type { DeviceHubAccess } from "@t3tools/client-runtime/state/deviceHubAccess"; +import type { DevicePlatform } from "@t3tools/contracts"; + +/** + * Read-only hub endpoints the Tools drawer consumes directly: the accessibility + * tree, the foreground app, and the event log. Everything that changes device + * state goes through the `device.action` RPC instead, so this file never POSTs. + */ + +export interface DeviceAxElement { + readonly id: string; + readonly label: string; + readonly role: string; + /** Normalized to the displayed screen: 0..1 on both axes. */ + readonly x: number; + readonly y: number; + readonly width: number; + readonly height: number; +} + +export interface DeviceAxTree { + readonly elements: ReadonlyArray; + readonly errors: ReadonlyArray; +} + +export interface DeviceEventLogEntry { + readonly id: number; + readonly timestamp: string; + readonly kind: string; + readonly summary: string; +} + +export interface DeviceForegroundInfo { + readonly id: string; + readonly label?: string; + readonly pid?: number; + readonly isReactNative?: boolean; +} + +interface Target { + readonly access: DeviceHubAccess; + readonly platform: DevicePlatform; + readonly deviceId: string; +} + +const vendorBase = (target: Target) => + `${target.access.httpBase}${target.platform === "ios" ? "/vendor/serve-sim" : "/vendor/serve-emu"}`; + +const hubUrl = (target: Target, path: string, params?: Record) => { + const search = params ? `?${new URLSearchParams(params).toString()}` : ""; + return withDeviceHubQuery(`${vendorBase(target)}${path}${search}`, target.access); +}; + +const fetchJson = async (target: Target, url: string, signal?: AbortSignal): Promise => { + const response = await fetch(url, { + cache: "no-store", + credentials: target.access.credentials ? "include" : "same-origin", + ...(signal ? { signal } : {}), + }); + if (!response.ok) throw new Error(`${response.status} ${response.statusText}`); + return response.json(); +}; + +const isRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null; + +const numberOr = (value: unknown, fallback: number) => + typeof value === "number" && Number.isFinite(value) ? value : fallback; + +export async function fetchDeviceAxTree( + target: Target, + signal?: AbortSignal, +): Promise { + if (target.platform === "ios") { + const payload = await fetchJson( + target, + hubUrl(target, `/helper/${encodeURIComponent(target.deviceId)}/ax`), + signal, + ); + if (!isRecord(payload) || !Array.isArray(payload.elements)) { + return { elements: [], errors: ["Unexpected accessibility payload."] }; + } + const screen = isRecord(payload.screen) ? payload.screen : {}; + const screenWidth = Math.max(1, numberOr(screen.width, 1)); + const screenHeight = Math.max(1, numberOr(screen.height, 1)); + const elements = payload.elements.flatMap((raw): DeviceAxElement[] => { + if (!isRecord(raw) || !isRecord(raw.frame)) return []; + const frame = raw.frame; + return [ + { + id: String(raw.id ?? raw.path ?? ""), + label: typeof raw.label === "string" ? raw.label : "", + role: typeof raw.role === "string" ? raw.role : "", + x: numberOr(frame.x, 0) / screenWidth, + y: numberOr(frame.y, 0) / screenHeight, + width: numberOr(frame.width, 0) / screenWidth, + height: numberOr(frame.height, 0) / screenHeight, + }, + ]; + }); + const errors = Array.isArray(payload.errors) ? payload.errors.map(String) : []; + return { elements, errors }; + } + const payload = await fetchJson( + target, + hubUrl(target, "/api/accessibility", { device: target.deviceId }), + signal, + ); + if (!isRecord(payload) || !Array.isArray(payload.nodes)) { + const error = isRecord(payload) && typeof payload.error === "string" ? payload.error : null; + return { elements: [], errors: [error ?? "Unexpected accessibility payload."] }; + } + // uiautomator reports pixel bounds; the first node is the full window. + const nodes = payload.nodes.filter( + (node): node is Record => isRecord(node) && isRecord(node.bounds), + ); + const root = nodes[0]?.bounds as Record | undefined; + const screenWidth = Math.max(1, numberOr(root?.right, 1)); + const screenHeight = Math.max(1, numberOr(root?.bottom, 1)); + const elements = nodes.slice(1).map((node): DeviceAxElement => { + const bounds = node.bounds as Record; + const left = numberOr(bounds.left, 0); + const top = numberOr(bounds.top, 0); + const text = typeof node.text === "string" ? node.text : ""; + const description = typeof node.contentDescription === "string" ? node.contentDescription : ""; + const className = typeof node.className === "string" ? node.className : ""; + return { + id: String(node.id ?? ""), + label: text || description, + role: className.split(".").at(-1) ?? "", + x: left / screenWidth, + y: top / screenHeight, + width: (numberOr(bounds.right, left) - left) / screenWidth, + height: (numberOr(bounds.bottom, top) - top) / screenHeight, + }; + }); + return { elements, errors: [] }; +} + +const openEventSource = ( + target: Target, + url: string, + onMessage: (data: unknown) => void, +): (() => void) => { + const source = new EventSource(url, { withCredentials: target.access.credentials }); + source.addEventListener("message", (event) => { + try { + onMessage(JSON.parse(String(event.data))); + } catch { + // Keep-alive comments and malformed frames carry nothing to render. + } + }); + return () => source.close(); +}; + +/** iOS only: the frontmost app, pushed by serve-sim whenever it changes. */ +export function subscribeDeviceForeground( + target: Target, + onChange: (app: DeviceForegroundInfo | null) => void, +): () => void { + if (target.platform !== "ios") return () => {}; + return openEventSource( + target, + hubUrl(target, "/appstate", { device: target.deviceId }), + (data) => { + if (!isRecord(data) || typeof data.bundleId !== "string") return; + onChange({ + id: data.bundleId, + ...(typeof data.pid === "number" ? { pid: data.pid } : {}), + ...(typeof data.isReactNative === "boolean" ? { isReactNative: data.isReactNative } : {}), + }); + }, + ); +} + +const toEventLogEntry = (raw: unknown): DeviceEventLogEntry | null => { + if (!isRecord(raw) || typeof raw.id !== "number") return null; + return { + id: raw.id, + timestamp: typeof raw.timestamp === "string" ? raw.timestamp : "", + kind: typeof raw.kind === "string" ? raw.kind : "", + summary: + typeof raw.summary === "string" ? raw.summary : typeof raw.msg === "string" ? raw.msg : "", + }; +}; + +/** + * iOS only: serve-sim's event log, seeded with recent history and then pushed + * live. Android's session recorder only tracks replayable gestures, which the + * user already sees themselves, so it is not surfaced. + */ +export function subscribeDeviceEventLog( + target: Target, + onEvents: (entries: ReadonlyArray, reset: boolean) => void, +): () => void { + if (target.platform !== "ios") return () => {}; + return openEventSource( + target, + hubUrl(target, "/api/event-log/events", { device: target.deviceId, limit: "100" }), + (data) => { + if (!isRecord(data)) return; + if (Array.isArray(data.events)) { + onEvents( + data.events.flatMap((raw) => toEventLogEntry(raw) ?? []), + true, + ); + return; + } + const entry = toEventLogEntry(data.event); + if (entry) onEvents([entry], false); + }, + ); +} diff --git a/docs/internals/devices.md b/docs/internals/devices.md index 7eb01ac2e589..57122f580352 100644 --- a/docs/internals/devices.md +++ b/docs/internals/devices.md @@ -38,6 +38,18 @@ Stream responses carry `Cache-Control: no-transform`; the compression middleware would otherwise buffer an MJPEG body that never ends. In browser dev, the Vite proxy must forward WebSocket upgrades for `/api`, not only `/ws`. +## Device settings never go through the hub + +serve-sim's preview drives its Tools panel by sending shell commands over that +same exec channel. Proxying it, even allowlisted, would hand any environment +session arbitrary command execution on the host, so T3 does not. The +[`device.action`](../../apps/server/src/device/DeviceActions.ts) RPC runs the +underlying `simctl`, `adb`, and serve-sim helper binaries itself through +`DeviceHostReady.run`, one typed action per control, and returns the settings +it reads back. The proxy allowlist grows only with read routes (accessibility +tree, foreground app, event log) and refuses non-GET methods everywhere except +screenshot capture and stream tuning. + ## Agents drive through the CLI The `device_*` toolkit is deliberately four tools: list, open, screenshot, and diff --git a/docs/user/devices.md b/docs/user/devices.md index 3ff87d992107..0ccdc7e4d5a3 100644 --- a/docs/user/devices.md +++ b/docs/user/devices.md @@ -22,6 +22,17 @@ focused, and use the toolbar for Home, Back, and Recents on Android, rotate on iOS, and power off. Close the tab to stop watching; the device keeps running unless you power it off. +## Tools + +The toolbar's **Tools** button opens a drawer for the open device. It shows the +foreground app, and lets you switch light and dark mode, change text size, +flip accessibility settings, overlay the accessibility element frames on the +screen, set a fake location, and grant or revoke app permissions. iOS also +exposes Liquid Glass, color filters, VoiceOver, and sending a test push +notification; Android adds orientation and toggling the network. The drawer +only shows what the platform can do, and every control reflects the value read +back from the device after a change. + ## Agents and devices When an agent opens a device, the panel opens in every client connected to the diff --git a/packages/client-runtime/src/state/device.ts b/packages/client-runtime/src/state/device.ts index 1b983ab671fe..bec29b6dfc29 100644 --- a/packages/client-runtime/src/state/device.ts +++ b/packages/client-runtime/src/state/device.ts @@ -46,5 +46,17 @@ export function createDeviceEnvironmentAtoms( scheduler, concurrency, }), + detail: createEnvironmentRpcCommand(runtime, { + label: "environment-data:device:detail", + tag: WS_METHODS.deviceDetail, + scheduler, + concurrency, + }), + action: createEnvironmentRpcCommand(runtime, { + label: "environment-data:device:action", + tag: WS_METHODS.deviceAction, + scheduler, + concurrency, + }), }; } diff --git a/packages/contracts/src/device.ts b/packages/contracts/src/device.ts index 39ce80df2968..dc482710bf5b 100644 --- a/packages/contracts/src/device.ts +++ b/packages/contracts/src/device.ts @@ -129,6 +129,168 @@ export const DeviceShutdownInput = Schema.Struct({ }); export type DeviceShutdownInput = typeof DeviceShutdownInput.Type; +// Device settings and actions. Each setting names the platforms that support +// it; the panel hides the rest. Values are normalized across platforms where +// both have the concept (appearance, text size) and platform-specific where +// only one does. + +export const DeviceAppearance = Schema.Literals(["light", "dark"]); +export type DeviceAppearance = typeof DeviceAppearance.Type; + +/** + * iOS content-size categories map onto twelve steps; Android `font_scale` + * is continuous. Four shared steps cover what people actually reach for. + */ +export const DeviceTextSize = Schema.Literals(["small", "default", "large", "extra-large"]); +export type DeviceTextSize = typeof DeviceTextSize.Type; + +export const DeviceColorFilter = Schema.Literals([ + "none", + "grayscale", + "red-green", + "green-red", + "blue-yellow", +]); +export type DeviceColorFilter = typeof DeviceColorFilter.Type; + +export const DeviceOrientation = Schema.Literals([ + "portrait", + "landscape_left", + "portrait_upside_down", + "landscape_right", +]); +export type DeviceOrientation = typeof DeviceOrientation.Type; + +/** Current values as read from the device; `undefined` means unsupported or unread. */ +export const DeviceSettings = Schema.Struct({ + appearance: Schema.optional(DeviceAppearance), + textSize: Schema.optional(DeviceTextSize), + reduceMotion: Schema.optional(Schema.Boolean), + increaseContrast: Schema.optional(Schema.Boolean), + reduceTransparency: Schema.optional(Schema.Boolean), + showBorders: Schema.optional(Schema.Boolean), + voiceOver: Schema.optional(Schema.Boolean), + liquidGlass: Schema.optional(Schema.Literals(["clear", "tinted"])), + colorFilter: Schema.optional(DeviceColorFilter), + networkEnabled: Schema.optional(Schema.Boolean), + location: Schema.optional( + Schema.NullOr(Schema.Struct({ latitude: Schema.Number, longitude: Schema.Number })), + ), +}); +export type DeviceSettings = typeof DeviceSettings.Type; + +/** The app in the foreground, when the platform can tell us. */ +export const DeviceForegroundApp = Schema.Struct({ + id: Schema.String, + name: Schema.optional(Schema.String), + version: Schema.optional(Schema.String), +}); +export type DeviceForegroundApp = typeof DeviceForegroundApp.Type; + +export const DeviceDetail = Schema.Struct({ + hostId: DeviceHostId, + deviceId: DeviceId, + settings: DeviceSettings, + foregroundApp: Schema.NullOr(DeviceForegroundApp), + readAt: Schema.String, +}); +export type DeviceDetail = typeof DeviceDetail.Type; + +export const DevicePermission = Schema.Literals([ + "camera", + "microphone", + "photos", + "contacts", + "calendar", + "reminders", + "location", + "notifications", + "motion", + "media-library", + "faceid", +]); +export type DevicePermission = typeof DevicePermission.Type; + +const DeviceTarget = { + hostId: Schema.optional(DeviceHostId), + deviceId: DeviceId, +}; + +export const DeviceActionInput = Schema.Union([ + Schema.Struct({ + ...DeviceTarget, + type: Schema.Literal("setAppearance"), + value: DeviceAppearance, + }), + Schema.Struct({ ...DeviceTarget, type: Schema.Literal("setTextSize"), value: DeviceTextSize }), + Schema.Struct({ + ...DeviceTarget, + type: Schema.Literal("setToggle"), + setting: Schema.Literals([ + "reduceMotion", + "increaseContrast", + "reduceTransparency", + "showBorders", + "voiceOver", + "networkEnabled", + ]), + value: Schema.Boolean, + }), + Schema.Struct({ + ...DeviceTarget, + type: Schema.Literal("setLiquidGlass"), + value: Schema.Literals(["clear", "tinted"]), + }), + Schema.Struct({ + ...DeviceTarget, + type: Schema.Literal("setColorFilter"), + value: DeviceColorFilter, + }), + Schema.Struct({ + ...DeviceTarget, + type: Schema.Literal("setOrientation"), + value: DeviceOrientation, + }), + Schema.Struct({ + ...DeviceTarget, + type: Schema.Literal("setLocation"), + latitude: Schema.Number.check(Schema.isBetween({ minimum: -90, maximum: 90 })), + longitude: Schema.Number.check(Schema.isBetween({ minimum: -180, maximum: 180 })), + }), + Schema.Struct({ ...DeviceTarget, type: Schema.Literal("clearLocation") }), + Schema.Struct({ + ...DeviceTarget, + type: Schema.Literal("setPermission"), + appId: TrimmedNonEmptyString, + permission: DevicePermission, + decision: Schema.Literals(["grant", "revoke", "reset"]), + }), + Schema.Struct({ ...DeviceTarget, type: Schema.Literal("openUrl"), url: TrimmedNonEmptyString }), + Schema.Struct({ + ...DeviceTarget, + type: Schema.Literal("launchApp"), + appId: TrimmedNonEmptyString, + }), + Schema.Struct({ + ...DeviceTarget, + type: Schema.Literal("terminateApp"), + appId: TrimmedNonEmptyString, + }), + Schema.Struct({ ...DeviceTarget, type: Schema.Literal("shake") }), + Schema.Struct({ + ...DeviceTarget, + type: Schema.Literal("sendPush"), + appId: TrimmedNonEmptyString, + /** APNs-style payload; a bare string becomes the alert body. */ + payload: Schema.Union([Schema.String, Schema.Record(Schema.String, Schema.Unknown)]), + }), +]); +export type DeviceActionInput = typeof DeviceActionInput.Type; +export type DeviceActionType = DeviceActionInput["type"]; + +export const DeviceDetailInput = Schema.Struct(DeviceTarget); +export type DeviceDetailInput = typeof DeviceDetailInput.Type; + export class DeviceHostUnavailableError extends Schema.TaggedError()( "DeviceHostUnavailableError", { diff --git a/packages/contracts/src/rpc.ts b/packages/contracts/src/rpc.ts index 7d65551a8454..970bceb0e376 100644 --- a/packages/contracts/src/rpc.ts +++ b/packages/contracts/src/rpc.ts @@ -180,7 +180,10 @@ import { PreviewSessionSnapshot, } from "./preview.ts"; import { + DeviceActionInput, DeviceCloseInput, + DeviceDetail, + DeviceDetailInput, DeviceError, DeviceListInput, DeviceOpenInput, @@ -324,6 +327,8 @@ export const WS_METHODS = { deviceOpen: "device.open", deviceClose: "device.close", deviceShutdown: "device.shutdown", + deviceDetail: "device.detail", + deviceAction: "device.action", // Server meta serverProbe: "server.probe", @@ -1100,6 +1105,18 @@ const WsDeviceShutdownRpc = Rpc.make(WS_METHODS.deviceShutdown, { error: Schema.Union([DeviceError, EnvironmentAuthorizationError]), }); +const WsDeviceDetailRpc = Rpc.make(WS_METHODS.deviceDetail, { + payload: DeviceDetailInput, + success: DeviceDetail, + error: Schema.Union([DeviceError, EnvironmentAuthorizationError]), +}); + +const WsDeviceActionRpc = Rpc.make(WS_METHODS.deviceAction, { + payload: DeviceActionInput, + success: DeviceDetail, + error: Schema.Union([DeviceError, EnvironmentAuthorizationError]), +}); + const WsSubscribeDeviceStateRpc = Rpc.make(WS_METHODS.subscribeDeviceState, { payload: Schema.Struct({}), success: DeviceServiceState, @@ -1339,6 +1356,8 @@ export const WsRpcGroup = RpcGroup.make( WsDeviceOpenRpc, WsDeviceCloseRpc, WsDeviceShutdownRpc, + WsDeviceDetailRpc, + WsDeviceActionRpc, WsSubscribeDeviceStateRpc, WsSubscribeServerConfigRpc, WsSubscribeServerLifecycleRpc, From 9a4e3915ecf10b63cc3470956a8ad009207770bd Mon Sep 17 00:00:00 2001 From: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com> Date: Mon, 7 Sep 2026 23:38:19 -0700 Subject: [PATCH 11/28] fix(devices): flatten the nested iOS accessibility tree for the overlay Co-Authored-By: Claude Fable 5 --- .../web/src/components/device/deviceHubApi.ts | 64 ++++++++++++------- 1 file changed, 42 insertions(+), 22 deletions(-) diff --git a/apps/web/src/components/device/deviceHubApi.ts b/apps/web/src/components/device/deviceHubApi.ts index 9479189c329f..f069a8207cfe 100644 --- a/apps/web/src/components/device/deviceHubApi.ts +++ b/apps/web/src/components/device/deviceHubApi.ts @@ -68,6 +68,44 @@ const isRecord = (value: unknown): value is Record => const numberOr = (value: unknown, fallback: number) => typeof value === "number" && Number.isFinite(value) ? value : fallback; +const AX_ELEMENT_LIMIT = 500; + +/** + * serve-sim's helper returns the native nested tree; the root node is the + * application covering the whole screen. Flatten it the way serve-sim's own + * overlay does: skip nodes with the root's frame, cap the count. + */ +const flattenIosAxTree = (roots: ReadonlyArray): ReadonlyArray => { + const first = roots[0]; + const rootFrame = isRecord(first) && isRecord(first.frame) ? first.frame : null; + const screenWidth = Math.max(1, numberOr(rootFrame?.width, 1)); + const screenHeight = Math.max(1, numberOr(rootFrame?.height, 1)); + const elements: DeviceAxElement[] = []; + const visit = (node: unknown, path: string) => { + if (elements.length >= AX_ELEMENT_LIMIT || !isRecord(node) || !isRecord(node.frame)) return; + const frame = node.frame; + const width = numberOr(frame.width, 0); + const height = numberOr(frame.height, 0); + const coversScreen = + Math.abs(width - screenWidth) < 0.5 && Math.abs(height - screenHeight) < 0.5; + if (!coversScreen && width > 0 && height > 0) { + elements.push({ + id: typeof node.AXUniqueId === "string" ? node.AXUniqueId : path, + label: typeof node.AXLabel === "string" ? node.AXLabel : "", + role: typeof node.type === "string" ? node.type : "", + x: numberOr(frame.x, 0) / screenWidth, + y: numberOr(frame.y, 0) / screenHeight, + width: width / screenWidth, + height: height / screenHeight, + }); + } + const children = Array.isArray(node.children) ? node.children : []; + children.forEach((child, index) => visit(child, `${path}.${index}`)); + }; + roots.forEach((root, index) => visit(root, String(index))); + return elements; +}; + export async function fetchDeviceAxTree( target: Target, signal?: AbortSignal, @@ -78,29 +116,11 @@ export async function fetchDeviceAxTree( hubUrl(target, `/helper/${encodeURIComponent(target.deviceId)}/ax`), signal, ); - if (!isRecord(payload) || !Array.isArray(payload.elements)) { - return { elements: [], errors: ["Unexpected accessibility payload."] }; + if (!Array.isArray(payload)) { + const error = isRecord(payload) && typeof payload.error === "string" ? payload.error : null; + return { elements: [], errors: [error ?? "Unexpected accessibility payload."] }; } - const screen = isRecord(payload.screen) ? payload.screen : {}; - const screenWidth = Math.max(1, numberOr(screen.width, 1)); - const screenHeight = Math.max(1, numberOr(screen.height, 1)); - const elements = payload.elements.flatMap((raw): DeviceAxElement[] => { - if (!isRecord(raw) || !isRecord(raw.frame)) return []; - const frame = raw.frame; - return [ - { - id: String(raw.id ?? raw.path ?? ""), - label: typeof raw.label === "string" ? raw.label : "", - role: typeof raw.role === "string" ? raw.role : "", - x: numberOr(frame.x, 0) / screenWidth, - y: numberOr(frame.y, 0) / screenHeight, - width: numberOr(frame.width, 0) / screenWidth, - height: numberOr(frame.height, 0) / screenHeight, - }, - ]; - }); - const errors = Array.isArray(payload.errors) ? payload.errors.map(String) : []; - return { elements, errors }; + return { elements: flattenIosAxTree(payload), errors: [] }; } const payload = await fetchJson( target, From f36f4217674407b5dc692619911f04ecf6599113 Mon Sep 17 00:00:00 2001 From: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com> Date: Tue, 8 Sep 2026 00:23:39 -0700 Subject: [PATCH 12/28] fix(devices): read the Android foreground app from the unfiltered window dump API 36 no longer prints mCurrentFocus under `dumpsys window windows`. Co-Authored-By: Claude Fable 5 --- apps/server/src/device/DeviceActions.test.ts | 4 ++-- apps/server/src/device/DeviceActions.ts | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/apps/server/src/device/DeviceActions.test.ts b/apps/server/src/device/DeviceActions.test.ts index d1ecd9fa670e..987a63d649b6 100644 --- a/apps/server/src/device/DeviceActions.test.ts +++ b/apps/server/src/device/DeviceActions.test.ts @@ -225,10 +225,10 @@ describe("readDeviceDetail", () => { if (key.endsWith("font_scale")) return { stdout: "0.85\n" }; if (key.endsWith("animator_duration_scale")) return { stdout: "0\n" }; if (key.endsWith("wifi_on")) return { stdout: "1\n" }; - if (key.endsWith("dumpsys window windows")) { + if (key.endsWith("dumpsys window")) { return { stdout: - " mCurrentFocus=Window{1a2b u0 com.example.app/com.example.app.MainActivity}\n", + " mFocusedApp=ActivityRecord{155579877 u0 com.example.app/.MainActivity t15}\n mCurrentFocus=Window{1a2b u0 com.example.app/com.example.app.MainActivity}\n", }; } return { code: 1 }; diff --git a/apps/server/src/device/DeviceActions.ts b/apps/server/src/device/DeviceActions.ts index 39300c000fe4..69143f317cb6 100644 --- a/apps/server/src/device/DeviceActions.ts +++ b/apps/server/src/device/DeviceActions.ts @@ -476,12 +476,14 @@ const readAndroid = Effect.fn("DeviceActions.readAndroid")(function* (run: Runne shell(["settings", "get", "system", "font_scale"]), shell(["settings", "get", "global", "animator_duration_scale"]), shell(["settings", "get", "global", "wifi_on"]), - shell(["dumpsys", "window", "windows"]), + // `dumpsys window windows` stopped printing the focus on API 36; the + // unfiltered dump still does. + shell(["dumpsys", "window"]), ], { concurrency: 5 }, ); const scale = fontScale && fontScale !== "null" ? Number(fontScale) : Number.NaN; - const focused = focus?.match(/mCurrentFocus=Window\{[^ ]+ u\d+ ([^/ ]+)\/([^ }]+)\}/); + const focused = focus?.match(/m(?:CurrentFocus|FocusedApp)=\w+\{[^ ]+ u\d+ ([^/ ]+)\//); const settings: DeviceSettings = { ...(night?.includes("yes") ? { appearance: "dark" } From 0e51f87b0294bc5882470985ac9425a06dfa4d6e Mon Sep 17 00:00:00 2001 From: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com> Date: Tue, 8 Sep 2026 00:29:05 -0700 Subject: [PATCH 13/28] fix(devices): rebuild the Android decoder when the emulator rotates serve-emu restarts the encoder at the new size and announces it with a video-session message; keeping the old decoder froze the stream sideways. Co-Authored-By: Claude Fable 5 --- apps/web/src/components/device/deviceStream.ts | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/apps/web/src/components/device/deviceStream.ts b/apps/web/src/components/device/deviceStream.ts index 99f247bbfce1..a993bf81f1e7 100644 --- a/apps/web/src/components/device/deviceStream.ts +++ b/apps/web/src/components/device/deviceStream.ts @@ -111,6 +111,15 @@ export function parseSemuPacket(raw: ArrayBuffer): { return { data: bytes, isKey: null, timestamp: null }; } +const isVideoSessionMessage = (text: string) => { + try { + const message = JSON.parse(text) as { type?: unknown }; + return message.type === "video-session"; + } catch { + return false; + } +}; + /** Walk an Annex-B access unit for its keyframe flag and SPS bytes. */ export function scanAccessUnit(buf: Uint8Array): { isKey: boolean; sps: Uint8Array | null } { let isKey = false; @@ -533,6 +542,12 @@ export function createDeviceStreamClient( events.onInputConnected(true); }; ws.onmessage = (event) => { + if (typeof event.data === "string") { + // The encoder restarts at a new size when the device rotates; the + // next keyframe carries a fresh SPS, so the decoder is rebuilt from it. + if (isVideoSessionMessage(event.data)) closeDecoder(); + return; + } if (!(event.data instanceof ArrayBuffer)) return; const packet = parseSemuPacket(event.data); const needsScan = From fe89109163a708889199d6df0f292c297d5227c9 Mon Sep 17 00:00:00 2001 From: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com> Date: Tue, 8 Sep 2026 00:38:51 -0700 Subject: [PATCH 14/28] fix(devices): rotate Android emulators through the accelerometer user-rotation lock only rotates window content on recent system images; the display the encoder captures stays portrait, so the panel showed a sideways app in a portrait frame. Tilting the emulator's gravity vector rotates the display for real. Physical devices keep the lock. Co-Authored-By: Claude Fable 5 --- apps/server/src/device/DeviceActions.test.ts | 34 +++++++++++++++ apps/server/src/device/DeviceActions.ts | 44 ++++++++++++++++---- 2 files changed, 69 insertions(+), 9 deletions(-) diff --git a/apps/server/src/device/DeviceActions.test.ts b/apps/server/src/device/DeviceActions.test.ts index 987a63d649b6..1b94cda5a73e 100644 --- a/apps/server/src/device/DeviceActions.test.ts +++ b/apps/server/src/device/DeviceActions.test.ts @@ -94,6 +94,40 @@ describe("runDeviceAction", () => { }), ); + it.effect( + "rotates emulators through the accelerometer and physical devices through the lock", + () => + Effect.gen(function* () { + const emulator = makeReady(); + yield* runDeviceAction(emulator.ready, "android", { + type: "setOrientation", + deviceId: "emulator-5554", + value: "landscape_left", + }); + expect(emulator.calls.at(-1)?.args).toEqual([ + "-s", + "emulator-5554", + "emu", + "sensor", + "set", + "acceleration", + "9.81:0:0", + ]); + const phone = makeReady(); + yield* runDeviceAction(phone.ready, "android", { + type: "setOrientation", + deviceId: "R5CT1234", + value: "landscape_right", + }); + expect(phone.calls).toEqual([ + { + command: "adb", + args: ["-s", "R5CT1234", "shell", "cmd", "window", "user-rotation", "lock", "3"], + }, + ]); + }), + ); + it.effect("runs accessibility toggles through the bundled helper via simctl spawn", () => Effect.gen(function* () { const { ready, calls } = makeReady(); diff --git a/apps/server/src/device/DeviceActions.ts b/apps/server/src/device/DeviceActions.ts index 69143f317cb6..80b0f23ba5b1 100644 --- a/apps/server/src/device/DeviceActions.ts +++ b/apps/server/src/device/DeviceActions.ts @@ -17,6 +17,7 @@ import { type DeviceActionType, type DeviceForegroundApp, DeviceOperationError, + type DeviceOrientation, type DevicePlatform, type DeviceSettings, type DeviceTextSize, @@ -159,6 +160,21 @@ const ANDROID_PERMISSIONS: Record> = { motion: ["android.permission.ACTIVITY_RECOGNITION"], }; +// Gravity vector (x:y:z) that makes the emulator report each orientation, +// and the window-manager rotation index for the same. +const ANDROID_GRAVITY: Record = { + portrait: "0:9.81:0", + landscape_left: "9.81:0:0", + portrait_upside_down: "0:-9.81:0", + landscape_right: "-9.81:0:0", +}; +const ANDROID_ROTATION: Record = { + portrait: "0", + landscape_left: "1", + portrait_upside_down: "2", + landscape_right: "3", +}; + export const runDeviceAction = Effect.fn("DeviceActions.run")(function* ( ready: DeviceHostReady, platform: DevicePlatform, @@ -336,15 +352,25 @@ const runAndroid = Effect.fn("DeviceActions.runAndroid")(function* ( } return yield* fail(input.type, `${input.setting} is not supported on Android.`); case "setOrientation": { - const rotation = - input.value === "portrait" - ? "0" - : input.value === "landscape_left" - ? "1" - : input.value === "portrait_upside_down" - ? "2" - : "3"; - yield* shell(["cmd", "window", "user-rotation", "lock", rotation], "orientation"); + // `user-rotation lock` only rotates window content on recent images; + // the display the encoder captures stays put. Tilting the emulator's + // accelerometer rotates it for real, so that is used whenever the + // target is an emulator. Physical devices get the lock. + if (serial.startsWith("emulator-")) { + yield* shell(["settings", "put", "system", "accelerometer_rotation", "1"], "orientation"); + yield* shell(["cmd", "window", "user-rotation", "free"], "orientation"); + yield* adb( + run, + serial, + ["emu", "sensor", "set", "acceleration", ANDROID_GRAVITY[input.value]], + "orientation", + ); + return; + } + yield* shell( + ["cmd", "window", "user-rotation", "lock", ANDROID_ROTATION[input.value]], + "orientation", + ); return; } case "setLocation": From b743d721d299e4c0808ed4b633f5586e512410a1 Mon Sep 17 00:00:00 2001 From: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com> Date: Tue, 8 Sep 2026 00:42:29 -0700 Subject: [PATCH 15/28] fix(devices): only overlay pointable Android accessibility nodes Co-Authored-By: Claude Fable 5 --- .../web/src/components/device/deviceHubApi.ts | 29 ++++++++++++------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/apps/web/src/components/device/deviceHubApi.ts b/apps/web/src/components/device/deviceHubApi.ts index f069a8207cfe..a121effea83d 100644 --- a/apps/web/src/components/device/deviceHubApi.ts +++ b/apps/web/src/components/device/deviceHubApi.ts @@ -138,22 +138,31 @@ export async function fetchDeviceAxTree( const root = nodes[0]?.bounds as Record | undefined; const screenWidth = Math.max(1, numberOr(root?.right, 1)); const screenHeight = Math.max(1, numberOr(root?.bottom, 1)); - const elements = nodes.slice(1).map((node): DeviceAxElement => { + // Layout containers span the whole window and would tint the entire + // screen; only nodes a user could point at are worth drawing. + const elements = nodes.slice(1).flatMap((node): DeviceAxElement[] => { const bounds = node.bounds as Record; const left = numberOr(bounds.left, 0); const top = numberOr(bounds.top, 0); + const width = (numberOr(bounds.right, left) - left) / screenWidth; + const height = (numberOr(bounds.bottom, top) - top) / screenHeight; const text = typeof node.text === "string" ? node.text : ""; const description = typeof node.contentDescription === "string" ? node.contentDescription : ""; + const label = text || description; + if (width >= 0.95 && height >= 0.9) return []; + if (!label && node.clickable !== true) return []; const className = typeof node.className === "string" ? node.className : ""; - return { - id: String(node.id ?? ""), - label: text || description, - role: className.split(".").at(-1) ?? "", - x: left / screenWidth, - y: top / screenHeight, - width: (numberOr(bounds.right, left) - left) / screenWidth, - height: (numberOr(bounds.bottom, top) - top) / screenHeight, - }; + return [ + { + id: String(node.id ?? ""), + label, + role: className.split(".").at(-1) ?? "", + x: left / screenWidth, + y: top / screenHeight, + width, + height, + }, + ]; }); return { elements, errors: [] }; } From a7feca56f0a4ecca0a7c48147bb09fe4b4ec03dc Mon Sep 17 00:00:00 2001 From: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com> Date: Tue, 8 Sep 2026 16:14:14 -0700 Subject: [PATCH 16/28] fix(devices): preserve typed errors through screenshot registration --- apps/server/src/device/DeviceActions.ts | 2 +- apps/server/src/mcp/McpHttpServer.ts | 10 ++++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/apps/server/src/device/DeviceActions.ts b/apps/server/src/device/DeviceActions.ts index 80b0f23ba5b1..8ebc17475cef 100644 --- a/apps/server/src/device/DeviceActions.ts +++ b/apps/server/src/device/DeviceActions.ts @@ -431,7 +431,7 @@ export const readDeviceDetail = Effect.fn("DeviceActions.readDetail")(function* : yield* readAndroid(ready.run, deviceId); }); -const quiet = (effect: Effect.Effect) => +const quiet = (effect: Effect.Effect) => effect.pipe(Effect.orElseSucceed((): A | undefined => undefined)); const readIos = Effect.fn("DeviceActions.readIos")(function* ( diff --git a/apps/server/src/mcp/McpHttpServer.ts b/apps/server/src/mcp/McpHttpServer.ts index 965426b2139a..5a32ab13bdcf 100644 --- a/apps/server/src/mcp/McpHttpServer.ts +++ b/apps/server/src/mcp/McpHttpServer.ts @@ -386,16 +386,14 @@ const imageToolFailure = * result carries a `screenshot` field are registered by hand so the PNG goes * out as an image block and the rest of the payload as JSON metadata. */ -const registerImageTool = ( +const registerImageTool = ( tool: T, - handle: ( - payload: Tool.Parameters, - ) => Effect.Effect<{ readonly encodedResult: unknown }, unknown, R>, + handle: (payload: Tool.Parameters) => Effect.Effect<{ readonly encodedResult: unknown }, E, R>, provide: ( - effect: Effect.Effect<{ readonly encodedResult: unknown }, unknown, R>, + effect: Effect.Effect<{ readonly encodedResult: unknown }, E, R>, ) => Effect.Effect< { readonly encodedResult: unknown }, - unknown, + E, McpInvocationContext.McpInvocationContext >, operation: string, From ca5f8a0199749e385f5a2a2f514d860872387e20 Mon Sep 17 00:00:00 2001 From: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com> Date: Tue, 8 Sep 2026 16:32:41 -0700 Subject: [PATCH 17/28] fix(devices): preserve snapshot handling after main rebase --- apps/server/src/mcp/McpHttpServer.ts | 193 +++++++++--------- .../web/src/components/device/deviceStream.ts | 2 +- 2 files changed, 102 insertions(+), 93 deletions(-) diff --git a/apps/server/src/mcp/McpHttpServer.ts b/apps/server/src/mcp/McpHttpServer.ts index 5a32ab13bdcf..747f785bbd7d 100644 --- a/apps/server/src/mcp/McpHttpServer.ts +++ b/apps/server/src/mcp/McpHttpServer.ts @@ -333,6 +333,107 @@ const previewSnapshotFailure = (cause: Cause.Cause) => { }, // Agents usually see only the text content, so name the tag there too. content: [{ type: "text", text: `Preview snapshot failed: ${errorTag}.` }], + }); + return Effect.logWarning("preview snapshot failed", { + operation: "snapshot", + errorTag, + failureCount: failures.length, + }).pipe(Effect.as(result)); +}; + +const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot")(function* () { + const server = yield* McpServer.McpServer; + const broker = yield* PreviewAutomationBroker.PreviewAutomationBroker; + // The MCP tool runner only supplies the client, so hand the save path its services here. + const saveServices = yield* Effect.context< + ServerConfig.ServerConfig | FileSystem.FileSystem | Path.Path + >(); + const built = yield* PreviewSnapshotToolkit; + const tool = PreviewSnapshotTool; + yield* server.addTool({ + tool: new McpSchema.Tool({ + name: tool.name, + description: Tool.getDescription(tool), + inputSchema: Tool.getJsonSchema(tool), + annotations: { + ...Context.getOption(tool.annotations, Tool.Title).pipe( + Option.map((title) => ({ title })), + Option.getOrUndefined, + ), + readOnlyHint: Context.get(tool.annotations, Tool.Readonly), + destructiveHint: Context.get(tool.annotations, Tool.Destructive), + idempotentHint: Context.get(tool.annotations, Tool.Idempotent), + openWorldHint: Context.get(tool.annotations, Tool.OpenWorld), + }, + }), + annotations: tool.annotations, + handle: (payload) => + Effect.withFiber((fiber) => { + const invocation = Context.getUnsafe( + fiber.context, + McpInvocationContext.McpInvocationContext, + ); + return built.handle("preview_snapshot", payload).pipe( + Stream.unwrap, + Stream.run(Sink.last()), + Effect.flatMap(Effect.fromOption), + Effect.provideService(PreviewAutomationBroker.PreviewAutomationBroker, broker), + Effect.provideService(McpInvocationContext.McpInvocationContext, invocation), + Effect.flatMap(({ encodedResult }) => + Effect.gen(function* () { + const snapshot = encodedResult as SnapshotMetadata & { + readonly url: string; + readonly screenshot: { + readonly mimeType: "image/png"; + readonly data: string; + readonly width: number; + readonly height: number; + }; + }; + const { screenshot, ...page } = snapshot; + const png = new Uint8Array(Buffer.from(screenshot.data, "base64")); + const screenshotPath = + payload?.save === true ? yield* saveScreenshot(snapshot.url, png) : undefined; + const metadata = { + ...page, + screenshot: { + mimeType: screenshot.mimeType, + width: screenshot.width, + height: screenshot.height, + }, + ...(screenshotPath === undefined ? {} : { screenshotPath }), + }; + const bounded = boundSnapshotMetadata(metadata); + return new McpSchema.CallToolResult({ + isError: false, + structuredContent: metadata, + content: [ + { type: "text", text: bounded.text }, + ...(bounded.omitted.length === 0 + ? [] + : [ + { + type: "text" as const, + text: `Snapshot text was bounded. Omitted: ${bounded.omitted.join("; ")}.`, + }, + ]), + ...(payload?.includeImage === false + ? [] + : [{ type: "image" as const, data: png, mimeType: screenshot.mimeType }]), + ], + }); + }), + ), + Effect.provide(saveServices), + Effect.matchCauseEffect({ + onFailure: previewSnapshotFailure, + onSuccess: Effect.succeed, + }), + ); + }), + }); +}); + interface ImageToolResult { readonly screenshot: { readonly mimeType: "image/png"; @@ -466,98 +567,6 @@ const registerImageTool = ( }); }); -const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot")(function* () { - const broker = yield* PreviewAutomationBroker.PreviewAutomationBroker; - // The MCP tool runner only supplies the client, so hand the save path its services here. - const saveServices = yield* Effect.context< - ServerConfig.ServerConfig | FileSystem.FileSystem | Path.Path - >(); - const built = yield* PreviewSnapshotToolkit; - const tool = PreviewSnapshotTool; - yield* server.addTool({ - tool: new McpSchema.Tool({ - name: tool.name, - description: Tool.getDescription(tool), - inputSchema: Tool.getJsonSchema(tool), - annotations: { - ...Context.getOption(tool.annotations, Tool.Title).pipe( - Option.map((title) => ({ title })), - Option.getOrUndefined, - ), - readOnlyHint: Context.get(tool.annotations, Tool.Readonly), - destructiveHint: Context.get(tool.annotations, Tool.Destructive), - idempotentHint: Context.get(tool.annotations, Tool.Idempotent), - openWorldHint: Context.get(tool.annotations, Tool.OpenWorld), - }, - }), - annotations: tool.annotations, - handle: (payload) => - Effect.withFiber((fiber) => { - const invocation = Context.getUnsafe( - fiber.context, - McpInvocationContext.McpInvocationContext, - ); - return built.handle("preview_snapshot", payload).pipe( - Stream.unwrap, - Stream.run(Sink.last()), - Effect.flatMap(Effect.fromOption), - Effect.provideService(PreviewAutomationBroker.PreviewAutomationBroker, broker), - Effect.provideService(McpInvocationContext.McpInvocationContext, invocation), - Effect.flatMap(({ encodedResult }) => - Effect.gen(function* () { - const snapshot = encodedResult as SnapshotMetadata & { - readonly url: string; - readonly screenshot: { - readonly mimeType: "image/png"; - readonly data: string; - readonly width: number; - readonly height: number; - }; - }; - const { screenshot, ...page } = snapshot; - const png = new Uint8Array(Buffer.from(screenshot.data, "base64")); - const screenshotPath = - payload?.save === true ? yield* saveScreenshot(snapshot.url, png) : undefined; - const metadata = { - ...page, - screenshot: { - mimeType: screenshot.mimeType, - width: screenshot.width, - height: screenshot.height, - }, - ...(screenshotPath === undefined ? {} : { screenshotPath }), - }; - const bounded = boundSnapshotMetadata(metadata); - return new McpSchema.CallToolResult({ - isError: false, - structuredContent: metadata, - content: [ - { type: "text", text: bounded.text }, - ...(bounded.omitted.length === 0 - ? [] - : [ - { - type: "text" as const, - text: `Snapshot text was bounded. Omitted: ${bounded.omitted.join("; ")}.`, - }, - ]), - ...(payload?.includeImage === false - ? [] - : [{ type: "image" as const, data: png, mimeType: screenshot.mimeType }]), - ], - }); - }), - ), - Effect.provide(saveServices), - Effect.matchCauseEffect({ - onFailure: previewSnapshotFailure, - onSuccess: Effect.succeed, - }), - ); - }), - }); -}); - const registerDeviceScreenshot = Effect.fn("McpHttpServer.registerDeviceScreenshot")(function* () { const devices = yield* DeviceService.DeviceService; const built = yield* DeviceScreenshotToolkit; diff --git a/apps/web/src/components/device/deviceStream.ts b/apps/web/src/components/device/deviceStream.ts index a993bf81f1e7..a1e13b22fe8b 100644 --- a/apps/web/src/components/device/deviceStream.ts +++ b/apps/web/src/components/device/deviceStream.ts @@ -70,7 +70,7 @@ const IOS_TAG_SCREEN_CONFIG = 0x82; const encoder = new TextEncoder(); const decoder = new TextDecoder(); -export const isWebCodecsSupported = (): boolean => +const isWebCodecsSupported = (): boolean => typeof globalThis !== "undefined" && "VideoDecoder" in globalThis && "EncodedVideoChunk" in globalThis; From d3039fdca4741fa582ca0f072e53a0135975e9fa Mon Sep 17 00:00:00 2001 From: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com> Date: Tue, 8 Sep 2026 16:36:28 -0700 Subject: [PATCH 18/28] fix(devices): keep implementation-only symbols private --- apps/server/src/device/DeviceActions.ts | 4 ++-- apps/server/src/device/DeviceService.ts | 2 +- apps/server/src/device/DeviceToolchain.ts | 10 +++++----- apps/server/src/device/LocalDeviceHost.ts | 3 --- apps/server/src/mcp/toolkits/device/handlers.ts | 2 -- apps/server/src/mcp/toolkits/device/tools.ts | 6 +++--- apps/server/src/provider/CodexDeveloperInstructions.ts | 2 +- apps/server/src/provider/Layers/CodexSessionRuntime.ts | 2 +- 8 files changed, 13 insertions(+), 18 deletions(-) diff --git a/apps/server/src/device/DeviceActions.ts b/apps/server/src/device/DeviceActions.ts index 8ebc17475cef..df284f4b1839 100644 --- a/apps/server/src/device/DeviceActions.ts +++ b/apps/server/src/device/DeviceActions.ts @@ -36,7 +36,7 @@ const encodePushPayload = Schema.encodeUnknownEffect( Schema.fromJsonString(Schema.Record(Schema.String, Schema.Unknown)), ); -export const IOS_ACTIONS: ReadonlySet = new Set([ +const IOS_ACTIONS: ReadonlySet = new Set([ "setAppearance", "setTextSize", "setToggle", @@ -51,7 +51,7 @@ export const IOS_ACTIONS: ReadonlySet = new Set([ "sendPush", ]); -export const ANDROID_ACTIONS: ReadonlySet = new Set([ +const ANDROID_ACTIONS: ReadonlySet = new Set([ "setAppearance", "setTextSize", "setToggle", diff --git a/apps/server/src/device/DeviceService.ts b/apps/server/src/device/DeviceService.ts index 46267a72490d..3a6c22cbce6f 100644 --- a/apps/server/src/device/DeviceService.ts +++ b/apps/server/src/device/DeviceService.ts @@ -122,7 +122,7 @@ interface ServiceState { const vendorPrefix = (platform: DevicePlatform) => platform === "ios" ? "/vendor/serve-sim" : "/vendor/serve-emu"; -export const make = Effect.gen(function* () { +const make = Effect.gen(function* () { const localHost = yield* LocalDeviceHost.make(); const hosts: ReadonlyMap = new Map([[localHost.id, localHost]]); const httpClient = (yield* HttpClient.HttpClient).pipe(HttpClient.withScope); diff --git a/apps/server/src/device/DeviceToolchain.ts b/apps/server/src/device/DeviceToolchain.ts index b8959ba1dc5b..de707592ed98 100644 --- a/apps/server/src/device/DeviceToolchain.ts +++ b/apps/server/src/device/DeviceToolchain.ts @@ -23,10 +23,10 @@ import * as Semaphore from "effect/Semaphore"; import * as ProcessRunner from "../processRunner.ts"; -export const DEVICE_HUB_PACKAGE = "expo-device-hub"; -export const DEVICE_HUB_VERSION = "0.9.0"; -export const AGENT_DEVICE_PACKAGE = "agent-device"; -export const AGENT_DEVICE_VERSION = "0.20.10"; +const DEVICE_HUB_PACKAGE = "expo-device-hub"; +const DEVICE_HUB_VERSION = "0.9.0"; +const AGENT_DEVICE_PACKAGE = "agent-device"; +const AGENT_DEVICE_VERSION = "0.20.10"; const DEVICE_TOOLS_DIR = "device"; const INSTALL_TIMEOUT = Duration.minutes(10); @@ -87,7 +87,7 @@ const toolPaths = (path: Path.Path, baseDir: string, spec: ToolSpec): DeviceTool }; }; -export const deviceToolchainPaths = (path: Path.Path, baseDir: string): DeviceToolchainPaths => ({ +const deviceToolchainPaths = (path: Path.Path, baseDir: string): DeviceToolchainPaths => ({ hub: toolPaths(path, baseDir, HUB_SPEC), agentDevice: toolPaths(path, baseDir, AGENT_DEVICE_SPEC), }); diff --git a/apps/server/src/device/LocalDeviceHost.ts b/apps/server/src/device/LocalDeviceHost.ts index 3066de791dc9..27791a2f0934 100644 --- a/apps/server/src/device/LocalDeviceHost.ts +++ b/apps/server/src/device/LocalDeviceHost.ts @@ -547,6 +547,3 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () { }; return host; }); - -/** Exposed for tests. */ -export const __testing = { AgentDeviceDaemonFile }; diff --git a/apps/server/src/mcp/toolkits/device/handlers.ts b/apps/server/src/mcp/toolkits/device/handlers.ts index dad91114d5c9..fa4d2b90fde3 100644 --- a/apps/server/src/mcp/toolkits/device/handlers.ts +++ b/apps/server/src/mcp/toolkits/device/handlers.ts @@ -198,5 +198,3 @@ export const DeviceStandardToolkitHandlersLive = DeviceStandardToolkit.toLayer(s export const DeviceScreenshotToolkitHandlersLive = DeviceScreenshotToolkit.toLayer({ device_screenshot, }); - -export const DeviceToolkitHandlersLive = DeviceToolkit.toLayer(handlers); diff --git a/apps/server/src/mcp/toolkits/device/tools.ts b/apps/server/src/mcp/toolkits/device/tools.ts index 68b083174870..e98d6ab64829 100644 --- a/apps/server/src/mcp/toolkits/device/tools.ts +++ b/apps/server/src/mcp/toolkits/device/tools.ts @@ -22,7 +22,7 @@ const dependencies = [McpInvocationContext.McpInvocationContext, DeviceService.D * semantic snapshot model agents need and stays current with its own * releases. Wrapping its commands here would only lag behind it. */ -export const DeviceListTool = Tool.make("device_list", { +const DeviceListTool = Tool.make("device_list", { description: "List iOS Simulators and Android Emulators on this environment's device hosts, which platforms each host can run, and which devices are already open in this thread's Device panel. Call this before device_open when you do not know a device id.", // An empty struct serializes as `anyOf [object, array]`, which some @@ -42,7 +42,7 @@ export const DeviceListTool = Tool.make("device_list", { .annotate(Tool.Idempotent, true) .annotate(Tool.OpenWorld, false); -export const DeviceOpenTool = Tool.make("device_open", { +const DeviceOpenTool = Tool.make("device_open", { description: "Open a simulator or emulator for this thread: boots it if needed, starts its live stream, and shows it in the user's Device panel so they can watch. Returns the agent-device CLI invocation pinned to the device; drive the device with that CLI afterwards.", parameters: DeviceToolOpenInput, @@ -70,7 +70,7 @@ export const DeviceScreenshotTool = Tool.make("device_screenshot", { .annotate(Tool.Idempotent, true) .annotate(Tool.OpenWorld, true); -export const DeviceCloseTool = Tool.make("device_close", { +const DeviceCloseTool = Tool.make("device_close", { description: "Remove a device from this thread's Device panel. Pass shutdown=true to also power the simulator or emulator off.", parameters: DeviceToolCloseInput, diff --git a/apps/server/src/provider/CodexDeveloperInstructions.ts b/apps/server/src/provider/CodexDeveloperInstructions.ts index 0e9f4a35a188..85784d21ca4b 100644 --- a/apps/server/src/provider/CodexDeveloperInstructions.ts +++ b/apps/server/src/provider/CodexDeveloperInstructions.ts @@ -12,7 +12,7 @@ For browser work, first call \`preview_status\`. If no automation-capable previe Do not switch to global browser skills, Chrome, Node REPL browser automation, standalone Playwright, or agent-browser merely because the preview is initially closed or a first call fails. Use an alternative browser system only when the T3 preview tools are absent, the user explicitly requests another browser, or \`preview_open\` returns an explicit unsupported/unavailable error. A failed T3 preview tool call should be inspected and retried with corrected arguments when the error is actionable. `; -export const T3_CODE_DEVICE_TOOL_INSTRUCTIONS = ` +const T3_CODE_DEVICE_TOOL_INSTRUCTIONS = ` ## T3 Code devices diff --git a/apps/server/src/provider/Layers/CodexSessionRuntime.ts b/apps/server/src/provider/Layers/CodexSessionRuntime.ts index 29499d700689..d41cfd73d76a 100644 --- a/apps/server/src/provider/Layers/CodexSessionRuntime.ts +++ b/apps/server/src/provider/Layers/CodexSessionRuntime.ts @@ -69,7 +69,7 @@ export function hasConfiguredMcpServer(appServerArgs: ReadonlyArray | un return appServerArgs?.some((argument) => argument.includes("mcp_servers.")) === true; } -export function configuredMcpToolAvailability( +function configuredMcpToolAvailability( appServerArgs: ReadonlyArray | undefined, mcpCapabilities: ReadonlySet | undefined, ): T3CodeToolAvailability { From ff148f96e9039a7c0261b2f814f4bd89c873b121 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Tue, 8 Sep 2026 13:50:33 -0700 Subject: [PATCH 19/28] feat(devices): add consent-based device onboarding and setup --- apps/server/src/auth/RpcAuthorization.ts | 1 + apps/server/src/device/DeviceActions.test.ts | 1 - apps/server/src/device/DeviceHost.ts | 18 +- apps/server/src/device/DeviceService.test.ts | 215 +++++++++++- apps/server/src/device/DeviceService.ts | 244 ++++++++++++-- apps/server/src/device/DeviceToolchain.ts | 50 +-- .../server/src/device/LocalDeviceHost.test.ts | 59 ++++ apps/server/src/device/LocalDeviceHost.ts | 263 +++++++++++---- apps/server/src/mcp/McpDeviceToolkit.test.ts | 4 + .../src/mcp/toolkits/device/handlers.ts | 12 + .../src/provider/Layers/ProviderService.ts | 23 +- apps/server/src/server.test.ts | 2 + apps/server/src/server.ts | 1 + apps/server/src/ws.ts | 4 + apps/web/src/components/ChatView.tsx | 45 ++- .../web/src/components/device/DevicePanel.tsx | Bin 12531 -> 17207 bytes .../web/src/components/device/DeviceSetup.tsx | 314 ++++++++++++++++++ .../components/preview/PreviewEmptyState.tsx | 9 +- .../preview/PreviewLocalServerCard.tsx | 18 +- .../settings/IntegrationsSettings.test.tsx | 62 +++- .../settings/IntegrationsSettings.tsx | 123 +++++++ .../settings/ProjectDefaultsSettings.tsx | 57 ---- .../components/settings/SettingsPanels.tsx | 5 - .../src/components/settings/settingsSearch.ts | 17 +- apps/web/src/components/ui/discovery-list.tsx | 37 +++ apps/web/src/state/device.ts | 2 + docs/internals/devices.md | 15 +- docs/user/devices.md | 34 +- packages/client-runtime/src/state/device.ts | 6 + packages/contracts/src/device.ts | 15 + packages/contracts/src/rpc.ts | 9 + packages/contracts/src/settings.ts | 12 +- 32 files changed, 1443 insertions(+), 234 deletions(-) create mode 100644 apps/server/src/device/LocalDeviceHost.test.ts create mode 100644 apps/web/src/components/device/DeviceSetup.tsx create mode 100644 apps/web/src/components/ui/discovery-list.tsx diff --git a/apps/server/src/auth/RpcAuthorization.ts b/apps/server/src/auth/RpcAuthorization.ts index 24b6f5cb58d0..5689d3e9e105 100644 --- a/apps/server/src/auth/RpcAuthorization.ts +++ b/apps/server/src/auth/RpcAuthorization.ts @@ -142,6 +142,7 @@ export const RPC_REQUIRED_SCOPES = { [WS_METHODS.previewAutomationFocusHost]: AuthOrchestrationOperateScope, [WS_METHODS.subscribePreviewEvents]: AuthOrchestrationReadScope, [WS_METHODS.subscribeDiscoveredLocalServers]: AuthOrchestrationReadScope, + [WS_METHODS.deviceConfigure]: AuthOrchestrationOperateScope, [WS_METHODS.deviceList]: AuthOrchestrationReadScope, [WS_METHODS.deviceOpen]: AuthOrchestrationOperateScope, [WS_METHODS.deviceClose]: AuthOrchestrationOperateScope, diff --git a/apps/server/src/device/DeviceActions.test.ts b/apps/server/src/device/DeviceActions.test.ts index 1b94cda5a73e..a2719d8cc4b7 100644 --- a/apps/server/src/device/DeviceActions.test.ts +++ b/apps/server/src/device/DeviceActions.test.ts @@ -17,7 +17,6 @@ const makeReady = ( const calls: Call[] = []; const ready: DeviceHostReady = { hub: { origin: "http://127.0.0.1:1" }, - agentDevice: { baseUrl: "http://127.0.0.1:2", token: "t", entryPath: "/x" }, helpers, run: (command, args, options) => { const call = { command, args, ...(options?.stdin ? { stdin: options.stdin } : {}) }; diff --git a/apps/server/src/device/DeviceHost.ts b/apps/server/src/device/DeviceHost.ts index bc162f9a109b..23edc61bb97a 100644 --- a/apps/server/src/device/DeviceHost.ts +++ b/apps/server/src/device/DeviceHost.ts @@ -4,9 +4,10 @@ * in beside `LocalDeviceHost` without touching discovery, the proxy, or the * MCP tools. * - * Every host presents the same two things once it is ready: a loopback origin - * where expo-device-hub answers, and an agent-device daemon endpoint. For the - * local host both run on this machine; a remote host would forward them here. + * Every ready host presents a loopback origin where expo-device-hub answers. + * Hosts add an agent-device daemon endpoint only after agent access is granted. + * For the local host both run on this machine; a remote host would forward + * them here. */ import type { DeviceHostId, @@ -42,7 +43,6 @@ export interface AgentDeviceEndpoint { export interface DeviceHostReady { readonly hub: DeviceHubEndpoint; - readonly agentDevice: AgentDeviceEndpoint; /** * Runs a host command (`xcrun`, `adb`, or a helper bundled with the hub) * where the devices live. On the local host this is a plain spawn; a @@ -60,6 +60,10 @@ export interface DeviceHostReady { }; } +export interface DeviceHostAgentReady extends DeviceHostReady { + readonly agentDevice: AgentDeviceEndpoint; +} + export interface DeviceHost { readonly id: DeviceHostId; readonly summary: Effect.Effect; @@ -73,8 +77,14 @@ export interface DeviceHost { readonly ensureReady: ( onPhase: (phase: "installing" | "starting") => Effect.Effect, ) => Effect.Effect; + /** Installs and starts agent-device after the user grants agent access. */ + readonly ensureAgentReady: ( + onPhase: (phase: "installing" | "starting") => Effect.Effect, + ) => Effect.Effect; /** Current endpoints when already running, without starting anything. */ readonly current: Effect.Effect; + /** Stops only agent-device. Manual viewing through the hub stays available. */ + readonly stopAgent: Effect.Effect; /** Stops helpers. Devices themselves keep running; the user owns those. */ readonly stop: Effect.Effect; } diff --git a/apps/server/src/device/DeviceService.test.ts b/apps/server/src/device/DeviceService.test.ts index a41f1242c337..e999bb973bea 100644 --- a/apps/server/src/device/DeviceService.test.ts +++ b/apps/server/src/device/DeviceService.test.ts @@ -1,18 +1,29 @@ import { describe, expect, it } from "@effect/vitest"; -import type { DeviceServiceState } from "@t3tools/contracts"; +import { + DEFAULT_SERVER_SETTINGS, + LOCAL_DEVICE_HOST_ID, + ThreadId, + type DeviceServiceState, +} from "@t3tools/contracts"; import * as Effect from "effect/Effect"; +import * as Deferred from "effect/Deferred"; import * as Fiber from "effect/Fiber"; import * as PubSub from "effect/PubSub"; import * as Ref from "effect/Ref"; import * as Stream from "effect/Stream"; +import { HttpClient, HttpClientResponse } from "effect/unstable/http"; +import { ServerSettingsService } from "../serverSettings.ts"; +import type { DeviceHost, DeviceHostReady } from "./DeviceHost.ts"; -import { type DeviceService, stateStream } from "./DeviceService.ts"; +import { type DeviceService, makeWithHost, stateStream } from "./DeviceService.ts"; const baseState: DeviceServiceState = { hosts: [], hostStatus: "idle", devices: [], sessions: [], + onboardingCompleted: false, + agentAccessEnabled: false, hubBasePath: "/api/device-hub", revision: 0, }; @@ -43,3 +54,203 @@ describe("DeviceService.stateStream", () => { }), ); }); + +const fixture = Effect.fn("fixture")(function* (onBoot: Effect.Effect = Effect.void) { + const settings = yield* Ref.make(DEFAULT_SERVER_SETTINGS); + const starts: string[] = []; + const agentStarts: string[] = []; + const agentStops: string[] = []; + const requests: string[] = []; + let booted = false; + const ready: DeviceHostReady = { + hub: { origin: "http://device.test" }, + helpers: { serveSimAxSettings: null, serveSimCli: null }, + run: () => Effect.succeed({ code: 0, stdout: "Pixel_API_35\n", stderr: "" }), + }; + const host: DeviceHost = { + id: LOCAL_DEVICE_HOST_ID, + summary: Effect.succeed({ + id: LOCAL_DEVICE_HOST_ID, + kind: "local", + label: "Test server", + platforms: [{ platform: "android", available: true }], + hubInstalled: true, + agentDeviceInstalled: false, + }), + platformAvailability: (platform) => Effect.succeed({ platform, available: true }), + ensureReady: (onPhase) => + Effect.gen(function* () { + starts.push("start"); + yield* onPhase("starting"); + return ready; + }), + ensureAgentReady: (onPhase) => + Effect.gen(function* () { + agentStarts.push("start"); + yield* onPhase("starting"); + return { + ...ready, + agentDevice: { baseUrl: "http://agent.test", token: "test", entryPath: "/agent" }, + }; + }), + current: Effect.succeed(null), + stopAgent: Effect.sync(() => { + agentStops.push("stop"); + }), + stop: Effect.sync(() => { + starts.push("stop"); + }), + }; + const service = yield* makeWithHost(host).pipe( + Effect.provideService( + ServerSettingsService, + ServerSettingsService.of({ + start: Effect.void, + ready: Effect.void, + getSettings: Ref.get(settings), + updateSettings: (patch) => + Ref.updateAndGet(settings, (current) => ({ + ...current, + enableDeviceSupport: patch.enableDeviceSupport ?? current.enableDeviceSupport, + enableAgentDeviceAccess: + patch.enableAgentDeviceAccess ?? current.enableAgentDeviceAccess, + deviceOnboardingCompleted: + patch.deviceOnboardingCompleted ?? current.deviceOnboardingCompleted, + })), + streamChanges: Stream.empty, + subscribeChanges: Effect.succeed(Stream.empty), + }), + ), + Effect.provideService( + HttpClient.HttpClient, + HttpClient.make((request) => + Effect.gen(function* () { + requests.push(request.url); + if (request.url.endsWith("/boot")) { + yield* onBoot; + booted = true; + return HttpClientResponse.fromWeb( + request, + Response.json({ ok: true, serial: "emulator-5554" }), + ); + } + return HttpClientResponse.fromWeb( + request, + Response.json({ + simulators: [], + emulators: booted + ? [ + { + id: "emulator-5554", + name: "Pixel_API_35", + platform: "android", + version: "Android 15", + booted: true, + physical: false, + }, + ] + : [], + }), + ); + }), + ), + ), + ); + return { service, starts, agentStarts, agentStops, requests, settings }; +}); + +describe("device setup consent", () => { + it.effect("listing and provider startup do not start helpers before consent", () => + Effect.gen(function* () { + const { service, starts, requests } = yield* fixture(); + expect((yield* service.list).hostStatus).toBe("disabled"); + expect(yield* service.readinessIfSupported()).toBeNull(); + const readiness = yield* service.readiness().pipe(Effect.result); + expect(readiness._tag).toBe("Failure"); + expect(starts).toEqual([]); + expect(requests).toEqual([]); + }).pipe(Effect.scoped), + ); + + it.effect( + "explicit setup discovers never-booted AVDs; disabling stops helpers and blocks agents", + () => + Effect.gen(function* () { + const { service, starts, settings } = yield* fixture(); + const state = yield* service.configure({ enabled: true }); + expect((yield* Ref.get(settings)).enableDeviceSupport).toBe(true); + expect(state.devices.map((device) => [device.id, device.booted])).toEqual([ + ["Pixel_API_35", false], + ]); + expect(starts).toEqual(["start"]); + const disabled = yield* service.configure({ enabled: false }); + expect(disabled.hostStatus).toBe("disabled"); + expect(disabled.devices).toEqual([]); + expect((yield* Ref.get(settings)).enableDeviceSupport).toBe(false); + expect(yield* service.readinessIfSupported()).toBeNull(); + expect(starts).toEqual(["start", "stop"]); + }).pipe(Effect.scoped), + ); + + it.effect("boots a stopped Android AVD and uses its emulator serial without duplicating it", () => + Effect.gen(function* () { + const { service, requests } = yield* fixture(); + yield* service.configure({ enabled: true }); + const session = yield* service.open({ + threadId: ThreadId.make("thread-1"), + deviceId: "Pixel_API_35", + platform: "android", + }); + expect(session.deviceId).toBe("emulator-5554"); + expect(requests.filter((url) => url.endsWith("/boot"))).toHaveLength(1); + const state = yield* service.state; + expect(state.devices.map((device) => device.id)).toEqual(["emulator-5554"]); + expect(state.bootingDevices).toEqual([]); + }).pipe(Effect.scoped), + ); + + it.effect("installs agent support only after the separate agent permission", () => + Effect.gen(function* () { + const { service, agentStarts, agentStops, settings } = yield* fixture(); + yield* service.configure({ enabled: true }); + expect(agentStarts).toEqual([]); + expect(yield* service.agentReadinessIfSupported()).toBeNull(); + + yield* service.configure({ agentAccessEnabled: true }); + expect(agentStarts).toEqual(["start"]); + expect((yield* Ref.get(settings)).enableAgentDeviceAccess).toBe(true); + expect((yield* service.state).agentAccessEnabled).toBe(true); + + yield* service.configure({ agentAccessEnabled: false, onboardingCompleted: true }); + expect(agentStops).toEqual(["stop"]); + expect((yield* service.state).onboardingCompleted).toBe(true); + expect((yield* Ref.get(settings)).deviceOnboardingCompleted).toBe(true); + }).pipe(Effect.scoped), + ); +}); + +it.effect("publishes boot progress and does not restore sessions after support is disabled", () => + Effect.gen(function* () { + const started = yield* Deferred.make(); + const finish = yield* Deferred.make(); + const { service } = yield* fixture( + Deferred.succeed(started, undefined).pipe(Effect.andThen(Deferred.await(finish))), + ); + yield* service.configure({ enabled: true }); + const opening = yield* service + .open({ threadId: ThreadId.make("thread-1"), deviceId: "Pixel_API_35", platform: "android" }) + .pipe(Effect.result, Effect.forkChild); + yield* Deferred.await(started); + expect((yield* service.state).bootingDevices?.map((device) => device.name)).toEqual([ + "Pixel_API_35", + ]); + yield* service.configure({ enabled: false }); + yield* Deferred.succeed(finish, undefined); + expect((yield* Fiber.join(opening))._tag).toBe("Failure"); + const state = yield* service.state; + expect(state.hostStatus).toBe("disabled"); + expect(state.devices).toEqual([]); + expect(state.sessions).toEqual([]); + expect(state.bootingDevices).toEqual([]); + }).pipe(Effect.scoped), +); diff --git a/apps/server/src/device/DeviceService.ts b/apps/server/src/device/DeviceService.ts index 3a6c22cbce6f..573933de2d5b 100644 --- a/apps/server/src/device/DeviceService.ts +++ b/apps/server/src/device/DeviceService.ts @@ -13,6 +13,7 @@ import { type DeviceActionInput, type DeviceCloseInput, + type DeviceConfigureInput, type DeviceDetail, type DeviceDetailInput, type DeviceError, @@ -40,12 +41,15 @@ import * as Layer from "effect/Layer"; import * as PubSub from "effect/PubSub"; import * as Schema from "effect/Schema"; import * as Scope from "effect/Scope"; +import * as Semaphore from "effect/Semaphore"; import * as Stream from "effect/Stream"; import * as SynchronizedRef from "effect/SynchronizedRef"; import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"; +import { ServerSettingsService } from "../serverSettings.ts"; + import { readDeviceDetail, runDeviceAction } from "./DeviceActions.ts"; -import type { DeviceHost, DeviceHostReady } from "./DeviceHost.ts"; +import type { AgentDeviceEndpoint, DeviceHost, DeviceHostReady } from "./DeviceHost.ts"; import * as LocalDeviceHost from "./LocalDeviceHost.ts"; /** Origin-relative prefix the hub is proxied under. See DeviceHubProxy. */ @@ -83,12 +87,19 @@ export interface DeviceReadiness extends DeviceHostReady { readonly hostId: DeviceHostId; } +export interface DeviceAgentReadiness extends DeviceReadiness { + readonly agentDevice: AgentDeviceEndpoint; +} + export class DeviceService extends Context.Service< DeviceService, { readonly state: Effect.Effect; readonly subscribe: Effect.Effect, never, Scope.Scope>; - /** Refreshes device discovery; starts the host helpers on first call. */ + readonly configure: ( + input: DeviceConfigureInput, + ) => Effect.Effect; + /** Refreshes devices only after device support has been enabled. */ readonly list: Effect.Effect; readonly open: (input: DeviceOpenInput) => Effect.Effect; readonly close: (input: DeviceCloseInput) => Effect.Effect; @@ -110,6 +121,9 @@ export class DeviceService extends Context.Service< readonly readinessIfSupported: ( hostId?: DeviceHostId, ) => Effect.Effect; + readonly agentReadinessIfSupported: ( + hostId?: DeviceHostId, + ) => Effect.Effect; readonly currentReadiness: (hostId?: DeviceHostId) => Effect.Effect; readonly sessionsForThread: (threadId: ThreadId) => Effect.Effect>; } @@ -122,8 +136,22 @@ interface ServiceState { const vendorPrefix = (platform: DevicePlatform) => platform === "ios" ? "/vendor/serve-sim" : "/vendor/serve-emu"; -const make = Effect.gen(function* () { - const localHost = yield* LocalDeviceHost.make(); +export const makeWithHost = Effect.fn("DeviceService.makeWithHost")(function* ( + localHost: DeviceHost, +) { + const settings = yield* ServerSettingsService; + const lifecycleLock = yield* Semaphore.make(1); + const readDeviceSettings = settings.getSettings.pipe( + Effect.map((value) => ({ + enabled: value.enableDeviceSupport, + agentAccessEnabled: value.enableAgentDeviceAccess, + onboardingCompleted: value.deviceOnboardingCompleted, + })), + Effect.mapError( + (cause) => new DeviceOperationError({ operation: "settings", detail: cause.message }), + ), + ); + const initialSettings = yield* readDeviceSettings; const hosts: ReadonlyMap = new Map([[localHost.id, localHost]]); const httpClient = (yield* HttpClient.HttpClient).pipe(HttpClient.withScope); const statePubSub = yield* PubSub.unbounded(); @@ -131,9 +159,11 @@ const make = Effect.gen(function* () { const stateRef = yield* SynchronizedRef.make({ state: { hosts: initialHosts, - hostStatus: "idle", + hostStatus: initialSettings.enabled ? "idle" : "disabled", devices: [], sessions: [], + onboardingCompleted: initialSettings.onboardingCompleted, + agentAccessEnabled: initialSettings.agentAccessEnabled, hubBasePath: DEVICE_HUB_ROUTE_PREFIX, revision: 0, }, @@ -158,6 +188,13 @@ const make = Effect.gen(function* () { const readiness: DeviceService["Service"]["readiness"] = Effect.fn("DeviceService.readiness")( function* (hostId) { const host = yield* resolveHost(hostId); + if (!(yield* readDeviceSettings).enabled) { + return yield* new DeviceHostUnavailableError({ + hostId: host.id, + reason: + "Device support is off. Enable it in the Device panel before installing or starting device tools.", + }); + } const ready = yield* host .ensureReady((phase) => publish((state) => ({ ...state, hostStatus: phase, hostStatusDetail: undefined })).pipe( @@ -189,17 +226,49 @@ const make = Effect.gen(function* () { ); return { hostId: host.id, ...ready }; }, + lifecycleLock.withPermit, ); const readinessIfSupported: DeviceService["Service"]["readinessIfSupported"] = Effect.fn( "DeviceService.readinessIfSupported", )(function* (hostId) { + if (!(yield* readDeviceSettings).enabled) return null; const host = yield* resolveHost(hostId); const summary = yield* host.summary; if (!summary.platforms.some((platform) => platform.available)) return null; return yield* readiness(host.id); }); + const agentReadinessIfSupported: DeviceService["Service"]["agentReadinessIfSupported"] = + Effect.fn("DeviceService.agentReadinessIfSupported")(function* (hostId) { + const deviceSettings = yield* readDeviceSettings; + if (!deviceSettings.enabled || !deviceSettings.agentAccessEnabled) return null; + const host = yield* resolveHost(hostId); + const summary = yield* host.summary; + if (!summary.platforms.some((platform) => platform.available)) return null; + const ready = yield* host + .ensureAgentReady((phase) => + publish((state) => ({ ...state, hostStatus: phase, hostStatusDetail: undefined })).pipe( + Effect.asVoid, + ), + ) + .pipe( + Effect.tapError((error) => + publish((state) => ({ + ...state, + hostStatus: "failed", + hostStatusDetail: error.message, + })), + ), + Effect.mapError( + (error) => new DeviceHostUnavailableError({ hostId: host.id, reason: error.message }), + ), + ); + const hostSummaries = yield* Effect.forEach(hosts.values(), (candidate) => candidate.summary); + yield* publish((state) => ({ ...state, hosts: hostSummaries, hostStatus: "ready" })); + return { hostId: host.id, ...ready }; + }, lifecycleLock.withPermit); + const currentReadiness: DeviceService["Service"]["currentReadiness"] = (hostId) => resolveHost(hostId).pipe( Effect.flatMap((host) => @@ -245,19 +314,114 @@ const make = Effect.gen(function* () { booted: device.booted, physical: device.physical, }); - return [...list.simulators, ...list.emulators].map(toSummary); + const devices = [...list.simulators, ...list.emulators].map(toSummary); + const host = yield* resolveHost(ready.hostId); + if ((yield* host.platformAvailability("android")).available) { + const avds = yield* ready.run("emulator", ["-list-avds"]); + if (avds.code !== 0) { + return yield* new DeviceOperationError({ + operation: "list", + detail: `Could not list Android virtual devices: ${avds.stderr || avds.stdout}`, + }); + } + for (const name of avds.stdout + .split(/\r?\n/) + .map((line) => line.trim()) + .filter(Boolean)) { + if (!devices.some((device) => device.platform === "android" && device.name === name)) { + devices.push({ + hostId: ready.hostId, + id: name, + name, + platform: "android", + version: "Android", + booted: false, + physical: false, + }); + } + } + } + return { devices, detail: list.errors?.map((error) => error.message).join("\n") || undefined }; }); const refresh = Effect.fn("DeviceService.refresh")(function* (ready: DeviceReadiness) { - const devices = yield* fetchDevices(ready); + const { devices, detail } = yield* fetchDevices(ready); const hostSummaries = yield* Effect.forEach(hosts.values(), (host) => host.summary); - return yield* publish((state) => ({ ...state, hosts: hostSummaries, devices })); + return yield* lifecycleLock.withPermit( + Effect.gen(function* () { + if (!(yield* readDeviceSettings).enabled) + return (yield* SynchronizedRef.get(stateRef)).state; + return yield* publish((state) => ({ + ...state, + hosts: hostSummaries, + devices, + hostStatusDetail: detail, + })); + }), + ); }); const list: DeviceService["Service"]["list"] = Effect.gen(function* () { + if (!(yield* readDeviceSettings).enabled) return (yield* SynchronizedRef.get(stateRef)).state; const ready = yield* readiness(); return yield* refresh(ready); - }).pipe(Effect.withSpan("DeviceService.list")); + }).pipe( + Effect.tapError((error) => + publish((state) => + state.hostStatus === "disabled" + ? state + : { ...state, hostStatus: "failed", hostStatusDetail: error.message }, + ), + ), + Effect.withSpan("DeviceService.list"), + ); + + const configure: DeviceService["Service"]["configure"] = Effect.fn("DeviceService.configure")( + function* (input) { + const currentSettings = yield* readDeviceSettings; + const nextEnabled = input.enabled ?? currentSettings.enabled; + const nextAgentAccess = input.agentAccessEnabled ?? currentSettings.agentAccessEnabled; + yield* lifecycleLock.withPermit( + Effect.gen(function* () { + yield* settings + .updateSettings({ + ...(input.enabled === undefined ? {} : { enableDeviceSupport: input.enabled }), + ...(input.agentAccessEnabled === undefined + ? {} + : { enableAgentDeviceAccess: input.agentAccessEnabled }), + ...(input.onboardingCompleted === undefined + ? {} + : { deviceOnboardingCompleted: input.onboardingCompleted }), + }) + .pipe( + Effect.mapError( + (cause) => + new DeviceOperationError({ operation: "configure", detail: cause.message }), + ), + ); + if (!nextEnabled) { + yield* Effect.forEach(hosts.values(), (host) => host.stop, { discard: true }); + } else if (input.agentAccessEnabled === false) { + yield* Effect.forEach(hosts.values(), (host) => host.stopAgent, { discard: true }); + } + yield* publish((state) => ({ + ...state, + hostStatus: nextEnabled ? "idle" : "disabled", + hostStatusDetail: undefined, + devices: nextEnabled ? state.devices : [], + sessions: nextEnabled ? state.sessions : [], + bootingDevices: nextEnabled ? state.bootingDevices : [], + agentAccessEnabled: nextAgentAccess, + onboardingCompleted: input.onboardingCompleted ?? state.onboardingCompleted, + })); + }), + ); + if (nextEnabled && nextAgentAccess && input.agentAccessEnabled === true) { + yield* agentReadinessIfSupported(); + } + return yield* list; + }, + ); const findDevice = ( state: DeviceServiceState, @@ -331,7 +495,26 @@ const make = Effect.gen(function* () { return yield* new DeviceNotFoundError({ hostId: host.id, deviceId: input.deviceId }); } if (!device.booted && input.boot !== false) { - const bootedId = yield* boot(ready, device); + const booting = { ...device, threadId: input.threadId }; + yield* publish((current) => ({ + ...current, + bootingDevices: [ + ...(current.bootingDevices ?? []).filter( + (entry) => entry.hostId !== booting.hostId || entry.id !== booting.id, + ), + booting, + ], + })); + const bootedId = yield* boot(ready, device).pipe( + Effect.ensuring( + publish((current) => ({ + ...current, + bootingDevices: (current.bootingDevices ?? []).filter( + (entry) => entry.hostId !== booting.hostId || entry.id !== booting.id, + ), + })), + ), + ); state = yield* refresh(ready); device = findDevice(state, host.id, bootedId) ?? findDevice(state, host.id, device.id); if (!device) { @@ -359,20 +542,29 @@ const make = Effect.gen(function* () { platform: device.platform, openedAt, }; - yield* publish((current) => ({ - ...current, - sessions: [ - ...current.sessions.filter( - (existing) => - !( - existing.threadId === session.threadId && - existing.hostId === session.hostId && - existing.deviceId === session.deviceId + yield* lifecycleLock.withPermit( + Effect.gen(function* () { + if (!(yield* readDeviceSettings).enabled) + return yield* new DeviceHostUnavailableError({ + hostId: host.id, + reason: "Device support was turned off while the device was opening.", + }); + yield* publish((current) => ({ + ...current, + sessions: [ + ...current.sessions.filter( + (existing) => + !( + existing.threadId === session.threadId && + existing.hostId === session.hostId && + existing.deviceId === session.deviceId + ), ), - ), - session, - ], - })); + session, + ], + })); + }), + ); return session; }); @@ -509,6 +701,7 @@ const make = Effect.gen(function* () { return DeviceService.of({ state: SynchronizedRef.get(stateRef).pipe(Effect.map(({ state }) => state)), subscribe: PubSub.subscribe(statePubSub), + configure, list, open, close, @@ -518,9 +711,14 @@ const make = Effect.gen(function* () { screenshot, readiness, readinessIfSupported, + agentReadinessIfSupported, currentReadiness, sessionsForThread, }); +}); + +export const make = Effect.gen(function* () { + return yield* makeWithHost(yield* LocalDeviceHost.make()); }).pipe(Effect.withSpan("DeviceService.make")); export const layer = Layer.effect(DeviceService, make); diff --git a/apps/server/src/device/DeviceToolchain.ts b/apps/server/src/device/DeviceToolchain.ts index de707592ed98..bba940412f9f 100644 --- a/apps/server/src/device/DeviceToolchain.ts +++ b/apps/server/src/device/DeviceToolchain.ts @@ -2,8 +2,9 @@ * Pinned installs of the two external tools device support is built on. * * `expo-device-hub` streams simulator and emulator screens and `agent-device` - * drives them. Both are npm-installed once into `/device/tools/` - * and executed from there with the server's own Node, never `npx`: an ephemeral + * drives them. Each is npm-installed separately after its matching consent + * step into `/tools//` and executed from there with the + * server's own Node, never `npx`: an ephemeral * npx cache would make every first `device_open` after a reboot depend on the * registry, and the pinned versions are part of the contract the injected * agent instructions describe. @@ -28,7 +29,6 @@ const DEVICE_HUB_VERSION = "0.9.0"; const AGENT_DEVICE_PACKAGE = "agent-device"; const AGENT_DEVICE_VERSION = "0.20.10"; -const DEVICE_TOOLS_DIR = "device"; const INSTALL_TIMEOUT = Duration.minutes(10); const installLock = Semaphore.makeUnsafe(1); @@ -79,7 +79,7 @@ const AGENT_DEVICE_SPEC: ToolSpec = { }; const toolPaths = (path: Path.Path, baseDir: string, spec: ToolSpec): DeviceToolPaths => { - const installDir = path.join(baseDir, DEVICE_TOOLS_DIR, "tools", `${spec.name}@${spec.version}`); + const installDir = path.join(baseDir, "tools", spec.name, spec.version); return { installDir, entryPath: path.join(installDir, "node_modules", spec.name, ...spec.entry), @@ -92,9 +92,9 @@ const deviceToolchainPaths = (path: Path.Path, baseDir: string): DeviceToolchain agentDevice: toolPaths(path, baseDir, AGENT_DEVICE_SPEC), }); -/** The agent-device daemon state (daemon.json, sessions) lives beside the tools. */ +/** Keep daemon state (daemon.json, sessions) in userdata, separate from tool installs. */ export const agentDeviceStateDir = (path: Path.Path, stateDir: string): string => - path.join(stateDir, DEVICE_TOOLS_DIR, "agent-device"); + path.join(stateDir, "device", "agent-device"); const isInstalled = Effect.fn("DeviceToolchain.isInstalled")(function* ( fs: FileSystem.FileSystem, @@ -190,33 +190,35 @@ const installTool = Effect.fn("DeviceToolchain.installTool")(function* ( ); }); -/** - * Installs whichever of the two tools is missing and returns their paths. - * Serialized process-wide so two threads opening devices at once do not race - * npm against the same directory. - */ -export const ensureDeviceToolchain = Effect.fn("DeviceToolchain.ensure")(function* ( +const ensureTool = Effect.fn("DeviceToolchain.ensureTool")(function* ( baseDir: string, + spec: ToolSpec, + select: (paths: DeviceToolchainPaths) => DeviceToolPaths, ) { const path = yield* Path.Path; const paths = deviceToolchainPaths(path, baseDir); - return yield* installLock.withPermit( - Effect.all( - [installTool(HUB_SPEC, paths.hub), installTool(AGENT_DEVICE_SPEC, paths.agentDevice)], - { concurrency: 1 }, - ).pipe(Effect.as(paths)), - ); + return yield* installLock.withPermit(installTool(spec, select(paths))); }); -export const isDeviceToolchainInstalled = Effect.fn("DeviceToolchain.isInstalled")(function* ( +export const ensureDeviceHub = (baseDir: string) => + ensureTool(baseDir, HUB_SPEC, (paths) => paths.hub); + +export const ensureAgentDevice = (baseDir: string) => + ensureTool(baseDir, AGENT_DEVICE_SPEC, (paths) => paths.agentDevice); + +const isToolInstalled = Effect.fn("DeviceToolchain.isToolInstalled")(function* ( baseDir: string, + spec: ToolSpec, + select: (paths: DeviceToolchainPaths) => DeviceToolPaths, ) { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const paths = deviceToolchainPaths(path, baseDir); - const [hub, agentDevice] = yield* Effect.all([ - isInstalled(fs, paths.hub, DEVICE_HUB_VERSION), - isInstalled(fs, paths.agentDevice, AGENT_DEVICE_VERSION), - ]); - return hub && agentDevice; + return yield* isInstalled(fs, select(paths), spec.version); }); + +export const isDeviceHubInstalled = (baseDir: string) => + isToolInstalled(baseDir, HUB_SPEC, (paths) => paths.hub); + +export const isAgentDeviceInstalled = (baseDir: string) => + isToolInstalled(baseDir, AGENT_DEVICE_SPEC, (paths) => paths.agentDevice); diff --git a/apps/server/src/device/LocalDeviceHost.test.ts b/apps/server/src/device/LocalDeviceHost.test.ts new file mode 100644 index 000000000000..dc09bf1dec9c --- /dev/null +++ b/apps/server/src/device/LocalDeviceHost.test.ts @@ -0,0 +1,59 @@ +import { describe, expect, it } from "@effect/vitest"; +import * as NodePath from "@effect/platform-node/NodePath"; +import { HostProcessEnvironment, HostProcessPlatform } from "@t3tools/shared/hostProcess"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; + +import { __testing } from "./LocalDeviceHost.ts"; + +const diagnose = (files: ReadonlyArray, environment: NodeJS.ProcessEnv) => + __testing.platformReason("android").pipe( + Effect.provideService(HostProcessEnvironment, environment), + Effect.provideService(HostProcessPlatform, "darwin"), + Effect.provideService( + FileSystem.FileSystem, + FileSystem.makeNoop({ + exists: (file) => Effect.succeed(files.includes(file)), + }), + ), + Effect.provide(NodePath.layer), + ); + +describe("Android SDK availability", () => { + it.effect("explains that adb alone is insufficient to launch an emulator", () => + Effect.gen(function* () { + const reason = yield* diagnose(["/sdk/platform-tools/adb"], { ANDROID_HOME: "/sdk" }); + expect(reason).toContain("Android Emulator is missing"); + }), + ); + + it.effect("identifies command-line tools required by the device hub", () => + Effect.gen(function* () { + const reason = yield* diagnose(["/sdk/platform-tools/adb", "/sdk/emulator/emulator"], { + ANDROID_HOME: "/sdk", + }); + expect(reason).toContain("Command-line Tools (latest)"); + }), + ); + + it.effect("discovers the standard macOS SDK without ANDROID_HOME", () => + Effect.gen(function* () { + const root = "/test/home/Library/Android/sdk"; + const reason = yield* diagnose( + [ + `${root}/platform-tools/adb`, + `${root}/emulator/emulator`, + `${root}/cmdline-tools/latest/bin/avdmanager`, + ], + { HOME: "/test/home" }, + ); + expect(reason).toBeNull(); + }), + ); + + it.effect("reports an absent SDK without running or installing tools", () => + Effect.gen(function* () { + expect(yield* diagnose([], { HOME: "/test/home" })).toContain("Android SDK was not found"); + }), + ); +}); diff --git a/apps/server/src/device/LocalDeviceHost.ts b/apps/server/src/device/LocalDeviceHost.ts index 27791a2f0934..7b723d04ce8d 100644 --- a/apps/server/src/device/LocalDeviceHost.ts +++ b/apps/server/src/device/LocalDeviceHost.ts @@ -3,9 +3,8 @@ * * Runs expo-device-hub as a supervised child on a loopback port and starts the * agent-device daemon in HTTP mode under a T3-owned state directory. Both are - * lazy: nothing is installed or spawned until a device is first listed with - * the intent to open one, so a server that never touches simulators pays - * nothing. + * lazy: the device service requires explicit setup consent before it calls + * ensureReady to install tools or start helper processes. * * The hub runs in its standalone mode (origin root). The T3 proxy strips its * own prefix, and the Device panel derives stream and socket URLs from the @@ -41,15 +40,18 @@ import * as ProcessRunner from "../processRunner.ts"; import { type AgentDeviceEndpoint, type DeviceHost, + type DeviceHostAgentReady, DeviceHostError, type DeviceHostReady, type DeviceHubEndpoint, } from "./DeviceHost.ts"; import { agentDeviceStateDir, - type DeviceToolchainPaths, - ensureDeviceToolchain, - isDeviceToolchainInstalled, + type DeviceToolPaths, + ensureAgentDevice, + ensureDeviceHub, + isAgentDeviceInstalled, + isDeviceHubInstalled, } from "./DeviceToolchain.ts"; const HUB_READY_TIMEOUT_MS = 30_000; @@ -88,7 +90,7 @@ interface HubProcess { interface RunningHost { readonly hub: HubProcess; - readonly agentDevice: AgentDeviceEndpoint; + readonly agentDevice: AgentDeviceEndpoint | null; readonly helpers: DeviceHostReady["helpers"]; } @@ -96,19 +98,75 @@ const platformReason = Effect.fn("LocalDeviceHost.platformReason")(function* ( platform: DevicePlatform, ): Effect.fn.Return { const hostPlatform = yield* HostProcessPlatform; - const environment = yield* HostProcessEnvironment; if (platform === "ios") { if (hostPlatform !== "darwin") return "iOS Simulators need macOS with Xcode."; if (!(yield* isCommandAvailable("xcrun"))) return "Xcode command line tools were not found."; return null; } - const sdkRoot = environment.ANDROID_HOME?.trim() || environment.ANDROID_SDK_ROOT?.trim(); - if (!sdkRoot && !(yield* isCommandAvailable("adb"))) { - return "Android SDK was not found. Set ANDROID_HOME or put adb on PATH."; - } + const sdk = yield* androidSdk; + if (!sdk.root) + return "Android SDK was not found. Install it with Android Studio or set ANDROID_HOME to your SDK directory."; + if (!sdk.adb) + return `Android SDK Platform-Tools are missing from ${sdk.root}. Install them in Android Studio's SDK Manager.`; + if (!sdk.emulator) + return `Android Emulator is missing from ${sdk.root}. Install it in Android Studio's SDK Manager.`; + if (!sdk.avdmanager) + return `Android SDK Command-line Tools (latest) are missing from ${sdk.root}. Install them in Android Studio's SDK Manager.`; return null; }); +/** Resolve the SDK once for both diagnostics and the environment passed to helpers. */ +const androidSdk = Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const environment = yield* HostProcessEnvironment; + const platform = yield* HostProcessPlatform; + const home = environment.HOME ?? environment.USERPROFILE ?? ""; + const explicit = environment.ANDROID_HOME?.trim() || environment.ANDROID_SDK_ROOT?.trim(); + const candidates = explicit + ? [explicit] + : [ + path.join(home, "Library", "Android", "sdk"), + path.join(home, "Android", "Sdk"), + path.join( + environment.LOCALAPPDATA ?? path.join(home, "AppData", "Local"), + "Android", + "Sdk", + ), + ]; + if (!explicit) { + for (const directory of (environment.PATH ?? "").split(platform === "win32" ? ";" : ":")) { + if (!directory) continue; + const resolved = yield* fs + .realPath(path.join(directory, platform === "win32" ? "adb.exe" : "adb")) + .pipe(Effect.option); + if (resolved._tag === "Some") candidates.push(path.dirname(path.dirname(resolved.value))); + } + } + const exists = (file: string) => fs.exists(file).pipe(Effect.orElseSucceed(() => false)); + for (const root of candidates) { + const adb = yield* exists( + path.join(root, "platform-tools", platform === "win32" ? "adb.exe" : "adb"), + ); + const emulator = yield* exists( + path.join(root, "emulator", platform === "win32" ? "emulator.exe" : "emulator"), + ); + if (explicit || adb || emulator) { + const avdmanager = yield* exists( + path.join( + root, + "cmdline-tools", + "latest", + "bin", + platform === "win32" ? "avdmanager.bat" : "avdmanager", + ), + ); + return { root, adb, emulator, avdmanager }; + } + } + return { root: null, adb: false, emulator: false, avdmanager: false }; +}); + export const make = Effect.fn("LocalDeviceHost.make")(function* () { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const config = yield* ServerConfig.ServerConfig; @@ -117,7 +175,10 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () { const net = yield* NetService.NetService; const runner = yield* ProcessRunner.ProcessRunner; const httpClient = yield* HttpClient.HttpClient; - const hostEnvironment = yield* HostProcessEnvironment; + const environment = yield* HostProcessEnvironment; + const hostPlatform = yield* HostProcessPlatform; + const sdk = yield* androidSdk; + const hostEnvironment = sdk.root ? { ...environment, ANDROID_HOME: sdk.root } : environment; const startLock = yield* Semaphore.make(1); const runningRef = yield* Ref.make(null); const restartDelayRef = yield* Ref.make(0); @@ -134,11 +195,25 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () { }); const summary: Effect.Effect = Effect.gen(function* () { - const platforms = yield* Effect.all([ - platformAvailability("ios"), - platformAvailability("android"), + const [platforms, hubInstalled, agentDeviceInstalled] = yield* Effect.all([ + Effect.all([platformAvailability("ios"), platformAvailability("android")]), + isDeviceHubInstalled(config.baseDir).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Path.Path, path), + ), + isAgentDeviceInstalled(config.baseDir).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Path.Path, path), + ), ]); - return { id: hostId, kind: "local", label: "This machine", platforms }; + return { + id: hostId, + kind: "local", + label: "This machine", + platforms, + hubInstalled, + agentDeviceInstalled, + }; }); const hubEnvironment = (): NodeJS.ProcessEnv => ({ @@ -202,18 +277,18 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () { yield* fs.remove(hubStatePath(), { force: true }).pipe(Effect.ignore); }).pipe(Effect.catchCause(() => Effect.void)); - const recordHub = (hub: HubProcess, tools: DeviceToolchainPaths) => + const recordHub = (hub: HubProcess, hubTool: DeviceToolPaths) => encodeHubStateFile({ pid: Number(hub.child.pid), port: Number(new URL(hub.origin).port), - entryPath: tools.hub.entryPath, + entryPath: hubTool.entryPath, }).pipe( Effect.flatMap((json) => fs.writeFileString(hubStatePath(), json)), Effect.ignore, ); const spawnHub = Effect.fn("LocalDeviceHost.spawnHub")(function* ( - tools: DeviceToolchainPaths, + hubTool: DeviceToolPaths, ): Effect.fn.Return { yield* reapStaleHub; yield* fs @@ -237,7 +312,7 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () { ChildProcess.make( process.execPath, [ - tools.hub.entryPath, + hubTool.entryPath, "--port", String(port), "--host", @@ -284,7 +359,7 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () { Effect.provideService(HttpClient.HttpClient, httpClient), Effect.tapError(() => stopHub(hub)), ); - yield* recordHub(hub, tools); + yield* recordHub(hub, hubTool); yield* Effect.logInfo("Device hub started", { pid: Number(child.pid), port }); return hub; }); @@ -305,7 +380,7 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () { * Restart the hub when it dies under us, with the same doubling backoff the * relay connector uses so a hub that crashes on boot cannot spin. */ - const superviseHub = (hub: HubProcess, tools: DeviceToolchainPaths): Effect.Effect => + const superviseHub = (hub: HubProcess, hubTool: DeviceToolPaths): Effect.Effect => Effect.gen(function* () { yield* Effect.result(hub.child.exitCode); const running = yield* Ref.get(runningRef); @@ -325,9 +400,9 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () { Effect.gen(function* () { const current = yield* Ref.get(runningRef); if (current?.hub.child.pid !== hub.child.pid) return; - const replacement = yield* spawnHub(tools); + const replacement = yield* spawnHub(hubTool); yield* Ref.set(runningRef, { ...current, hub: replacement }); - yield* Effect.forkDetach(superviseHub(replacement, tools)); + yield* Effect.forkDetach(superviseHub(replacement, hubTool)); }), ); }).pipe( @@ -347,7 +422,7 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () { * daemon.json this reads back. */ const startAgentDeviceDaemon = Effect.fn("LocalDeviceHost.startAgentDeviceDaemon")(function* ( - tools: DeviceToolchainPaths, + agentTool: DeviceToolPaths, ): Effect.fn.Return { const stateDir = agentDeviceStateDir(path, config.stateDir); yield* fs.makeDirectory(stateDir, { recursive: true }).pipe(Effect.ignore); @@ -366,7 +441,7 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () { const toEndpoint = (file: typeof AgentDeviceDaemonFile.Type): AgentDeviceEndpoint => ({ baseUrl: `http://127.0.0.1:${file.httpPort}`, token: file.token, - entryPath: tools.agentDevice.entryPath, + entryPath: agentTool.entryPath, }); if (existing._tag === "Some") { const alive = yield* httpClient @@ -384,7 +459,7 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () { yield* runner .run({ command: process.execPath, - args: [tools.agentDevice.entryPath, "devices", "--json"], + args: [agentTool.entryPath, "devices", "--json"], env: daemonEnvironment, timeout: Duration.millis(DAEMON_READY_TIMEOUT_MS), timeoutBehavior: "timedOutResult", @@ -405,13 +480,13 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () { } }); - const stopAgentDeviceDaemon = (tools: DeviceToolchainPaths | null) => - tools + const stopAgentDeviceDaemon = (agentTool: DeviceToolPaths | null) => + agentTool ? runner .run({ command: process.execPath, args: [ - tools.agentDevice.entryPath, + agentTool.entryPath, "daemon", "stop", "--state-dir", @@ -424,23 +499,74 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () { .pipe(Effect.ignore) : Effect.void; - let toolsRef: DeviceToolchainPaths | null = null; + let agentToolRef: DeviceToolPaths | null = null; + + const ensureHubReady = Effect.fn("LocalDeviceHost.ensureHubReady")(function* ( + onPhase: (phase: "installing" | "starting") => Effect.Effect, + ): Effect.fn.Return { + const running = yield* Ref.get(runningRef); + if (running) { + const alive = yield* running.hub.child.isRunning.pipe(Effect.orElseSucceed(() => false)); + if (alive) return running; + yield* Ref.set(runningRef, null); + } + const installed = yield* isDeviceHubInstalled(config.baseDir).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Path.Path, path), + ); + if (!installed) yield* onPhase("installing"); + const hubTool = yield* ensureDeviceHub(config.baseDir).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Path.Path, path), + Effect.provideService(ProcessRunner.ProcessRunner, runner), + Effect.mapError( + (cause) => + new DeviceHostError({ + hostId, + step: `installing ${cause.tool}`, + detail: cause.message, + cause, + }), + ), + ); + yield* onPhase("starting"); + const hub = yield* spawnHub(hubTool); + const candidate = helperPaths(hubTool); + const [axExists, cliExists] = yield* Effect.all([ + fs.exists(candidate.serveSimAxSettings).pipe(Effect.orElseSucceed(() => false)), + fs.exists(candidate.serveSimCli).pipe(Effect.orElseSucceed(() => false)), + ]); + const next: RunningHost = { + hub, + agentDevice: null, + helpers: { + serveSimAxSettings: axExists ? candidate.serveSimAxSettings : null, + serveSimCli: cliExists ? candidate.serveSimCli : null, + }, + }; + yield* Ref.set(runningRef, next); + yield* Ref.set(restartDelayRef, 0); + yield* Effect.forkDetach(superviseHub(hub, hubTool)); + return next; + }); const ensureReady: DeviceHost["ensureReady"] = (onPhase) => + startLock.withPermits(1)(ensureHubReady(onPhase).pipe(Effect.map(toReady))); + + const ensureAgentReady: DeviceHost["ensureAgentReady"] = (onPhase) => startLock.withPermits(1)( - Effect.gen(function* (): Generator, DeviceHostReady> { - const running = yield* Ref.get(runningRef); - if (running) { - const alive = yield* running.hub.child.isRunning.pipe(Effect.orElseSucceed(() => false)); - if (alive) return toReady(running); - yield* Ref.set(runningRef, null); - } - const installed = yield* isDeviceToolchainInstalled(config.baseDir).pipe( + Effect.gen(function* (): Generator< + Effect.Effect, + DeviceHostAgentReady + > { + const running = yield* ensureHubReady(onPhase); + if (running.agentDevice) return { ...toReady(running), agentDevice: running.agentDevice }; + const installed = yield* isAgentDeviceInstalled(config.baseDir).pipe( Effect.provideService(FileSystem.FileSystem, fs), Effect.provideService(Path.Path, path), ); if (!installed) yield* onPhase("installing"); - const tools = yield* ensureDeviceToolchain(config.baseDir).pipe( + const agentTool = yield* ensureAgentDevice(config.baseDir).pipe( Effect.provideService(FileSystem.FileSystem, fs), Effect.provideService(Path.Path, path), Effect.provideService(ProcessRunner.ProcessRunner, runner), @@ -454,35 +580,18 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () { }), ), ); - toolsRef = tools; + agentToolRef = agentTool; yield* onPhase("starting"); - const hub = yield* spawnHub(tools); - const agentDevice = yield* startAgentDeviceDaemon(tools).pipe( - Effect.tapError(() => stopHub(hub)), - ); - const candidate = helperPaths(tools); - const [axExists, cliExists] = yield* Effect.all([ - fs.exists(candidate.serveSimAxSettings).pipe(Effect.orElseSucceed(() => false)), - fs.exists(candidate.serveSimCli).pipe(Effect.orElseSucceed(() => false)), - ]); - const next: RunningHost = { - hub, - agentDevice, - helpers: { - serveSimAxSettings: axExists ? candidate.serveSimAxSettings : null, - serveSimCli: cliExists ? candidate.serveSimCli : null, - }, - }; + const agentDevice = yield* startAgentDeviceDaemon(agentTool); + const next = { ...running, agentDevice }; yield* Ref.set(runningRef, next); - yield* Ref.set(restartDelayRef, 0); - yield* Effect.forkDetach(superviseHub(hub, tools)); - return toReady(next); + return { ...toReady(next), agentDevice }; }), ); - const helperPaths = (tools: DeviceToolchainPaths) => { + const helperPaths = (hubTool: DeviceToolPaths) => { const serveSimDist = path.join( - tools.hub.installDir, + hubTool.installDir, "node_modules", "expo-device-hub", "vendor", @@ -498,7 +607,14 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () { const run: DeviceHostReady["run"] = (command, args, options) => runner .run({ - command, + command: + command === "emulator" && sdk.root + ? path.join( + sdk.root, + "emulator", + hostPlatform === "win32" ? "emulator.exe" : "emulator", + ) + : command, args, env: hostEnvironment, timeout: Duration.millis(options?.timeoutMs ?? 20_000), @@ -516,7 +632,6 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () { const toReady = (running: RunningHost): DeviceHostReady => ({ hub: { origin: running.hub.origin } satisfies DeviceHubEndpoint, - agentDevice: running.agentDevice, run, helpers: running.helpers, }); @@ -525,12 +640,21 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () { Effect.map((running) => (running ? toReady(running) : null)), ); + const stopAgent: DeviceHost["stopAgent"] = startLock.withPermits(1)( + Effect.gen(function* () { + yield* stopAgentDeviceDaemon(agentToolRef); + yield* Ref.update(runningRef, (running) => + running ? { ...running, agentDevice: null } : running, + ); + }), + ); + const stop: DeviceHost["stop"] = startLock.withPermits(1)( Effect.gen(function* () { const running = yield* Ref.getAndSet(runningRef, null); yield* stopHub(running?.hub); yield* fs.remove(hubStatePath(), { force: true }).pipe(Effect.ignore); - yield* stopAgentDeviceDaemon(toolsRef); + yield* stopAgentDeviceDaemon(agentToolRef); }), ); @@ -542,8 +666,13 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () { summary, platformAvailability, ensureReady, + ensureAgentReady, current, + stopAgent, stop, }; return host; }); + +/** Exposed for tests. */ +export const __testing = { AgentDeviceDaemonFile, androidSdk, platformReason }; diff --git a/apps/server/src/mcp/McpDeviceToolkit.test.ts b/apps/server/src/mcp/McpDeviceToolkit.test.ts index a4e99b639a63..083ac0c034fe 100644 --- a/apps/server/src/mcp/McpDeviceToolkit.test.ts +++ b/apps/server/src/mcp/McpDeviceToolkit.test.ts @@ -51,11 +51,15 @@ const state = { { platform: "ios" as const, available: true }, { platform: "android" as const, available: false, reason: "No SDK" }, ], + hubInstalled: true, + agentDeviceInstalled: true, }, ], hostStatus: "ready" as const, devices: [device], sessions: [], + onboardingCompleted: true, + agentAccessEnabled: true, hubBasePath: "/api/device-hub", revision: 1, }; diff --git a/apps/server/src/mcp/toolkits/device/handlers.ts b/apps/server/src/mcp/toolkits/device/handlers.ts index fa4d2b90fde3..bab7391af24d 100644 --- a/apps/server/src/mcp/toolkits/device/handlers.ts +++ b/apps/server/src/mcp/toolkits/device/handlers.ts @@ -106,6 +106,12 @@ const handlers = { const scope = yield* requireDeviceAccess; const devices = yield* DeviceService.DeviceService; const state = yield* devices.list; + if (state.hostStatus === "disabled") { + return yield* new DeviceToolUnavailableError({ + reason: + "Device support is off. Ask the user to enable it in the Device panel before installing or starting device tools.", + }); + } const hostId = input?.hostId; const open = state.sessions .filter((session) => session.threadId === scope.threadId) @@ -123,6 +129,12 @@ const handlers = { const scope = yield* requireDeviceAccess; const devices = yield* DeviceService.DeviceService; const state = yield* devices.list; + if (state.hostStatus === "disabled") { + return yield* new DeviceToolUnavailableError({ + reason: + "Device support is off. Ask the user to enable it in the Device panel before installing or starting device tools.", + }); + } const target = yield* pickDevice(state.devices, input); const session = yield* devices.open({ threadId: scope.threadId, diff --git a/apps/server/src/provider/Layers/ProviderService.ts b/apps/server/src/provider/Layers/ProviderService.ts index 58fa02cfbbc4..310e371c45aa 100644 --- a/apps/server/src/provider/Layers/ProviderService.ts +++ b/apps/server/src/provider/Layers/ProviderService.ts @@ -257,7 +257,10 @@ export interface ProviderServiceLiveOptions { /** Same seam as `issueMcpCredential`, for observing the deny path's revoke. */ readonly revokeMcpCredential?: typeof McpSessionRegistry.revokeActiveMcpThread; /** Overrides the device host lookup used to build the agent-device environment. */ - readonly deviceReadiness?: () => Effect.Effect; + readonly deviceReadiness?: () => Effect.Effect< + DeviceService.DeviceAgentReadiness | null, + unknown + >; } interface TurnAnalyticsMetadata { @@ -493,7 +496,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( (() => Effect.serviceOption(DeviceService.DeviceService).pipe( Effect.flatMap((service) => - Option.isSome(service) ? service.value.readinessIfSupported() : Effect.succeed(null), + Option.isSome(service) ? service.value.agentReadinessIfSupported() : Effect.succeed(null), ), )); const fileSystem = yield* FileSystem.FileSystem; @@ -915,14 +918,14 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( ), ); - const agentAccessCapabilities = Effect.fn("ProviderService.agentAccessCapabilities")( - function* (threadId: ThreadId) { - const capabilities = new Set(); - if (yield* agentBrowserAccessEnabled(threadId)) capabilities.add("preview"); - if (yield* agentDeviceAccessEnabled) capabilities.add("device"); - return capabilities; - }, - ); + const agentAccessCapabilities = Effect.fn("ProviderService.agentAccessCapabilities")(function* ( + threadId: ThreadId, + ) { + const capabilities = new Set(); + if (yield* agentBrowserAccessEnabled(threadId)) capabilities.add("preview"); + if (yield* agentDeviceAccessEnabled) capabilities.add("device"); + return capabilities; + }); /** * Starting a session with device access also brings the device host up, so diff --git a/apps/server/src/server.test.ts b/apps/server/src/server.test.ts index a45a5ff0039b..41d380410516 100644 --- a/apps/server/src/server.test.ts +++ b/apps/server/src/server.test.ts @@ -1664,6 +1664,8 @@ const EMPTY_DEVICE_STATE: DeviceServiceState = { hostStatus: "idle", devices: [], sessions: [], + onboardingCompleted: false, + agentAccessEnabled: false, hubBasePath: DeviceService.DEVICE_HUB_ROUTE_PREFIX, revision: 0, }; diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 0a2b7bacef81..8a083b56d4d6 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -385,6 +385,7 @@ const PreviewLayerLive = Layer.empty.pipe( ); const DeviceLayerLive = DeviceService.layer.pipe( + Layer.provide(ServerSettingsLayerLive), Layer.provide(ProcessRunner.layer), Layer.provide(NetService.layer), ); diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index b39dda102ddd..a7730e832ddc 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -2687,6 +2687,10 @@ const makeWsRpcLayer = ( observeRpcStream(WS_METHODS.subscribePreviewEvents, previewManager.events, { "rpc.aggregate": "preview", }), + [WS_METHODS.deviceConfigure]: (input) => + observeRpcEffect(WS_METHODS.deviceConfigure, deviceService.configure(input), { + "rpc.aggregate": "device", + }), [WS_METHODS.deviceList]: (_input) => observeRpcEffect(WS_METHODS.deviceList, deviceService.list, { "rpc.aggregate": "device", diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index f56e5af3d3d2..78e3478ab529 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -204,6 +204,8 @@ import { PullRequestsUnavailableState } from "./pullRequest/PullRequestsUnavaila import { RightPanelTabs } from "./RightPanelTabs"; import { AgentsPanel } from "./AgentsPanel"; import { useDeviceState } from "~/state/device"; +import { DeviceSetup } from "./device/DeviceSetup"; +import { Dialog, DialogPopup } from "./ui/dialog"; import { deriveAgentPanelModel, foldSubagentActivities, @@ -4131,17 +4133,25 @@ export default function ChatView(props: ChatViewProps) { if (!activeThreadRef) return; useRightPanelStore.getState().open(activeThreadRef, "agents"); }, [activeThreadRef]); + const { state: deviceState } = useDeviceState(activeThreadRef?.environmentId ?? null); + const [deviceSetupThread, setDeviceSetupThread] = useState(null); const addDeviceSurface = useCallback(() => { if (!activeThreadRef) return; + if (!deviceState.onboardingCompleted || deviceState.hostStatus === "disabled") { + setDeviceSetupThread(activeThreadRef); + return; + } useRightPanelStore.getState().open(activeThreadRef, "device"); - }, [activeThreadRef]); + }, [activeThreadRef, deviceState.onboardingCompleted, deviceState.hostStatus]); // An agent's `device_open` surfaces in every client the same way a - // `preview_open` does: the thread gains a device session and the panel + // `preview_open` does: the thread starts a device or gains a session and the panel // opens on it. Closing the last session leaves the tab in place so the // user keeps their picker; only new sessions raise the panel. - const { state: deviceState } = useDeviceState(activeThreadRef?.environmentId ?? null); const threadDeviceSessionCount = activeThreadRef - ? deviceState.sessions.filter((session) => session.threadId === activeThreadRef.threadId).length + ? deviceState.sessions.filter((session) => session.threadId === activeThreadRef.threadId) + .length + + (deviceState.bootingDevices?.filter((device) => device.threadId === activeThreadRef.threadId) + .length ?? 0) : 0; const previousDeviceSessionCount = useRef(threadDeviceSessionCount); useEffect(() => { @@ -8100,6 +8110,10 @@ export default function ChatView(props: ChatViewProps) { threadRef={activeThreadRef} deviceId={null} visible={rightPanelOpen} + onDismissSetup={() => { + closeRightPanelSurface(renderedRightPanelSurface); + useRightPanelStore.getState().show(activeThreadRef); + }} /> ) : (renderedRightPanelSurface?.kind === "files" || @@ -8157,6 +8171,29 @@ export default function ChatView(props: ChatViewProps) { return (
+ { + if (!open) setDeviceSetupThread(null); + }} + > + + {activeThreadRef ? ( + { + useRightPanelStore.getState().open(activeThreadRef, "device"); + setDeviceSetupThread(null); + }} + /> + ) : null} + + {rightPanelControlsAtRoot ? panelLayoutControls : null}
m?@&GS|D&9&1m=9s)>2ejqFY=?l8aYKo=UWZGu#;Tx0usor%@n&!2E2 z3vqzma~(}Ams_^yU)Q={Y_HL;w>9-q1n(#(7Ho?WHFeJu`z}L4hS~<~+q%|EJlUQ& zUAHaI2fmr?)IEb~_{L23Wy)MkC})aNslY9VS-RB(Pb6hAIyp35)$%=Xn9Bp`wgtWo zI{8x#18f)QgAUtb23g=lTv>!yUxgLJRyC&a&#|bL`1aZ0uM#+}?Re#V-P0+d(wJTE z_4o%ier6XgKw%|tW?Or<>S`2f%QhW@`NtENW$1|TE+OMuP!JP<^`Nx<8Guay);eP+ z>kj|CXK{eHTB_A#wN63z_*9fCaL&&Mz>{1F{|j7=*S8>F8Tjmm?l%TY{hS2dpl+z1 zcT+W4O)}NWp>m)cQq`!g4Z}WATDqn&OB(LwCmIqEe|v6uD?q%~5m z2`oAeT!!5p!MwrU;LF`zytVUWc=Ao9J-_RQRCme2HB6 zZSNZMRo$QfVOa?VP2V3N3gFrXA;|Z&J!xcp1m1&|?laF(>r8Q6Mp|Go*t1=YxyoJ~ zoI|Aw&XMvozMef35r@4OKq~I^XdYojDTxYY5dtgIkprXxnsmnRK-pkc({I5BtQ|YZ zc={nHW;rRs495~IE>S)%zkBd_6jOCDxt6}c!IsNkwsor@l|Yiaf4cue_*_f(pstca zsCOBtVCq(zF6#iV|9zg`Q__znvja{ZDQcS(h5)df*M70 zOD*9$`J7n$FrUC%%TfG&-!@XHaEjwv;kFp>7tV|E?ZWe7{7d1C80E}mF>c7G@GF_T z`2Mb372}68i1Dv-L5!)@z~>9ApBM9AWlqPrR&z}%OX6PqYc+}QS zw&g1()AY6};fXnEqE!_sk-I5Dx^Q-i#Brpfi)HTp)?5`IuEvt%fAbFiATuY>oIjhu z@6E;V;@KowPEQkZI{pIV%JMuLh+m@nQgP2oSX3p{C2Sz0>{zV4Gug|bOk z6qDyuJX?G?KlPGVcXfvsrPdifD5w+gp5BFDK7xUg4g^=)Zd?38Ad&V+JHG7@HKe}A za$@u(oCMc3BSU4=wg=5+B$HH+3YDzdj$i|j-9Nn@=7)G>S zRJb}?k4NdO02;D1W}oHX5`>lbzb6eL5;gvNc9CzJ0C_+<-phD}Pm$MViM}(j1aGj9 zQJ55Ny%O9wd?TF_rtjt^pX_%Jd{Bbwv8(==EcV|*0Tf=@vrR@qzk=Uinoi5qG#@v| zhs8{aTNMi7#TG9a{G>RCKUj)IWFZ`X$Y-Vlp*ZverElM~;i{#%yo&p}>$g>-kG}^E zek;nbMejJDflUomw+6;Lep}P+yaz$m;j^lxHkn%@uu>=e7PG)*4VQVXGQO2d;lEcC zT;k@%xrcAQ72fq zNf44HQh?O0lMwJRh%pg&AdIBX!$AQ+Ct)BEWR!Y73J`ph224hYfHagm(3z>2q$j`z z7)oGbDClRQNp5ky6em$EtZgp%R)FeN@xxhlG?z1!ti**?bRjlxg2Y`ps}i^-9h)s$v1+tx zRp^mhr?J(bgUe-lYc10c%cgmIjYjqoy|BCV&AvpV+P&+z8%O3;)%4RXU}0~%)vRb~ zRc+_9IL1&w6l#6iY}IJL<>#83=R&M~#rEs>t22cfk%F3@|YK`ZSp4cn{wtH!AeH RQ>J>eN=LU!>QCox_8*kw%tinJ diff --git a/apps/web/src/components/device/DeviceSetup.tsx b/apps/web/src/components/device/DeviceSetup.tsx new file mode 100644 index 000000000000..428166e1daa5 --- /dev/null +++ b/apps/web/src/components/device/DeviceSetup.tsx @@ -0,0 +1,314 @@ +import type { DevicePlatform, DeviceServiceState, EnvironmentId } from "@t3tools/contracts"; +import { Check, CircleAlert } from "lucide-react"; +import { useState } from "react"; + +import { Button } from "~/components/ui/button"; +import { + DialogClose, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "~/components/ui/dialog"; +import { WizardPanel, WizardSteps } from "~/components/ui/wizard"; +import { Spinner } from "~/components/ui/spinner"; +import { Switch } from "~/components/ui/switch"; +import { deviceEnvironment } from "~/state/device"; +import { useAtomCommand } from "~/state/use-atom-command"; +import { cn } from "~/lib/utils"; + +const platformName = (platform: DevicePlatform) => (platform === "ios" ? "iOS" : "Android"); + +export const deviceHubDescription = + "Open simulators and emulators. Required device support is set up automatically."; +export const agentDeviceDescription = + "Allow agents to start and control devices. Required tools are set up automatically and available to new agent sessions."; + +export function platformSetupStatus(state: DeviceServiceState, platform: DevicePlatform) { + const availability = state.hosts + .flatMap((host) => host.platforms) + .find((candidate) => candidate.platform === platform); + if (!availability?.available) { + return { + ready: false, + message: availability?.reason ?? `${platformName(platform)} support was not detected.`, + }; + } + if ( + state.hostStatus === "ready" && + !state.devices.some((device) => device.platform === platform) + ) { + return { + ready: false, + message: + platform === "ios" + ? "Xcode is installed, but no iOS Simulator is available. Install a runtime in Xcode Settings → Components." + : "The Android SDK is installed, but no virtual device exists. Create one in Android Studio → Device Manager.", + }; + } + return { + ready: true, + message: + platform === "ios" + ? "Xcode and iOS Simulator are available." + : "The Android SDK and Emulator are available.", + }; +} + +export function DeviceSetup(props: { + readonly environmentId: EnvironmentId; + readonly state: DeviceServiceState; + readonly onComplete?: () => void; +}) { + const configure = useAtomCommand(deviceEnvironment.configure); + const list = useAtomCommand(deviceEnvironment.list, { reportFailure: false }); + const [pending, setPending] = useState<"hub" | "check" | "agent" | "complete" | null>(null); + const [step, setStep] = useState(0); + const enabled = props.state.hostStatus !== "disabled"; + const busy = props.state.hostStatus === "installing" || props.state.hostStatus === "starting"; + + const update = async ( + kind: NonNullable, + input: { enabled?: boolean; agentAccessEnabled?: boolean; onboardingCompleted?: boolean }, + ) => { + setPending(kind); + try { + const result = await configure({ environmentId: props.environmentId, input }); + if (kind === "complete" && result._tag === "Success") props.onComplete?.(); + } finally { + setPending(null); + } + }; + + return ( +
+ + Set up devices + + Review what runs on this environment before using simulators and emulators. + + busy || pending !== null || requested > step} + /> + + + + {step === 0 ? ( +
+

Enable the device hub

+
+

{deviceHubDescription}

+ + void update("hub", { + enabled: Boolean(checked), + ...(checked ? {} : { agentAccessEnabled: false }), + }) + } + /> +
+ +
+ ) : null} + + {step === 1 ? ( +
+

Check simulator support

+ { + setPending("check"); + void list({ environmentId: props.environmentId, input: {} }).finally(() => + setPending(null), + ); + }} + /> +
+ ) : null} + + {step === 2 ? ( +
+

Allow agent control

+
+

{agentDeviceDescription}

+ + void update("agent", { agentAccessEnabled: Boolean(checked) }) + } + /> +
+ +

+ Leave this off to keep manual device controls without giving agents access. +

+
+ ) : null} + {props.state.hostStatus === "failed" && props.state.hostStatusDetail ? ( +

+ {props.state.hostStatusDetail} +

+ ) : null} + + + + {step === 0 ? ( + }>Cancel + ) : ( + + )} + {step < 2 ? ( + + ) : ( + + )} + +
+ ); +} + +export function DeviceHubSetupStatus({ + state, + pending, + compact = false, +}: { + readonly state: DeviceServiceState; + readonly pending: boolean; + readonly compact?: boolean; +}) { + if (!pending && state.hostStatus !== "ready") return null; + return ( +

+ {pending ? : } + {pending + ? state.hostStatus === "installing" + ? compact + ? "Installing…" + : "Installing device hub…" + : state.hostStatus === "starting" + ? compact + ? "Starting…" + : "Starting device hub…" + : compact + ? "Updating…" + : "Updating device hub…" + : "Device hub is ready."} +

+ ); +} + +export function DevicePlatformSetup(props: { + readonly state: DeviceServiceState; + readonly checking: boolean; + readonly disabled: boolean; + readonly onCheck: () => void; +}) { + return ( +
+ + +

+ You can use either platform. Fixing a missing platform does not block the other one. +

+ +
+ ); +} + +export function AgentDeviceSetupStatus(props: { + readonly state: DeviceServiceState; + readonly pending: boolean; + readonly compact?: boolean; +}) { + if (props.pending) { + const label = + props.state.hostStatus === "installing" + ? props.compact + ? "Installing…" + : "Installing agent tools…" + : props.state.hostStatus === "starting" + ? props.compact + ? "Starting…" + : "Starting agent tools…" + : props.compact + ? "Updating…" + : "Updating agent access…"; + return ( +

+ + {label} +

+ ); + } + if ( + props.state.agentAccessEnabled && + props.state.hostStatus === "ready" && + props.state.hosts.some((host) => host.agentDeviceInstalled) + ) { + return ( +

+ + Agent tools are ready. +

+ ); + } + return null; +} + +export function PlatformStatus(props: { + readonly platform: string; + readonly status: { readonly ready: boolean; readonly message: string }; + readonly compact?: boolean; +}) { + const Icon = props.status.ready ? Check : CircleAlert; + return ( +
+ +
+

{props.platform}

+

+ {props.compact && props.status.ready ? "Ready" : props.status.message} +

+
+
+ ); +} diff --git a/apps/web/src/components/preview/PreviewEmptyState.tsx b/apps/web/src/components/preview/PreviewEmptyState.tsx index 163849154000..35b4b2f13d64 100644 --- a/apps/web/src/components/preview/PreviewEmptyState.tsx +++ b/apps/web/src/components/preview/PreviewEmptyState.tsx @@ -3,6 +3,7 @@ import { Globe, History, RadioTower } from "lucide-react"; import type { BrowserHistoryEntry } from "~/browserHistoryStore"; import { Empty, EmptyDescription, EmptyMedia, EmptyTitle } from "~/components/ui/empty"; +import { DiscoveryList } from "../ui/discovery-list"; import { PreviewLocalServerCard } from "./PreviewLocalServerCard"; import { PreviewRecentUrlCard } from "./PreviewRecentUrlCard"; @@ -55,7 +56,7 @@ export function PreviewEmptyState({

Recently used

-
+ {recents.map((entry) => ( onRemoveRecent(entry.url)} /> ))} -
+
) : null} {servers.length > 0 ? ( @@ -74,7 +75,7 @@ export function PreviewEmptyState({

Local servers

-
+ {servers.map((server) => ( onOpenUrl(server.requestedUrl)} /> ))} -
+

Select a live local server to open it in this browser tab.

diff --git a/apps/web/src/components/preview/PreviewLocalServerCard.tsx b/apps/web/src/components/preview/PreviewLocalServerCard.tsx index 263cdb294f48..198d0495473c 100644 --- a/apps/web/src/components/preview/PreviewLocalServerCard.tsx +++ b/apps/web/src/components/preview/PreviewLocalServerCard.tsx @@ -1,4 +1,5 @@ import type { ScopedThreadRef } from "@t3tools/contracts"; +import { DiscoveryListRow } from "../ui/discovery-list"; import { PreviewFaviconIcon } from "./PreviewFaviconIcon"; import type { PreviewableServer } from "./useDiscoveredLocalServers"; @@ -12,19 +13,12 @@ interface Props { export function PreviewLocalServerCard({ threadRef, server, onOpen }: Props) { const subtitle = describeServer(server); return ( - + icon={} + title={subtitle} + description={`${server.host}:${server.port}`} + /> ); } diff --git a/apps/web/src/components/settings/IntegrationsSettings.test.tsx b/apps/web/src/components/settings/IntegrationsSettings.test.tsx index 5d185fee5824..9ce3b2126c0e 100644 --- a/apps/web/src/components/settings/IntegrationsSettings.test.tsx +++ b/apps/web/src/components/settings/IntegrationsSettings.test.tsx @@ -1,4 +1,8 @@ -import { DEFAULT_CLIENT_SETTINGS, DEFAULT_UNIFIED_SETTINGS } from "@t3tools/contracts"; +import { + DEFAULT_CLIENT_SETTINGS, + DEFAULT_UNIFIED_SETTINGS, + type DeviceServiceState, +} from "@t3tools/contracts"; import { createMemoryHistory, createRootRoute, @@ -36,6 +40,7 @@ vi.mock("./settingsLayout", async (importOriginal) => ({ })); import { IntegrationsSettingsPanel } from "./IntegrationsSettings"; +import { platformSetupStatus } from "../device/DeviceSetup"; let renderer: ReactTestRenderer | undefined; @@ -74,4 +79,59 @@ describe("Integrations browser discovery", () => { await openSettings(); expect(listBrowserImportSources).not.toHaveBeenCalled(); }); + + it("places device settings directly after browser settings", async () => { + await openSettings(); + const sections = renderer!.root + .findAll((node) => node.type === "section") + .map((node) => node.props.id) + .filter(Boolean); + expect(sections.indexOf("devices")).toBeGreaterThan(sections.indexOf("browser")); + }); +}); + +const deviceState = (overrides: Partial = {}): DeviceServiceState => ({ + hosts: [ + { + id: "local", + kind: "local", + label: "This machine", + hubInstalled: false, + agentDeviceInstalled: false, + platforms: [ + { platform: "ios", available: true }, + { platform: "android", available: true }, + ], + }, + ], + hostStatus: "ready", + devices: [], + sessions: [], + onboardingCompleted: false, + agentAccessEnabled: false, + hubBasePath: "/api/device-hub", + revision: 0, + ...overrides, +}); + +describe("device setup guidance", () => { + it("directs users to install an iOS runtime and create an Android virtual device", () => { + expect(platformSetupStatus(deviceState(), "ios").message).toContain("Xcode Settings"); + expect(platformSetupStatus(deviceState(), "android").message).toContain("Device Manager"); + }); + + it("preserves a specific missing-tool explanation from the server", () => { + const state = deviceState({ + hosts: [ + { + ...deviceState().hosts[0]!, + platforms: [ + { platform: "ios", available: true }, + { platform: "android", available: false, reason: "Android Emulator is missing." }, + ], + }, + ], + }); + expect(platformSetupStatus(state, "android").message).toBe("Android Emulator is missing."); + }); }); diff --git a/apps/web/src/components/settings/IntegrationsSettings.tsx b/apps/web/src/components/settings/IntegrationsSettings.tsx index a7a3c7553540..a3f76f49cd5f 100644 --- a/apps/web/src/components/settings/IntegrationsSettings.tsx +++ b/apps/web/src/components/settings/IntegrationsSettings.tsx @@ -39,10 +39,21 @@ import { MoreVertical, Plus as PlusIcon } from "lucide-react"; import { useCallback, useRef, useState } from "react"; import { ScreenRotationIcon } from "~/browser/ScreenRotationIcon"; +import { AnimatedHeight } from "~/components/AnimatedHeight"; import { resolveEnvironmentOptionLabel } from "~/components/BranchToolbar.logic"; import { previewBridge } from "~/components/preview/previewBridge"; import { cn, randomUUID } from "~/lib/utils"; import { useEnvironments, usePrimaryEnvironment } from "~/state/environments"; +import { deviceEnvironment, useDeviceState } from "~/state/device"; +import { useAtomCommand } from "~/state/use-atom-command"; +import { + AgentDeviceSetupStatus, + DeviceHubSetupStatus, + PlatformStatus, + platformSetupStatus, + deviceHubDescription, + agentDeviceDescription, +} from "~/components/device/DeviceSetup"; import { isElectron } from "../../env"; import { Badge } from "../ui/badge"; @@ -571,6 +582,117 @@ function AgentBrowserAccessSetting() { ); } +function DeviceIntegrationSettings() { + const primaryEnvironment = usePrimaryEnvironment(); + const environmentId = primaryEnvironment?.environmentId ?? null; + const { state, loaded } = useDeviceState(environmentId); + const configure = useAtomCommand(deviceEnvironment.configure); + const list = useAtomCommand(deviceEnvironment.list, { reportFailure: false }); + const [pending, setPending] = useState<"hub" | "check" | "agent" | null>(null); + const enabled = state.hostStatus !== "disabled"; + const busy = state.hostStatus === "installing" || state.hostStatus === "starting"; + const [platformsRevealed, setPlatformsRevealed] = useState(false); + // Keep diagnostics visible through subsequent agent setup and refresh phases. + if (platformsRevealed && !enabled) setPlatformsRevealed(false); + if (!platformsRevealed && state.hostStatus === "ready" && pending !== "hub") { + setPlatformsRevealed(true); + } + + const update = async ( + kind: NonNullable, + input: { enabled?: boolean; agentAccessEnabled?: boolean }, + ) => { + if (!environmentId) return; + setPending(kind); + try { + const result = await configure({ environmentId, input }); + if (result._tag === "Success" && input.enabled === true && !state.onboardingCompleted) { + await configure({ environmentId, input: { onboardingCompleted: true } }); + } + } finally { + setPending(null); + } + }; + + return ( + + + {pending === "hub" ? : null} + + void update("hub", { + enabled: Boolean(checked), + ...(checked ? {} : { agentAccessEnabled: false }), + }) + } + /> + + } + /> + + {platformsRevealed ? ( + + + +
+ } + control={ + + } + /> + ) : null} + + + {pending === "agent" ? : null} + + void update("agent", { agentAccessEnabled: Boolean(checked) }) + } + /> + + } + /> + {state.hostStatus === "failed" && state.hostStatusDetail ? ( +

+ {state.hostStatusDetail} +

+ ) : null} + + ); +} + function BrowserAutoShowFloatingPreviewSetting({ disabled }: { readonly disabled: boolean }) { const autoShow = useClientSettings((settings) => settings.browserAutoShowFloatingPreview); const updateSettings = useUpdatePrimarySettings(); @@ -1170,6 +1292,7 @@ export function IntegrationsSettingsPanel() { previewDefaults )} + ); } diff --git a/apps/web/src/components/settings/ProjectDefaultsSettings.tsx b/apps/web/src/components/settings/ProjectDefaultsSettings.tsx index 8de434d837e8..938000e01002 100644 --- a/apps/web/src/components/settings/ProjectDefaultsSettings.tsx +++ b/apps/web/src/components/settings/ProjectDefaultsSettings.tsx @@ -96,11 +96,6 @@ export function ProjectDefaultsSettings({ target.serverConfig?.settings.enableAgentBrowserAccess !== serverSettings.enableAgentBrowserAccess, ); - const mixedDevice = targets.some( - (target) => - target.serverConfig?.settings.enableAgentDeviceAccess !== - serverSettings.enableAgentDeviceAccess, - ); const disabled = (key: keyof ServerSettingsPatch) => targets.length === 0 || saving.has(key); const mixedAutoPull = targets.some( (target) => target.serverConfig?.settings.defaultAutoPull !== serverSettings.defaultAutoPull, @@ -393,58 +388,6 @@ export function ProjectDefaultsSettings({ } /> - - void save({ - enableAgentDeviceAccess: DEFAULT_SERVER_SETTINGS.enableAgentDeviceAccess, - }) - } - /> - ) : null - } - control={ - - } - /> void) { ...(settings.enableAgentBrowserAccess !== DEFAULT_UNIFIED_SETTINGS.enableAgentBrowserAccess ? ["Agent browser access"] : []), - ...(settings.enableAgentDeviceAccess !== DEFAULT_UNIFIED_SETTINGS.enableAgentDeviceAccess - ? ["Agent device access"] - : []), ], [ isTextGenerationModelDirty, @@ -601,7 +598,6 @@ export function useSettingsRestore(onRestored?: () => void) { settings.browserAutoShowFloatingPreview, settings.appearanceContrast, settings.enableAgentBrowserAccess, - settings.enableAgentDeviceAccess, settings.confirmQuit, settings.confirmThreadArchive, settings.confirmThreadDelete, @@ -753,7 +749,6 @@ export function useSettingsRestore(onRestored?: () => void) { // name, so a user restoring defaults is told the agent regains access // rather than discovering it later. enableAgentBrowserAccess: DEFAULT_UNIFIED_SETTINGS.enableAgentBrowserAccess, - enableAgentDeviceAccess: DEFAULT_UNIFIED_SETTINGS.enableAgentDeviceAccess, }); onRestored?.(); }, [ diff --git a/apps/web/src/components/settings/settingsSearch.ts b/apps/web/src/components/settings/settingsSearch.ts index 0fe040460832..655f1758c113 100644 --- a/apps/web/src/components/settings/settingsSearch.ts +++ b/apps/web/src/components/settings/settingsSearch.ts @@ -399,9 +399,24 @@ export const SETTINGS_SEARCH_ITEMS = [ { id: "agent-device-access", title: "Agent device access", - to: "/settings/projects", + to: "/settings/integrations", + targetId: "devices", searchTerms: ["allow simulator emulator ios android drive tools sessions"], }, + { + id: "device-hub", + title: "Device hub", + to: "/settings/integrations", + targetId: "devices", + searchTerms: ["simulator emulator ios android install start"], + }, + { + id: "device-platform-support", + title: "Simulator support", + to: "/settings/integrations", + targetId: "devices", + searchTerms: ["xcode android studio sdk avd runtime"], + }, { id: "browser-profiles", title: "Browser profiles", diff --git a/apps/web/src/components/ui/discovery-list.tsx b/apps/web/src/components/ui/discovery-list.tsx new file mode 100644 index 000000000000..d03ca0edd5e0 --- /dev/null +++ b/apps/web/src/components/ui/discovery-list.tsx @@ -0,0 +1,37 @@ +import type { ComponentProps, ReactNode } from "react"; + +export function DiscoveryList({ children }: { readonly children: ReactNode }) { + return ( +
+ {children} +
+ ); +} + +export function DiscoveryListRow({ + icon, + title, + description, + action, + ...props +}: Omit, "title" | "children" | "className"> & { + readonly icon: ReactNode; + readonly title: ReactNode; + readonly description: ReactNode; + readonly action?: ReactNode; +}) { + return ( + + ); +} diff --git a/apps/web/src/state/device.ts b/apps/web/src/state/device.ts index 1c9bcd1aa993..b9c1adf86cdd 100644 --- a/apps/web/src/state/device.ts +++ b/apps/web/src/state/device.ts @@ -21,6 +21,8 @@ const EMPTY_DEVICE_STATE: DeviceServiceState = { hostStatus: "idle", devices: [], sessions: [], + onboardingCompleted: false, + agentAccessEnabled: false, hubBasePath: "/api/device-hub", revision: 0, }; diff --git a/docs/internals/devices.md b/docs/internals/devices.md index 57122f580352..49027022fdbc 100644 --- a/docs/internals/devices.md +++ b/docs/internals/devices.md @@ -9,9 +9,11 @@ host on another machine slot in later. ## Two external tools, one seam [expo-device-hub](../../apps/server/src/device/LocalDeviceHost.ts) streams and -[agent-device](../../apps/server/src/device/AgentDeviceShim.ts) drives. Both -are npm-installed at pinned versions into the T3 home on first use and run with -the server's Node; `npx` would make the first `device_open` after a reboot +[agent-device](../../apps/server/src/device/AgentDeviceShim.ts) drives. Each is +npm-installed at a pinned version into the T3 home after its matching Device +panel consent step. Manual setup installs and starts only expo-device-hub; +agent-device remains absent and stopped until agent access is granted. Both run +with the server's Node; `npx` would make the first `device_open` after a reboot depend on the registry. The hub is a supervised child rather than an imported middleware because serve-sim loads private CoreSimulator frameworks through a native addon, and a crash there must not take the server down. @@ -61,9 +63,10 @@ agent never handles the endpoint or token. That environment is fixed when the provider subprocess spawns, so [`prepareMcpSession`](../../apps/server/src/provider/Layers/ProviderService.ts) -starts the device host whenever the session has the `device` capability and -the machine can run at least one platform. Starting it later from -`device_open` would leave the already-running agent without the CLI. +starts agent-device only when device support and agent access have both been +enabled, the session has the `device` capability, and the machine can run at +least one platform. Starting it later from `device_open` would leave the +already-running agent without the CLI. How to drive a device is returned from `device_open`, not kept in an always-loaded prompt or skill: it costs nothing in threads that never open a diff --git a/docs/user/devices.md b/docs/user/devices.md index 0ccdc7e4d5a3..2a69fe1b7115 100644 --- a/docs/user/devices.md +++ b/docs/user/devices.md @@ -7,15 +7,24 @@ yourself. Agents get the same device through `device_*` tools and the ## Open a device -Open the right panel in a project thread and choose **Device**, then pick a -simulator or emulator. A device that is not running boots when you pick it. -The first time, T3 Code installs its device tools on the server; that takes a -minute and happens once. +Open the right panel in a project thread and choose **Device**. On first use, +the panel walks through three steps: starting the device hub, checking iOS and +Android support, and choosing whether agents may control devices. Opening the +panel alone does not download or start anything. If the hub is already +installed, the setup screen says so and reuses it. + +Choose a running device to watch it, or choose **Start** next to a stopped +device to boot it. The panel shows when you or an agent starts a device. +Turn off the device hub in **Settings → Integrations → Devices** to stop the +helper processes; simulators and emulators keep running until you power them +off. Simulators run on the machine that hosts the environment server. iOS needs -macOS with Xcode. Android needs the Android SDK with `ANDROID_HOME` set or -`adb` on the path. The panel says which platforms the server can run and why one -cannot. +macOS with Xcode. Android needs the SDK Platform-Tools, Android Emulator, +and Command-line Tools (latest), plus a virtual device created in Android +Studio's Device Manager. T3 Code detects standard SDK locations; set +`ANDROID_HOME` for a custom location. The panel explains missing dependencies. +After installing them, restart the environment server and refresh devices. The screen is interactive: click and drag to touch, type while the screen is focused, and use the toolbar for Home, Back, and Recents on Android, rotate on @@ -36,13 +45,14 @@ back from the device after a change. ## Agents and devices When an agent opens a device, the panel opens in every client connected to the -thread. Agents drive the device through the `agent-device` command line, which -T3 Code preinstalls and connects for them. iOS taps through `agent-device` build -a small test runner on first use, which takes a couple of minutes once per -server. +thread. Agents drive the device through the `agent-device` command line. T3 +Code installs and starts it only after **Agent device access** is enabled. iOS +taps build a small test runner on first use, which takes a couple of minutes +once per server. Restart an existing agent session after granting access so it +receives the device CLI environment. To keep agents away from simulators, turn off **Agent device access** in -Settings → Projects → Project defaults. This hides the device tools from agents +**Settings → Integrations → Devices**. This hides the device tools from agents started from then on; your own Device panel is unaffected. ## Remote connections diff --git a/packages/client-runtime/src/state/device.ts b/packages/client-runtime/src/state/device.ts index bec29b6dfc29..df27f793720c 100644 --- a/packages/client-runtime/src/state/device.ts +++ b/packages/client-runtime/src/state/device.ts @@ -22,6 +22,12 @@ export function createDeviceEnvironmentAtoms( label: "environment-data:device:state", tag: WS_METHODS.subscribeDeviceState, }), + configure: createEnvironmentRpcCommand(runtime, { + label: "environment-data:device:configure", + tag: WS_METHODS.deviceConfigure, + scheduler, + concurrency, + }), list: createEnvironmentRpcCommand(runtime, { label: "environment-data:device:list", tag: WS_METHODS.deviceList, diff --git a/packages/contracts/src/device.ts b/packages/contracts/src/device.ts index dc482710bf5b..be896ee45a99 100644 --- a/packages/contracts/src/device.ts +++ b/packages/contracts/src/device.ts @@ -58,6 +58,8 @@ export const DeviceHostSummary = Schema.Struct({ kind: Schema.Literals(["local"]), label: TrimmedNonEmptyString, platforms: Schema.Array(DevicePlatformAvailability), + hubInstalled: Schema.Boolean, + agentDeviceInstalled: Schema.Boolean, }); export type DeviceHostSummary = typeof DeviceHostSummary.Type; @@ -67,6 +69,7 @@ export type DeviceHostSummary = typeof DeviceHostSummary.Type; * stream; the UI shows that instead of an empty picker. */ export const DeviceHostStatus = Schema.Literals([ + "disabled", "idle", "installing", "starting", @@ -94,6 +97,11 @@ export const DeviceServiceState = Schema.Struct({ hostStatusDetail: Schema.optional(Schema.String), devices: Schema.Array(DeviceSummary), sessions: Schema.Array(DeviceSession), + bootingDevices: Schema.optional( + Schema.Array(Schema.Struct({ ...DeviceSummary.fields, threadId: ThreadId })), + ), + onboardingCompleted: Schema.Boolean, + agentAccessEnabled: Schema.Boolean, /** Origin-relative path the client prefixes to hub routes. */ hubBasePath: Schema.String, revision: Schema.Int, @@ -103,6 +111,13 @@ export type DeviceServiceState = typeof DeviceServiceState.Type; export const DeviceListInput = Schema.Struct({}); export type DeviceListInput = typeof DeviceListInput.Type; +export const DeviceConfigureInput = Schema.Struct({ + enabled: Schema.optional(Schema.Boolean), + agentAccessEnabled: Schema.optional(Schema.Boolean), + onboardingCompleted: Schema.optional(Schema.Boolean), +}); +export type DeviceConfigureInput = typeof DeviceConfigureInput.Type; + export const DeviceOpenInput = Schema.Struct({ threadId: ThreadId, hostId: Schema.optional(DeviceHostId), diff --git a/packages/contracts/src/rpc.ts b/packages/contracts/src/rpc.ts index 970bceb0e376..d80d3b3949ef 100644 --- a/packages/contracts/src/rpc.ts +++ b/packages/contracts/src/rpc.ts @@ -182,6 +182,7 @@ import { import { DeviceActionInput, DeviceCloseInput, + DeviceConfigureInput, DeviceDetail, DeviceDetailInput, DeviceError, @@ -323,6 +324,7 @@ export const WS_METHODS = { previewAutomationFocusHost: "previewAutomation.focusHost", // Device methods + deviceConfigure: "device.configure", deviceList: "device.list", deviceOpen: "device.open", deviceClose: "device.close", @@ -1089,6 +1091,12 @@ const WsDeviceListRpc = Rpc.make(WS_METHODS.deviceList, { error: Schema.Union([DeviceError, EnvironmentAuthorizationError]), }); +const WsDeviceConfigureRpc = Rpc.make(WS_METHODS.deviceConfigure, { + payload: DeviceConfigureInput, + success: DeviceServiceState, + error: Schema.Union([DeviceError, EnvironmentAuthorizationError]), +}); + const WsDeviceOpenRpc = Rpc.make(WS_METHODS.deviceOpen, { payload: DeviceOpenInput, success: DeviceSession, @@ -1352,6 +1360,7 @@ export const WsRpcGroup = RpcGroup.make( WsPreviewAutomationFocusHostRpc, WsSubscribePreviewEventsRpc, WsSubscribeDiscoveredLocalServersRpc, + WsDeviceConfigureRpc, WsDeviceListRpc, WsDeviceOpenRpc, WsDeviceCloseRpc, diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index 680d7fe19683..1bfcaadc5d6e 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -969,7 +969,15 @@ export const ServerSettings = Schema.Struct({ * when the provider session is prepared. The user's own Device panel is * unaffected. */ - enableAgentDeviceAccess: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(true))), + enableAgentDeviceAccess: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(false))), + /** + * Whether this server may install and run T3's device helper processes. + * Kept separate from agent access so enabling the user's Device panel does + * not also grant providers control of simulators and emulators. + */ + enableDeviceSupport: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(false))), + /** Whether the server-local Device panel setup flow has been completed. */ + deviceOnboardingCompleted: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(false))), sidebarAutoSettleAfterDays: Schema.NullOr(SidebarAutoSettleAfterDays).pipe( Schema.withDecodingDefault(Effect.succeed(DEFAULT_SIDEBAR_AUTO_SETTLE_AFTER_DAYS)), ), @@ -1243,6 +1251,8 @@ export const ServerSettingsPatch = Schema.Struct({ ), defaultModelSelection: Schema.optionalKey(Schema.NullOr(ModelSelection)), enableAgentDeviceAccess: Schema.optionalKey(Schema.Boolean), + enableDeviceSupport: Schema.optionalKey(Schema.Boolean), + deviceOnboardingCompleted: Schema.optionalKey(Schema.Boolean), sidebarAutoSettleAfterDays: Schema.optionalKey(Schema.NullOr(SidebarAutoSettleAfterDays)), sidebarAutoSettleOnMerge: Schema.optionalKey(Schema.Boolean), backgroundActivity: Schema.optionalKey( From 1841952ad60e0383b198d0f5b556fa2c63bbb812 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Tue, 8 Sep 2026 13:56:25 -0700 Subject: [PATCH 20/28] refactor(devices): compose onboarding with shared wizard layout --- apps/web/src/components/ChatView.tsx | 7 ++-- .../web/src/components/device/DevicePanel.tsx | 7 ++-- .../web/src/components/device/DeviceSetup.tsx | 35 ++++++++----------- 3 files changed, 22 insertions(+), 27 deletions(-) diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index 78e3478ab529..913bb01bd831 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -205,7 +205,8 @@ import { RightPanelTabs } from "./RightPanelTabs"; import { AgentsPanel } from "./AgentsPanel"; import { useDeviceState } from "~/state/device"; import { DeviceSetup } from "./device/DeviceSetup"; -import { Dialog, DialogPopup } from "./ui/dialog"; +import { Dialog } from "./ui/dialog"; +import { WizardPopup } from "./ui/wizard"; import { deriveAgentPanelModel, foldSubagentActivities, @@ -8181,7 +8182,7 @@ export default function ChatView(props: ChatViewProps) { if (!open) setDeviceSetupThread(null); }} > - + {activeThreadRef ? ( ) : null} - + {rightPanelControlsAtRoot ? panelLayoutControls : null}
- + - + ); } diff --git a/apps/web/src/components/device/DeviceSetup.tsx b/apps/web/src/components/device/DeviceSetup.tsx index 428166e1daa5..e503a218192b 100644 --- a/apps/web/src/components/device/DeviceSetup.tsx +++ b/apps/web/src/components/device/DeviceSetup.tsx @@ -3,14 +3,8 @@ import { Check, CircleAlert } from "lucide-react"; import { useState } from "react"; import { Button } from "~/components/ui/button"; -import { - DialogClose, - DialogDescription, - DialogFooter, - DialogHeader, - DialogTitle, -} from "~/components/ui/dialog"; -import { WizardPanel, WizardSteps } from "~/components/ui/wizard"; +import { DialogClose } from "~/components/ui/dialog"; +import { WizardHeader, WizardPanel, WizardSteps, WizardFooter } from "~/components/ui/wizard"; import { Spinner } from "~/components/ui/spinner"; import { Switch } from "~/components/ui/switch"; import { deviceEnvironment } from "~/state/device"; @@ -81,23 +75,22 @@ export function DeviceSetup(props: { }; return ( -
- - Set up devices - - Review what runs on this environment before using simulators and emulators. - + <> + busy || pending !== null || requested > step} /> - + {step === 0 ? ( -
+

Enable the device hub

{deviceHubDescription}

@@ -121,7 +114,7 @@ export function DeviceSetup(props: { ) : null} {step === 1 ? ( -
+

Check simulator support

+

Allow agent control

{agentDeviceDescription}

@@ -164,7 +157,7 @@ export function DeviceSetup(props: { ) : null} - + {step === 0 ? ( }>Cancel ) : ( @@ -191,8 +184,8 @@ export function DeviceSetup(props: { {pending === "complete" ? "Saving…" : "Done"} )} - -
+ + ); } From a47370fc447cd273751a9a350225f12a1d01cb10 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Tue, 8 Sep 2026 14:15:57 -0700 Subject: [PATCH 21/28] fix(devices): close streams and recover device discovery reliably --- apps/server/src/device/DeviceHubProxy.test.ts | 113 +++++++++++++++++ apps/server/src/device/DeviceHubProxy.ts | 68 ++++++----- apps/server/src/device/DeviceService.test.ts | 21 ++++ apps/server/src/device/DeviceService.ts | 13 +- .../server/src/device/LocalDeviceHost.test.ts | 16 +++ apps/server/src/device/LocalDeviceHost.ts | 35 +++++- .../src/provider/Layers/ProviderService.ts | 5 +- .../web/src/components/device/DeviceSetup.tsx | 2 +- .../device/DeviceStreamView.test.tsx | 58 +++++++++ .../components/device/DeviceStreamView.tsx | 9 +- .../components/device/DeviceToolsPanel.tsx | 4 +- .../components/device/deviceHubApi.test.ts | 44 +++++++ .../web/src/components/device/deviceHubApi.ts | 7 +- .../components/device/deviceStream.test.ts | 115 +++++++++++++++++- .../web/src/components/device/deviceStream.ts | 38 +++--- docs/user/devices.md | 4 +- .../src/state/deviceHubAccess.ts | 4 +- 17 files changed, 476 insertions(+), 80 deletions(-) create mode 100644 apps/server/src/device/DeviceHubProxy.test.ts create mode 100644 apps/web/src/components/device/DeviceStreamView.test.tsx create mode 100644 apps/web/src/components/device/deviceHubApi.test.ts diff --git a/apps/server/src/device/DeviceHubProxy.test.ts b/apps/server/src/device/DeviceHubProxy.test.ts new file mode 100644 index 000000000000..7ba51f079928 --- /dev/null +++ b/apps/server/src/device/DeviceHubProxy.test.ts @@ -0,0 +1,113 @@ +import { afterEach, describe, expect, it } from "vite-plus/test"; +import { + AuthOrchestrationReadScope, + AuthOrchestrationOperateScope, + AuthSessionId, + LOCAL_DEVICE_HOST_ID, + type AuthEnvironmentScope, +} from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import { HttpClient, HttpClientResponse, HttpRouter } from "effect/unstable/http"; +import { EnvironmentAuth } from "../auth/EnvironmentAuth.ts"; +import { DeviceService } from "./DeviceService.ts"; +import { deviceHubProxyRouteLayer } from "./DeviceHubProxy.ts"; + +const disposers: Array<() => Promise> = []; +afterEach(async () => { + for (const dispose of disposers.splice(0)) await dispose(); +}); + +const fixture = (scopes: ReadonlyArray, fail = false) => { + let finalized = 0; + const requests: string[] = []; + const client = HttpClient.make((request, _url, signal) => + Effect.gen(function* () { + requests.push(request.url); + signal.addEventListener("abort", () => { + finalized++; + }); + if (fail) return yield* Effect.die(new Error("upstream failed")); + return HttpClientResponse.fromWeb(request, new Response("frame")); + }), + ); + const { handler, dispose } = HttpRouter.toWebHandler( + deviceHubProxyRouteLayer.pipe( + Layer.provideMerge( + Layer.succeed(EnvironmentAuth, { + authenticateWebSocketUpgrade: () => + Effect.succeed({ + sessionId: AuthSessionId.make("test"), + subject: "test", + method: "bearer-access-token", + scopes, + }), + } as unknown as EnvironmentAuth["Service"]), + ), + Layer.provideMerge( + Layer.succeed(DeviceService, { + currentReadiness: () => + Effect.succeed({ hostId: LOCAL_DEVICE_HOST_ID, hub: { origin: "http://hub.test" } }), + } as DeviceService["Service"]), + ), + Layer.provideMerge(Layer.succeed(HttpClient.HttpClient, client)), + ), + { disableLogger: true }, + ); + disposers.push(dispose); + return { handler, requests, finalized: () => finalized }; +}; + +describe("device hub proxy", () => { + it("releases the upstream response after forwarding its body and strips tickets", async () => { + const { handler, requests, finalized } = fixture([AuthOrchestrationReadScope]); + const response = await handler( + new Request("http://t3.test/api/device-hub/api/devices?wsTicket=secret"), + ); + expect(response.status).toBe(200); + expect(await response.text()).toBe("frame"); + expect(requests).toEqual(["http://hub.test/api/devices"]); + expect(finalized()).toBe(1); + }); + + it("releases resources when upstream acquisition fails", async () => { + const { handler, finalized } = fixture([AuthOrchestrationReadScope], true); + const response = await handler(new Request("http://t3.test/api/device-hub/api/devices")); + expect(response.status).toBe(500); + expect(finalized()).toBe(1); + }); + + it.each(["/vendor/serve-sim/helper/ws", "/vendor/serve-emu/ws"])( + "rejects input socket %s for a read-only session", + async (path) => { + const { handler, requests } = fixture([AuthOrchestrationReadScope]); + const response = await handler( + new Request(`http://t3.test/api/device-hub${path}`, { headers: { upgrade: "websocket" } }), + ); + expect(response.status).toBe(403); + expect(requests).toEqual([]); + }, + ); + + it("requires operate scope for stream tuning", async () => { + const readOnly = fixture([AuthOrchestrationReadScope]); + const path = "http://t3.test/api/device-hub/vendor/serve-emu/api/stream-settings"; + expect((await readOnly.handler(new Request(path, { method: "POST" }))).status).toBe(403); + const operator = fixture([AuthOrchestrationOperateScope]); + const response = await operator.handler(new Request(path, { method: "POST" })); + expect(response.status).toBe(200); + await response.text(); + }); + + it("never forwards the vendor shell endpoint", async () => { + const { handler, requests } = fixture([AuthOrchestrationOperateScope]); + expect( + ( + await handler( + new Request("http://t3.test/api/device-hub/vendor/serve-sim/exec", { method: "POST" }), + ) + ).status, + ).toBe(404); + expect(requests).toEqual([]); + }); +}); diff --git a/apps/server/src/device/DeviceHubProxy.ts b/apps/server/src/device/DeviceHubProxy.ts index 0822cac5b5d4..c18d077631df 100644 --- a/apps/server/src/device/DeviceHubProxy.ts +++ b/apps/server/src/device/DeviceHubProxy.ts @@ -4,18 +4,20 @@ * The hub binds loopback and is never reachable directly: serve-sim exposes a * shell-exec route and serve-emu's action routes are unauthenticated, so the * only way to a device stream is through this route, which requires an - * environment session with the orchestration read scope. Reusing the T3 + * environment session with read scope (operate scope for input and tuning). Reusing the T3 * origin is also what makes remote connections work unchanged — Tailscale and * T3 Connect already carry `/api/*` and WebSocket upgrades for the app itself. * * Only the routes the Device panel needs are forwarded. Anything under the * hub's dashboard, exec, or WebRTC surface is rejected here. */ -import { AuthOrchestrationReadScope } from "@t3tools/contracts"; +import { + AuthOrchestrationReadScope, + AuthOrchestrationOperateScope, + type AuthEnvironmentScope, +} from "@t3tools/contracts"; import * as Effect from "effect/Effect"; import * as Option from "effect/Option"; -import * as Scope from "effect/Scope"; -import * as Stream from "effect/Stream"; import { HttpClient, HttpClientRequest, @@ -83,24 +85,25 @@ const isWebSocketUpgrade = (request: HttpServerRequest.HttpServerRequest) => * bearer and DPoP clients. The upgrade authenticator already implements that * fallback order, so it is used for plain requests as well. */ -const authenticate = Effect.gen(function* () { - const request = yield* HttpServerRequest.HttpServerRequest; - const serverAuth = yield* EnvironmentAuth.EnvironmentAuth; - const session = yield* serverAuth.authenticateWebSocketUpgrade(request).pipe( - Effect.catchIf(EnvironmentAuth.isServerAuthCredentialError, (error) => - failEnvironmentAuthInvalid( - EnvironmentAuth.serverAuthCredentialReason(error), - EnvironmentAuth.serverAuthDpopFailureReason(error), +const authenticate = (requiredScope: AuthEnvironmentScope) => + Effect.gen(function* () { + const request = yield* HttpServerRequest.HttpServerRequest; + const serverAuth = yield* EnvironmentAuth.EnvironmentAuth; + const session = yield* serverAuth.authenticateWebSocketUpgrade(request).pipe( + Effect.catchIf(EnvironmentAuth.isServerAuthCredentialError, (error) => + failEnvironmentAuthInvalid( + EnvironmentAuth.serverAuthCredentialReason(error), + EnvironmentAuth.serverAuthDpopFailureReason(error), + ), ), - ), - Effect.catchIf(EnvironmentAuth.isServerAuthInternalError, (error) => - failEnvironmentInternal("internal_error", error), - ), - ); - if (!session.scopes.includes(AuthOrchestrationReadScope)) { - return yield* failEnvironmentScopeRequired(AuthOrchestrationReadScope); - } -}); + Effect.catchIf(EnvironmentAuth.isServerAuthInternalError, (error) => + failEnvironmentInternal("internal_error", error), + ), + ); + if (!session.scopes.includes(requiredScope)) { + return yield* failEnvironmentScopeRequired(requiredScope); + } + }); const forwardHeaders = (request: HttpServerRequest.HttpServerRequest, origin: string) => { const headers: Record = {}; @@ -143,8 +146,7 @@ const proxyHttp = Effect.fn("DeviceHubProxy.proxyHttp")(function* ( upstreamUrl: string, hubOrigin: string, ) { - const httpClient = yield* HttpClient.HttpClient; - const scope = yield* Scope.make(); + const httpClient = HttpClient.withScope(yield* HttpClient.HttpClient); const method = request.method; const upstreamRequest = HttpClientRequest.make(method)(upstreamUrl).pipe( HttpClientRequest.setHeaders(forwardHeaders(request, hubOrigin)), @@ -152,7 +154,7 @@ const proxyHttp = Effect.fn("DeviceHubProxy.proxyHttp")(function* ( ? (self) => self : HttpClientRequest.bodyStream(request.stream), ); - const response = yield* httpClient.execute(upstreamRequest).pipe(Scope.provide(scope)); + const response = yield* httpClient.execute(upstreamRequest); const headers: Record = {}; for (const [name, value] of Object.entries(response.headers)) { if (name === "content-encoding" || name === "transfer-encoding" || name === "connection") { @@ -162,14 +164,11 @@ const proxyHttp = Effect.fn("DeviceHubProxy.proxyHttp")(function* ( } // Long-lived MJPEG and AVCC responses must not be buffered by compression. headers["cache-control"] = "no-store, no-transform"; - return HttpServerResponse.stream( - response.stream.pipe(Stream.ensuring(Scope.close(scope, undefined as never))), - { - status: response.status, - headers, - ...(headers["content-type"] ? { contentType: headers["content-type"] } : {}), - }, - ); + return HttpServerResponse.stream(response.stream, { + status: response.status, + headers, + ...(headers["content-type"] ? { contentType: headers["content-type"] } : {}), + }); }); const handler = Effect.gen(function* () { @@ -190,7 +189,10 @@ const handler = Effect.gen(function* () { if (!upgrade && !readOnly && !MUTABLE_PATHS.some((pattern) => pattern.test(hubPath))) { return HttpServerResponse.text("Method Not Allowed", { status: 405 }); } - yield* authenticate; + const controlsDevice = + (upgrade && hubPath !== "/api/devices/ws") || + (!readOnly && /\/api\/stream-(mode|settings)$/.test(hubPath)); + yield* authenticate(controlsDevice ? AuthOrchestrationOperateScope : AuthOrchestrationReadScope); const devices = yield* DeviceService; const ready = yield* devices.currentReadiness(); if (!ready) { diff --git a/apps/server/src/device/DeviceService.test.ts b/apps/server/src/device/DeviceService.test.ts index e999bb973bea..4156b2836e3d 100644 --- a/apps/server/src/device/DeviceService.test.ts +++ b/apps/server/src/device/DeviceService.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "@effect/vitest"; import { DEFAULT_SERVER_SETTINGS, + DeviceId, LOCAL_DEVICE_HOST_ID, ThreadId, type DeviceServiceState, @@ -126,6 +127,12 @@ const fixture = Effect.fn("fixture")(function* (onBoot: Effect.Effect = Ef HttpClient.make((request) => Effect.gen(function* () { requests.push(request.url); + if (request.url.includes("/api/screenshot")) { + return HttpClientResponse.fromWeb( + request, + new Response(new Uint8Array([137, 80, 78, 71])), + ); + } if (request.url.endsWith("/boot")) { yield* onBoot; booted = true; @@ -254,3 +261,17 @@ it.effect("publishes boot progress and does not restore sessions after support i expect(state.bootingDevices).toEqual([]); }).pipe(Effect.scoped), ); + +describe("device discovery after server restart", () => { + it.effect("captures an explicit device before any client lists devices", () => + Effect.gen(function* () { + const { service, settings, requests } = yield* fixture(); + yield* Ref.update(settings, (current) => ({ ...current, enableDeviceSupport: true })); + expect((yield* service.state).devices).toEqual([]); + const capture = yield* service.screenshot({ deviceId: DeviceId.make("Pixel_API_35") }); + expect(capture.device.id).toBe("Pixel_API_35"); + expect(Array.from(capture.png)).toEqual([137, 80, 78, 71]); + expect(requests.some((url) => url.endsWith("/api/devices"))).toBe(true); + }).pipe(Effect.scoped), + ); +}); diff --git a/apps/server/src/device/DeviceService.ts b/apps/server/src/device/DeviceService.ts index 573933de2d5b..4f69a038e32f 100644 --- a/apps/server/src/device/DeviceService.ts +++ b/apps/server/src/device/DeviceService.ts @@ -633,13 +633,7 @@ export const makeWithHost = Effect.fn("DeviceService.makeWithHost")(function* ( const screenshot: DeviceService["Service"]["screenshot"] = Effect.fn("DeviceService.screenshot")( function* (input) { - const host = yield* resolveHost(input.hostId); - const ready = yield* readiness(host.id); - const { state } = yield* SynchronizedRef.get(stateRef); - const device = findDevice(state, host.id, input.deviceId); - if (!device) { - return yield* new DeviceNotFoundError({ hostId: host.id, deviceId: input.deviceId }); - } + const { ready, device } = yield* resolveDevice(input.hostId, input.deviceId); const url = `${ready.hub.origin}${vendorPrefix(device.platform)}/api/screenshot?device=${encodeURIComponent(device.id)}`; const png = yield* httpClient.execute(HttpClientRequest.post(url)).pipe( Effect.flatMap(HttpClientResponse.filterStatusOk), @@ -666,7 +660,8 @@ export const makeWithHost = Effect.fn("DeviceService.makeWithHost")(function* ( const host = yield* resolveHost(hostId); const ready = yield* readiness(host.id); const { state } = yield* SynchronizedRef.get(stateRef); - const device = findDevice(state, host.id, deviceId); + const device = + findDevice(state, host.id, deviceId) ?? findDevice(yield* refresh(ready), host.id, deviceId); if (!device) return yield* new DeviceNotFoundError({ hostId: host.id, deviceId }); return { ready, device }; }); @@ -717,7 +712,7 @@ export const makeWithHost = Effect.fn("DeviceService.makeWithHost")(function* ( }); }); -export const make = Effect.gen(function* () { +const make = Effect.gen(function* () { return yield* makeWithHost(yield* LocalDeviceHost.make()); }).pipe(Effect.withSpan("DeviceService.make")); diff --git a/apps/server/src/device/LocalDeviceHost.test.ts b/apps/server/src/device/LocalDeviceHost.test.ts index dc09bf1dec9c..c8ef28ed7e18 100644 --- a/apps/server/src/device/LocalDeviceHost.test.ts +++ b/apps/server/src/device/LocalDeviceHost.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from "@effect/vitest"; import * as NodePath from "@effect/platform-node/NodePath"; import { HostProcessEnvironment, HostProcessPlatform } from "@t3tools/shared/hostProcess"; import * as Effect from "effect/Effect"; +import * as Path from "effect/Path"; import * as FileSystem from "effect/FileSystem"; import { __testing } from "./LocalDeviceHost.ts"; @@ -57,3 +58,18 @@ describe("Android SDK availability", () => { }), ); }); + +it.effect("puts detected Android tools on the helper PATH without losing existing commands", () => + Effect.gen(function* () { + const path = yield* Path.Path; + const environment = __testing.deviceHostEnvironment( + { PATH: "/usr/bin", HOME: "/test/home" }, + "/sdk", + "darwin", + path, + ); + expect(environment.PATH).toBe("/sdk/platform-tools:/sdk/emulator:/usr/bin"); + expect(environment.ANDROID_HOME).toBe("/sdk"); + expect(environment.HOME).toBe("/test/home"); + }).pipe(Effect.provide(NodePath.layer)), +); diff --git a/apps/server/src/device/LocalDeviceHost.ts b/apps/server/src/device/LocalDeviceHost.ts index 7b723d04ce8d..fe88378cfb56 100644 --- a/apps/server/src/device/LocalDeviceHost.ts +++ b/apps/server/src/device/LocalDeviceHost.ts @@ -167,6 +167,25 @@ const androidSdk = Effect.gen(function* () { return { root: null, adb: false, emulator: false, avdmanager: false }; }); +const deviceHostEnvironment = ( + environment: NodeJS.ProcessEnv, + sdkRoot: string | null, + hostPlatform: NodeJS.Platform, + path: Path.Path, +): NodeJS.ProcessEnv => { + return sdkRoot + ? { + ...environment, + ANDROID_HOME: sdkRoot, + PATH: [ + path.join(sdkRoot, "platform-tools"), + path.join(sdkRoot, "emulator"), + environment.PATH ?? environment.Path ?? "", + ].join(hostPlatform === "win32" ? ";" : ":"), + } + : environment; +}; + export const make = Effect.fn("LocalDeviceHost.make")(function* () { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const config = yield* ServerConfig.ServerConfig; @@ -178,7 +197,7 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () { const environment = yield* HostProcessEnvironment; const hostPlatform = yield* HostProcessPlatform; const sdk = yield* androidSdk; - const hostEnvironment = sdk.root ? { ...environment, ANDROID_HOME: sdk.root } : environment; + const hostEnvironment = deviceHostEnvironment(environment, sdk.root, hostPlatform, path); const startLock = yield* Semaphore.make(1); const runningRef = yield* Ref.make(null); const restartDelayRef = yield* Ref.make(0); @@ -444,11 +463,14 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () { entryPath: agentTool.entryPath, }); if (existing._tag === "Some") { - const alive = yield* httpClient + const alive = yield* HttpClient.withScope(httpClient) .get(`http://127.0.0.1:${existing.value.httpPort}/health`) .pipe( Effect.timeout(Duration.seconds(2)), - Effect.map((response) => response.status === 200), + Effect.flatMap((response) => + response.arrayBuffer.pipe(Effect.as(response.status === 200)), + ), + Effect.scoped, Effect.orElseSucceed(() => false), ); if (alive) return toEndpoint(existing.value); @@ -675,4 +697,9 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () { }); /** Exposed for tests. */ -export const __testing = { AgentDeviceDaemonFile, androidSdk, platformReason }; +export const __testing = { + AgentDeviceDaemonFile, + androidSdk, + platformReason, + deviceHostEnvironment, +}; diff --git a/apps/server/src/provider/Layers/ProviderService.ts b/apps/server/src/provider/Layers/ProviderService.ts index 310e371c45aa..3b185b2d3574 100644 --- a/apps/server/src/provider/Layers/ProviderService.ts +++ b/apps/server/src/provider/Layers/ProviderService.ts @@ -257,10 +257,7 @@ export interface ProviderServiceLiveOptions { /** Same seam as `issueMcpCredential`, for observing the deny path's revoke. */ readonly revokeMcpCredential?: typeof McpSessionRegistry.revokeActiveMcpThread; /** Overrides the device host lookup used to build the agent-device environment. */ - readonly deviceReadiness?: () => Effect.Effect< - DeviceService.DeviceAgentReadiness | null, - unknown - >; + readonly deviceReadiness?: DeviceService.DeviceService["Service"]["agentReadinessIfSupported"]; } interface TurnAnalyticsMetadata { diff --git a/apps/web/src/components/device/DeviceSetup.tsx b/apps/web/src/components/device/DeviceSetup.tsx index e503a218192b..d203fe0ed965 100644 --- a/apps/web/src/components/device/DeviceSetup.tsx +++ b/apps/web/src/components/device/DeviceSetup.tsx @@ -219,7 +219,7 @@ export function DeviceHubSetupStatus({ ); } -export function DevicePlatformSetup(props: { +function DevicePlatformSetup(props: { readonly state: DeviceServiceState; readonly checking: boolean; readonly disabled: boolean; diff --git a/apps/web/src/components/device/DeviceStreamView.test.tsx b/apps/web/src/components/device/DeviceStreamView.test.tsx new file mode 100644 index 000000000000..a46e5ed7b7b8 --- /dev/null +++ b/apps/web/src/components/device/DeviceStreamView.test.tsx @@ -0,0 +1,58 @@ +import { act } from "react"; +import { create, type ReactTestRenderer } from "react-test-renderer"; +import { EnvironmentId } from "@t3tools/contracts"; +import { afterEach, expect, it, vi } from "vite-plus/test"; + +vi.mock("~/state/device", () => ({ + useDeviceHubAccess: () => access, + refreshDeviceHubAccess: vi.fn(), +})); +const access = { httpBase: "http://test", wsBase: "ws://test", query: {}, credentials: true }; +vi.mock("./deviceStream", () => ({ + createDeviceStreamClient: ( + _target: unknown, + _canvas: unknown, + events: { onMjpegFallback: (url: string) => void }, + ) => ({ + start: () => events.onMjpegFallback("http://test/stream.mjpeg"), + stop: vi.fn(), + }), +})); +import { DeviceStreamView } from "./DeviceStreamView"; +let renderer: ReactTestRenderer | undefined; +afterEach(async () => { + await act(async () => renderer?.unmount()); + vi.unstubAllGlobals(); +}); + +it("removes MJPEG requests while hidden and reconnects when shown", async () => { + vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true); + vi.stubGlobal( + "ResizeObserver", + class { + observe() {} + disconnect() {} + }, + ); + const view = (visible: boolean) => ( + + ); + await act(async () => { + renderer = create(view(true), { + createNodeMock: () => ({ + style: { setProperty() {} }, + getBoundingClientRect: () => ({ width: 400, height: 800 }), + }), + }); + }); + expect(renderer!.root.findAllByType("img")).toHaveLength(1); + await act(async () => renderer!.update(view(false))); + expect(renderer!.root.findAllByType("img")).toHaveLength(0); + await act(async () => renderer!.update(view(true))); + expect(renderer!.root.findAllByType("img")).toHaveLength(1); +}); diff --git a/apps/web/src/components/device/DeviceStreamView.tsx b/apps/web/src/components/device/DeviceStreamView.tsx index 7b440ea332a9..cb70a98476d6 100644 --- a/apps/web/src/components/device/DeviceStreamView.tsx +++ b/apps/web/src/components/device/DeviceStreamView.tsx @@ -221,11 +221,8 @@ export function DeviceStreamView(props: { const pointerActive = useRef(false); const normalizedPoint = (event: React.PointerEvent) => { const rect = event.currentTarget.getBoundingClientRect(); - let x = (event.clientX - rect.left) / rect.width; - let y = (event.clientY - rect.top) / rect.height; - if (rotation === -90) [x, y] = [1 - y, x]; - else if (rotation === 90) [x, y] = [y, 1 - x]; - else if (rotation === 180) [x, y] = [1 - x, 1 - y]; + const x = (event.clientX - rect.left) / rect.width; + const y = (event.clientY - rect.top) / rect.height; return { x: Math.min(1, Math.max(0, x)), y: Math.min(1, Math.max(0, y)) }; }; @@ -278,7 +275,7 @@ export function DeviceStreamView(props: { className={cn("absolute", mjpegUrl && "hidden")} style={mediaStyle} /> - {mjpegUrl ? ( + {props.visible && access && mjpegUrl ? ( (null); const [pending, setPending] = useState(false); const [error, setError] = useState(null); - const [foreground, setForeground] = useState(null); + const [foreground, setForeground] = useState(undefined); const isIos = device.platform === "ios"; const target = useMemo( @@ -164,7 +164,7 @@ export function DeviceToolsPanel(props: { ); const settings = detail?.settings; - const foregroundApp = foreground ?? detail?.foregroundApp ?? null; + const foregroundApp = foreground === undefined ? (detail?.foregroundApp ?? null) : foreground; const disabled = pending || detail === null; return ( diff --git a/apps/web/src/components/device/deviceHubApi.test.ts b/apps/web/src/components/device/deviceHubApi.test.ts new file mode 100644 index 000000000000..52575f4a4224 --- /dev/null +++ b/apps/web/src/components/device/deviceHubApi.test.ts @@ -0,0 +1,44 @@ +import { afterEach, describe, expect, it, vi } from "vite-plus/test"; +import { subscribeDeviceForeground } from "./deviceHubApi"; + +describe("foreground app events", () => { + afterEach(() => vi.unstubAllGlobals()); + + it("clears the last app when it exits and ignores malformed events", () => { + let source: FakeEventSource; + class FakeEventSource { + onmessage: ((event: { data: string }) => void) | null = null; + addEventListener(_type: string, listener: (event: { data: string }) => void) { + this.onmessage = listener; + } + close = vi.fn(); + constructor() { + source = this; + } + } + vi.stubGlobal("EventSource", FakeEventSource); + const onChange = vi.fn(); + const stop = subscribeDeviceForeground( + { + platform: "ios", + deviceId: "test", + access: { httpBase: "http://test", wsBase: "ws://test", query: {}, credentials: true }, + }, + onChange, + ); + const emit = (data: unknown) => source.onmessage?.({ data: JSON.stringify(data) }); + emit({ bundleId: "com.example.app", pid: 123 }); + emit({ bundleId: null }); + emit({ bundleId: "com.example.other" }); + emit({ bundleId: "" }); + emit({ other: "not app state" }); + expect(onChange.mock.calls.map(([app]) => app)).toEqual([ + { id: "com.example.app", pid: 123 }, + null, + { id: "com.example.other" }, + null, + ]); + stop(); + expect(source!.close).toHaveBeenCalledOnce(); + }); +}); diff --git a/apps/web/src/components/device/deviceHubApi.ts b/apps/web/src/components/device/deviceHubApi.ts index a121effea83d..dbcf4fa0015c 100644 --- a/apps/web/src/components/device/deviceHubApi.ts +++ b/apps/web/src/components/device/deviceHubApi.ts @@ -193,7 +193,12 @@ export function subscribeDeviceForeground( target, hubUrl(target, "/appstate", { device: target.deviceId }), (data) => { - if (!isRecord(data) || typeof data.bundleId !== "string") return; + if (!isRecord(data)) return; + if (data.bundleId === null || data.bundleId === "") { + onChange(null); + return; + } + if (typeof data.bundleId !== "string") return; onChange({ id: data.bundleId, ...(typeof data.pid === "number" ? { pid: data.pid } : {}), diff --git a/apps/web/src/components/device/deviceStream.test.ts b/apps/web/src/components/device/deviceStream.test.ts index 25b7a53ba6f2..0ce7ea041bbb 100644 --- a/apps/web/src/components/device/deviceStream.test.ts +++ b/apps/web/src/components/device/deviceStream.test.ts @@ -1,6 +1,12 @@ -import { describe, expect, it } from "vite-plus/test"; +import { afterEach, describe, expect, it, vi } from "vite-plus/test"; -import { AvccDemuxer, avcCodecString, parseSemuPacket, scanAccessUnit } from "./deviceStream"; +import { + createDeviceStreamClient, + AvccDemuxer, + avcCodecString, + parseSemuPacket, + scanAccessUnit, +} from "./deviceStream"; const envelope = (tag: number, payload: number[]) => { const length = 1 + payload.length; @@ -66,3 +72,108 @@ describe("serve-emu frames", () => { expect(scanAccessUnit(new Uint8Array([0, 0, 1, 0x41, 0x00])).isKey).toBe(false); }); }); + +describe("iOS input startup", () => { + afterEach(() => { + vi.useRealTimers(); + vi.unstubAllGlobals(); + }); + + const setup = () => { + vi.useFakeTimers(); + const sockets: FakeSocket[] = []; + class FakeSocket { + static OPEN = 1; + readyState = 1; + binaryType = ""; + onopen: (() => void) | null = null; + onmessage: ((event: { data: ArrayBuffer }) => void) | null = null; + onclose: ((event: { code: number; reason: string }) => void) | null = null; + send = vi.fn(); + close = vi.fn(); + constructor() { + sockets.push(this); + } + } + vi.stubGlobal("WebSocket", FakeSocket); + const signals: AbortSignal[] = []; + vi.stubGlobal( + "fetch", + vi.fn((_url, init: RequestInit) => { + const signal = init.signal!; + signals.push(signal); + return Promise.resolve( + new Response( + new ReadableStream({ + start(controller) { + signal.addEventListener("abort", () => controller.error(new Error("aborted"))); + }, + }), + ), + ); + }), + ); + const client = createDeviceStreamClient( + { + platform: "ios", + deviceId: "test-device", + access: { + httpBase: "http://test/api/device-hub", + wsBase: "ws://test/api/device-hub", + credentials: true, + query: {}, + }, + }, + { getContext: () => null } as unknown as HTMLCanvasElement, + { + onStatus: vi.fn(), + onScreen: vi.fn(), + onUnauthorized: vi.fn(), + onMjpegFallback: vi.fn(), + onInputConnected: vi.fn(), + }, + ); + return { client, sockets, signals }; + }; + + it("connects input when the MJPEG prime never produces a frame", async () => { + const { client, sockets, signals } = setup(); + client.start(); + await vi.advanceTimersByTimeAsync(2_000); + expect(signals[0]?.aborted).toBe(true); + expect(sockets).toHaveLength(1); + client.stop(); + }); + + it("aborts priming immediately when hidden without opening a socket later", async () => { + const { client, sockets, signals } = setup(); + client.start(); + await vi.advanceTimersByTimeAsync(0); + client.stop(); + expect(signals[0]?.aborted).toBe(true); + await vi.advanceTimersByTimeAsync(2_000); + expect(sockets).toHaveLength(0); + }); + + it("maps displayed landscape touches to raw iOS coordinates once", async () => { + const { client, sockets } = setup(); + client.start(); + await vi.advanceTimersByTimeAsync(2_000); + const socket = sockets[0]!; + const json = new TextEncoder().encode( + JSON.stringify({ width: 400, height: 800, orientation: "landscape_left" }), + ); + const packet = new Uint8Array(1 + json.length); + packet[0] = 0x82; + packet.set(json, 1); + socket.onmessage?.({ data: packet.buffer }); + client.sendTouch("begin", 0.2, 0.7); + const sent = socket.send.mock.calls[0]![0] as Uint8Array; + expect(JSON.parse(new TextDecoder().decode(sent.subarray(1)))).toEqual({ + type: "begin", + x: 1 - 0.7, + y: 0.2, + }); + client.stop(); + }); +}); diff --git a/apps/web/src/components/device/deviceStream.ts b/apps/web/src/components/device/deviceStream.ts index a1e13b22fe8b..8a79eb09e137 100644 --- a/apps/web/src/components/device/deviceStream.ts +++ b/apps/web/src/components/device/deviceStream.ts @@ -285,7 +285,8 @@ export function createDeviceStreamClient( let stopped = true; let socket: WebSocket | null = null; let controller: AbortController | null = null; - let retryTimer: ReturnType | null = null; + const retryTimers = new Map<"video" | "input", ReturnType>(); + let primeController: AbortController | null = null; let videoDecoder: VideoDecoder | null = null; let timestamp = 0; let awaitingKeyframe = true; @@ -405,12 +406,15 @@ export function createDeviceStreamClient( } }; - const scheduleRetry = (run: () => void) => { - if (stopped || retryTimer) return; - retryTimer = setTimeout(() => { - retryTimer = null; - run(); - }, RETRY_DELAY_MS); + const scheduleRetry = (channel: "video" | "input", run: () => void) => { + if (stopped || retryTimers.has(channel)) return; + retryTimers.set( + channel, + setTimeout(() => { + retryTimers.delete(channel); + run(); + }, RETRY_DELAY_MS), + ); }; const handleUnauthorized = () => { @@ -467,7 +471,7 @@ export function createDeviceStreamClient( if (stopped) return; setStatus("connecting", (cause as Error).message); } - if (!stopped) scheduleRetry(() => void readIosVideo()); + if (!stopped) scheduleRetry("video", () => void readIosVideo()); }; /** @@ -477,6 +481,8 @@ export function createDeviceStreamClient( */ const primeIosHelper = async () => { const controller = new AbortController(); + primeController = controller; + const timeout = setTimeout(() => controller.abort(), 2_000); try { const response = await fetch(httpUrl(`/helper/${device}/stream.mjpeg`), { signal: controller.signal, @@ -487,7 +493,9 @@ export function createDeviceStreamClient( } catch { // A failed prime just means the socket may take a retry to come up. } finally { + clearTimeout(timeout); controller.abort(); + if (primeController === controller) primeController = null; } }; @@ -526,7 +534,7 @@ export function createDeviceStreamClient( ); } if (event.code === 1008 || event.code === 4401) return handleUnauthorized(); - scheduleRetry(() => void connectIosInput()); + scheduleRetry("input", () => void connectIosInput()); }; ws.onerror = () => ws.close(); }; @@ -578,7 +586,7 @@ export function createDeviceStreamClient( if (event.code === 1008 || event.code === 4401) return handleUnauthorized(); if (!stopped) { setStatus("connecting", event.reason || undefined); - scheduleRetry(connectAndroid); + scheduleRetry("input", connectAndroid); } }; ws.onerror = () => ws.close(); @@ -604,8 +612,10 @@ export function createDeviceStreamClient( if (stopped) return; stopped = true; mjpeg = false; - if (retryTimer) clearTimeout(retryTimer); - retryTimer = null; + for (const timer of retryTimers.values()) clearTimeout(timer); + retryTimers.clear(); + primeController?.abort(); + primeController = null; controller?.abort(); controller = null; socket?.close(); @@ -623,9 +633,9 @@ export function createDeviceStreamClient( if (platform !== "ios" || !screen || screen.width > screen.height) return { x, y }; switch (screen.orientation) { case "landscape_left": - return { x: y, y: 1 - x }; - case "landscape_right": return { x: 1 - y, y: x }; + case "landscape_right": + return { x: y, y: 1 - x }; case "portrait_upside_down": return { x: 1 - x, y: 1 - y }; default: diff --git a/docs/user/devices.md b/docs/user/devices.md index 2a69fe1b7115..51a0a578b607 100644 --- a/docs/user/devices.md +++ b/docs/user/devices.md @@ -44,8 +44,8 @@ back from the device after a change. ## Agents and devices -When an agent opens a device, the panel opens in every client connected to the -thread. Agents drive the device through the `agent-device` command line. T3 +When an agent opens a device, the panel opens in web and desktop clients connected +to the thread. Mobile clients show device activity in the thread timeline. Agents drive the device through the `agent-device` command line. T3 Code installs and starts it only after **Agent device access** is enabled. iOS taps build a small test runner on first use, which takes a couple of minutes once per server. Restart an existing agent session after granting access so it diff --git a/packages/client-runtime/src/state/deviceHubAccess.ts b/packages/client-runtime/src/state/deviceHubAccess.ts index 561c56423761..328dea83ce7a 100644 --- a/packages/client-runtime/src/state/deviceHubAccess.ts +++ b/packages/client-runtime/src/state/deviceHubAccess.ts @@ -2,8 +2,8 @@ * Credentials for the Device panel's media requests. * * The panel reaches simulator streams through `/api/device-hub/*` on the - * environment origin, using ``, `fetch`, and `WebSocket`. None of those - * can carry a bearer or DPoP header, so bearer and DPoP connections mint a + * environment origin. ``, `EventSource`, and `WebSocket` cannot set + * bearer or DPoP headers, so bearer and DPoP connections mint a * short-lived WebSocket ticket and pass it as `wsTicket`, the same way the * app's own `/ws` upgrade authenticates. Cookie sessions send the cookie. * From 7c637d2288d42179e112a21b4ca2cb1e79e53760 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Tue, 8 Sep 2026 14:20:45 -0700 Subject: [PATCH 22/28] fix(devices): pass CLI access through all local providers --- .../server/src/mcp/McpProviderSession.test.ts | 30 +++++++++++++++++++ apps/server/src/mcp/McpProviderSession.ts | 2 +- .../src/provider/Drivers/AntigravityDriver.ts | 3 +- .../src/provider/Layers/AntigravityAdapter.ts | 3 ++ .../src/provider/Layers/OpenCodeAdapter.ts | 7 +++-- .../src/provider/acp/AntigravityAcpSupport.ts | 2 ++ 6 files changed, 43 insertions(+), 4 deletions(-) create mode 100644 apps/server/src/mcp/McpProviderSession.test.ts diff --git a/apps/server/src/mcp/McpProviderSession.test.ts b/apps/server/src/mcp/McpProviderSession.test.ts new file mode 100644 index 000000000000..1c3c3af5091e --- /dev/null +++ b/apps/server/src/mcp/McpProviderSession.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, it } from "vite-plus/test"; +import { withAgentDeviceEnvironment } from "./McpProviderSession.ts"; + +describe("device CLI environment", () => { + it("preserves provider credentials and commands while routing devices to the owned daemon", () => { + const environment = withAgentDeviceEnvironment( + { PATH: "/provider/bin:/usr/bin", PROVIDER_KEY: "fixture" }, + { + agentDeviceEnvironment: { + PATH: "/t3/device/bin", + PATH_SEPARATOR: ":", + AGENT_DEVICE_DAEMON_BASE_URL: "http://127.0.0.1:9000", + AGENT_DEVICE_DAEMON_AUTH_TOKEN: "fixture-device", + }, + }, + ); + expect(environment).toEqual({ + PATH: "/t3/device/bin:/provider/bin:/usr/bin", + PROVIDER_KEY: "fixture", + AGENT_DEVICE_DAEMON_BASE_URL: "http://127.0.0.1:9000", + AGENT_DEVICE_DAEMON_AUTH_TOKEN: "fixture-device", + }); + }); + + it("does not grant CLI access when device access was not supplied", () => { + const environment = { PATH: "/usr/bin", PROVIDER_KEY: "fixture" }; + expect(withAgentDeviceEnvironment(environment, undefined)).toBe(environment); + expect(withAgentDeviceEnvironment(environment, {})).toBe(environment); + }); +}); diff --git a/apps/server/src/mcp/McpProviderSession.ts b/apps/server/src/mcp/McpProviderSession.ts index dfa7dc6e09c2..4019cb37bd04 100644 --- a/apps/server/src/mcp/McpProviderSession.ts +++ b/apps/server/src/mcp/McpProviderSession.ts @@ -20,7 +20,7 @@ export interface McpProviderSessionConfig { /** Provider env with the device variables applied over `base`, or `base` untouched. */ export function withAgentDeviceEnvironment( base: NodeJS.ProcessEnv, - config: McpProviderSessionConfig | undefined, + config: Pick | undefined, ): NodeJS.ProcessEnv { const extra = config?.agentDeviceEnvironment; if (!extra) return base; diff --git a/apps/server/src/provider/Drivers/AntigravityDriver.ts b/apps/server/src/provider/Drivers/AntigravityDriver.ts index 65a8c97fe668..0082f3cbdc30 100644 --- a/apps/server/src/provider/Drivers/AntigravityDriver.ts +++ b/apps/server/src/provider/Drivers/AntigravityDriver.ts @@ -1,3 +1,4 @@ +import { withAgentDeviceEnvironment } from "../../mcp/McpProviderSession.ts"; import { AntigravitySettings, ProviderDriverKind, ProviderSetupError } from "@t3tools/contracts"; import { HostProcessPlatform } from "@t3tools/shared/hostProcess"; import * as Crypto from "effect/Crypto"; @@ -162,7 +163,7 @@ export const AntigravityDriver: ProviderDriver client.mcp.add({ diff --git a/apps/server/src/provider/acp/AntigravityAcpSupport.ts b/apps/server/src/provider/acp/AntigravityAcpSupport.ts index 6e625127cb18..17b73e552006 100644 --- a/apps/server/src/provider/acp/AntigravityAcpSupport.ts +++ b/apps/server/src/provider/acp/AntigravityAcpSupport.ts @@ -35,6 +35,8 @@ export interface AntigravityAcpRuntimeInput extends Omit< | "transformSessionUpdate" | "transformStdout" > { + /** Device CLI environment supplied for this provider session. */ + readonly agentDeviceEnvironment?: Readonly>; readonly childProcessSpawner: ChildProcessSpawner.ChildProcessSpawner["Service"]; readonly onAuthorizationUrl?: (url: string) => Effect.Effect; /** From e9d881a6a2d43aec90ca1bda42c4bfc8f0a04cdb Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Tue, 8 Sep 2026 14:22:15 -0700 Subject: [PATCH 23/28] fix(devices): show device launch and shutdown failures --- .../web/src/components/device/DevicePanel.tsx | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/apps/web/src/components/device/DevicePanel.tsx b/apps/web/src/components/device/DevicePanel.tsx index aa0ef0e0831a..d94913b03dff 100644 --- a/apps/web/src/components/device/DevicePanel.tsx +++ b/apps/web/src/components/device/DevicePanel.tsx @@ -35,6 +35,7 @@ import { Toggle } from "~/components/ui/toggle"; import { Tooltip, TooltipPopup, TooltipTrigger } from "~/components/ui/tooltip"; import { cn } from "~/lib/utils"; import { deviceEnvironment, useDeviceHubAccess, useDeviceState } from "~/state/device"; +import { formatEnvironmentQueryError } from "~/state/query"; import { useAtomCommand } from "~/state/use-atom-command"; import { DeviceStreamView, type DeviceStreamHandle } from "./DeviceStreamView"; import { DeviceSetup } from "./DeviceSetup"; @@ -67,6 +68,7 @@ export function DevicePanel(props: { const list = useAtomCommand(deviceEnvironment.list, { reportFailure: false }); const open = useAtomCommand(deviceEnvironment.open); const close = useAtomCommand(deviceEnvironment.close); + const [operationError, setOperationError] = useState(null); const [pendingDeviceKey, setPendingDeviceKey] = useState(null); const [handle, setHandle] = useState(null); const [toolsOpen, setToolsOpen] = useState(false); @@ -102,9 +104,10 @@ export function DevicePanel(props: { if (value === NEW_DEVICE_VALUE) return; const device = state.devices.find((candidate) => deviceKey(candidate) === value); if (!device) return; + setOperationError(null); setPendingDeviceKey(value); try { - await open({ + const result = await open({ environmentId, input: { threadId, @@ -113,6 +116,7 @@ export function DevicePanel(props: { platform: device.platform, }, }); + if (result._tag === "Failure") setOperationError(formatEnvironmentQueryError(result.cause)); } finally { setPendingDeviceKey(null); } @@ -123,9 +127,12 @@ export function DevicePanel(props: { const closeActive = useCallback( (powerOff: boolean) => { if (!activeSession) return; + setOperationError(null); void close({ environmentId, input: { threadId, deviceId: activeSession.deviceId, shutdown: powerOff }, + }).then((result) => { + if (result._tag === "Failure") setOperationError(formatEnvironmentQueryError(result.cause)); }); }, [activeSession, close, environmentId, threadId], @@ -280,6 +287,22 @@ export function DevicePanel(props: { Starting {bootingDevices.map((device) => device.name).join(", ")}… This can take a minute.
) : null} + {operationError ? ( +
+

{operationError}

+ +
+ ) : null}
{activeDevice && activeSession ? ( <> From ab733a22959f79640458080b9cb57824854410f3 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Tue, 8 Sep 2026 14:29:40 -0700 Subject: [PATCH 24/28] refactor(devices): expose host dependency and normalize setup failures --- apps/server/src/device/DeviceHost.ts | 66 ++++++++++------- apps/server/src/device/DeviceService.test.ts | 11 +-- apps/server/src/device/DeviceService.ts | 23 +++--- .../server/src/device/DeviceToolchain.test.ts | 39 ++++++++++ apps/server/src/device/DeviceToolchain.ts | 3 +- apps/server/src/device/LocalDeviceHost.ts | 72 +++++++++---------- 6 files changed, 127 insertions(+), 87 deletions(-) create mode 100644 apps/server/src/device/DeviceToolchain.test.ts diff --git a/apps/server/src/device/DeviceHost.ts b/apps/server/src/device/DeviceHost.ts index 23edc61bb97a..8d1bafbc5dfc 100644 --- a/apps/server/src/device/DeviceHost.ts +++ b/apps/server/src/device/DeviceHost.ts @@ -15,17 +15,26 @@ import type { DevicePlatform, DevicePlatformAvailability, } from "@t3tools/contracts"; +import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; import * as Schema from "effect/Schema"; export class DeviceHostError extends Schema.TaggedError()("DeviceHostError", { hostId: Schema.String, step: Schema.String, - detail: Schema.String, - cause: Schema.optional(Schema.Defect()), + cause: Schema.Defect(), }) { override get message(): string { - return `Device host ${this.hostId} failed while ${this.step}: ${this.detail}`; + return `Device host ${this.hostId} failed while ${this.step}.`; + } +} + +export class DeviceHostTimeoutError extends Schema.TaggedError()( + "DeviceHostTimeoutError", + { hostId: Schema.String, timeoutMs: Schema.Number }, +) { + override get message(): string { + return `Device host ${this.hostId} did not start agent tools within ${this.timeoutMs} ms.`; } } @@ -64,27 +73,30 @@ export interface DeviceHostAgentReady extends DeviceHostReady { readonly agentDevice: AgentDeviceEndpoint; } -export interface DeviceHost { - readonly id: DeviceHostId; - readonly summary: Effect.Effect; - readonly platformAvailability: ( - platform: DevicePlatform, - ) => Effect.Effect; - /** - * Installs tools on first use and starts the helper processes. Idempotent: - * concurrent callers share one start, and a ready host returns immediately. - */ - readonly ensureReady: ( - onPhase: (phase: "installing" | "starting") => Effect.Effect, - ) => Effect.Effect; - /** Installs and starts agent-device after the user grants agent access. */ - readonly ensureAgentReady: ( - onPhase: (phase: "installing" | "starting") => Effect.Effect, - ) => Effect.Effect; - /** Current endpoints when already running, without starting anything. */ - readonly current: Effect.Effect; - /** Stops only agent-device. Manual viewing through the hub stays available. */ - readonly stopAgent: Effect.Effect; - /** Stops helpers. Devices themselves keep running; the user owns those. */ - readonly stop: Effect.Effect; -} +export class DeviceHost extends Context.Service< + DeviceHost, + { + readonly id: DeviceHostId; + readonly summary: Effect.Effect; + readonly platformAvailability: ( + platform: DevicePlatform, + ) => Effect.Effect; + /** + * Installs tools on first use and starts the helper processes. Idempotent: + * concurrent callers share one start, and a ready host returns immediately. + */ + readonly ensureReady: ( + onPhase: (phase: "installing" | "starting") => Effect.Effect, + ) => Effect.Effect; + /** Installs and starts agent-device after the user grants agent access. */ + readonly ensureAgentReady: ( + onPhase: (phase: "installing" | "starting") => Effect.Effect, + ) => Effect.Effect; + /** Current endpoints when already running, without starting anything. */ + readonly current: Effect.Effect; + /** Stops only agent-device. Manual viewing through the hub stays available. */ + readonly stopAgent: Effect.Effect; + /** Stops helpers. Devices themselves keep running; the user owns those. */ + readonly stop: Effect.Effect; + } +>()("t3/device/DeviceHost") {} diff --git a/apps/server/src/device/DeviceService.test.ts b/apps/server/src/device/DeviceService.test.ts index 4156b2836e3d..f7b7ece1d679 100644 --- a/apps/server/src/device/DeviceService.test.ts +++ b/apps/server/src/device/DeviceService.test.ts @@ -14,9 +14,9 @@ import * as Ref from "effect/Ref"; import * as Stream from "effect/Stream"; import { HttpClient, HttpClientResponse } from "effect/unstable/http"; import { ServerSettingsService } from "../serverSettings.ts"; -import type { DeviceHost, DeviceHostReady } from "./DeviceHost.ts"; +import * as DeviceHost from "./DeviceHost.ts"; -import { type DeviceService, makeWithHost, stateStream } from "./DeviceService.ts"; +import { type DeviceService, make, stateStream } from "./DeviceService.ts"; const baseState: DeviceServiceState = { hosts: [], @@ -63,12 +63,12 @@ const fixture = Effect.fn("fixture")(function* (onBoot: Effect.Effect = Ef const agentStops: string[] = []; const requests: string[] = []; let booted = false; - const ready: DeviceHostReady = { + const ready: DeviceHost.DeviceHostReady = { hub: { origin: "http://device.test" }, helpers: { serveSimAxSettings: null, serveSimCli: null }, run: () => Effect.succeed({ code: 0, stdout: "Pixel_API_35\n", stderr: "" }), }; - const host: DeviceHost = { + const host: DeviceHost.DeviceHost["Service"] = { id: LOCAL_DEVICE_HOST_ID, summary: Effect.succeed({ id: LOCAL_DEVICE_HOST_ID, @@ -102,7 +102,8 @@ const fixture = Effect.fn("fixture")(function* (onBoot: Effect.Effect = Ef starts.push("stop"); }), }; - const service = yield* makeWithHost(host).pipe( + const service = yield* make.pipe( + Effect.provideService(DeviceHost.DeviceHost, host), Effect.provideService( ServerSettingsService, ServerSettingsService.of({ diff --git a/apps/server/src/device/DeviceService.ts b/apps/server/src/device/DeviceService.ts index 4f69a038e32f..ad61d3ebbdca 100644 --- a/apps/server/src/device/DeviceService.ts +++ b/apps/server/src/device/DeviceService.ts @@ -49,7 +49,7 @@ import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstab import { ServerSettingsService } from "../serverSettings.ts"; import { readDeviceDetail, runDeviceAction } from "./DeviceActions.ts"; -import type { AgentDeviceEndpoint, DeviceHost, DeviceHostReady } from "./DeviceHost.ts"; +import * as DeviceHost from "./DeviceHost.ts"; import * as LocalDeviceHost from "./LocalDeviceHost.ts"; /** Origin-relative prefix the hub is proxied under. See DeviceHubProxy. */ @@ -83,12 +83,12 @@ export interface DeviceScreenshot { readonly png: Uint8Array; } -export interface DeviceReadiness extends DeviceHostReady { +export interface DeviceReadiness extends DeviceHost.DeviceHostReady { readonly hostId: DeviceHostId; } export interface DeviceAgentReadiness extends DeviceReadiness { - readonly agentDevice: AgentDeviceEndpoint; + readonly agentDevice: DeviceHost.AgentDeviceEndpoint; } export class DeviceService extends Context.Service< @@ -136,9 +136,8 @@ interface ServiceState { const vendorPrefix = (platform: DevicePlatform) => platform === "ios" ? "/vendor/serve-sim" : "/vendor/serve-emu"; -export const makeWithHost = Effect.fn("DeviceService.makeWithHost")(function* ( - localHost: DeviceHost, -) { +export const make = Effect.gen(function* () { + const localHost = yield* DeviceHost.DeviceHost; const settings = yield* ServerSettingsService; const lifecycleLock = yield* Semaphore.make(1); const readDeviceSettings = settings.getSettings.pipe( @@ -152,7 +151,9 @@ export const makeWithHost = Effect.fn("DeviceService.makeWithHost")(function* ( ), ); const initialSettings = yield* readDeviceSettings; - const hosts: ReadonlyMap = new Map([[localHost.id, localHost]]); + const hosts: ReadonlyMap = new Map([ + [localHost.id, localHost], + ]); const httpClient = (yield* HttpClient.HttpClient).pipe(HttpClient.withScope); const statePubSub = yield* PubSub.unbounded(); const initialHosts = yield* Effect.forEach(hosts.values(), (host) => host.summary); @@ -431,7 +432,7 @@ export const makeWithHost = Effect.fn("DeviceService.makeWithHost")(function* ( state.devices.find((device) => device.hostId === hostId && device.id === deviceId); const ensurePlatform = Effect.fn("DeviceService.ensurePlatform")(function* ( - host: DeviceHost, + host: DeviceHost.DeviceHost["Service"], platform: DevicePlatform, ) { const availability = yield* host.platformAvailability(platform); @@ -712,11 +713,7 @@ export const makeWithHost = Effect.fn("DeviceService.makeWithHost")(function* ( }); }); -const make = Effect.gen(function* () { - return yield* makeWithHost(yield* LocalDeviceHost.make()); -}).pipe(Effect.withSpan("DeviceService.make")); - -export const layer = Layer.effect(DeviceService, make); +export const layer = Layer.effect(DeviceService, make).pipe(Layer.provide(LocalDeviceHost.layer)); /** State stream for WS subscribers: current snapshot first, then every change. */ export const stateStream = (service: DeviceService["Service"]): Stream.Stream => diff --git a/apps/server/src/device/DeviceToolchain.test.ts b/apps/server/src/device/DeviceToolchain.test.ts new file mode 100644 index 000000000000..9f70ca91a8e1 --- /dev/null +++ b/apps/server/src/device/DeviceToolchain.test.ts @@ -0,0 +1,39 @@ +import { expect, it } from "@effect/vitest"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; +import * as ChildProcessSpawner from "effect/unstable/process/ChildProcessSpawner"; + +import * as ProcessRunner from "../processRunner.ts"; +import { ensureDeviceHub, isDeviceHubInstalled } from "./DeviceToolchain.ts"; + +it.effect("failed installation cleans staging and exposes only a safe failure message", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-device-install-" }); + const result = { + code: ChildProcessSpawner.ExitCode(1), + stdout: "", + stderr: "registry rejected https://private:credential@example.test/package", + timedOut: false, + stdoutTruncated: false, + stderrTruncated: false, + stdoutInvalidUtf8: false, + stderrInvalidUtf8: false, + }; + const error = yield* ensureDeviceHub(baseDir).pipe( + Effect.provideService(ProcessRunner.ProcessRunner, { + run: () => Effect.succeed(result), + }), + Effect.flip, + ); + expect(error.message).toBe( + "Installing expo-device-hub failed while running npm install (exit code 1).", + ); + expect(error.cause).toBe(result); + expect(yield* isDeviceHubInstalled(baseDir)).toBe(false); + expect(yield* fs.readDirectory(path.join(baseDir, "tools", "expo-device-hub"))).toEqual([]); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), +); diff --git a/apps/server/src/device/DeviceToolchain.ts b/apps/server/src/device/DeviceToolchain.ts index bba940412f9f..e6b43cd81ee8 100644 --- a/apps/server/src/device/DeviceToolchain.ts +++ b/apps/server/src/device/DeviceToolchain.ts @@ -50,7 +50,6 @@ export class DeviceToolchainInstallError extends Schema.TaggedError { + ): Effect.fn.Return { yield* reapStaleHub; yield* fs .makeDirectory(agentDeviceStateDir(path, config.stateDir), { recursive: true }) @@ -316,10 +310,9 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () { const port = yield* net.reserveLoopbackPort("127.0.0.1").pipe( Effect.mapError( (cause) => - new DeviceHostError({ + new DeviceHost.DeviceHostError({ hostId, step: "reserving a port for the device hub", - detail: cause.message, cause, }), ), @@ -352,10 +345,9 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () { Effect.provideService(Scope.Scope, scope), Effect.mapError( (cause) => - new DeviceHostError({ + new DeviceHost.DeviceHostError({ hostId, step: "starting the device hub", - detail: String(cause), cause, }), ), @@ -368,10 +360,9 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () { path: "/readyz", timeoutMs: HUB_READY_TIMEOUT_MS, makeError: (info) => - new DeviceHostError({ + new DeviceHost.DeviceHostError({ hostId, step: "waiting for the device hub to answer", - detail: `No response from ${info.requestUrl} after ${info.attempt} attempts.`, cause: info.cause, }), }).pipe( @@ -442,7 +433,7 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () { */ const startAgentDeviceDaemon = Effect.fn("LocalDeviceHost.startAgentDeviceDaemon")(function* ( agentTool: DeviceToolPaths, - ): Effect.fn.Return { + ): Effect.fn.Return { const stateDir = agentDeviceStateDir(path, config.stateDir); yield* fs.makeDirectory(stateDir, { recursive: true }).pipe(Effect.ignore); const existing = yield* readDaemonFile().pipe(Effect.option); @@ -457,7 +448,9 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () { FORCE_COLOR: "0", NO_COLOR: "1", }; - const toEndpoint = (file: typeof AgentDeviceDaemonFile.Type): AgentDeviceEndpoint => ({ + const toEndpoint = ( + file: typeof AgentDeviceDaemonFile.Type, + ): DeviceHost.AgentDeviceEndpoint => ({ baseUrl: `http://127.0.0.1:${file.httpPort}`, token: file.token, entryPath: agentTool.entryPath, @@ -492,10 +485,9 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () { const file = yield* readDaemonFile().pipe(Effect.option); if (file._tag === "Some") return toEndpoint(file.value); if ((yield* Clock.currentTimeMillis) > deadline) { - return yield* new DeviceHostError({ + return yield* new DeviceHost.DeviceHostTimeoutError({ hostId, - step: "starting the agent-device daemon", - detail: `daemon.json did not appear in ${stateDir}.`, + timeoutMs: DAEMON_READY_TIMEOUT_MS, }); } yield* Effect.sleep(Duration.millis(DAEMON_POLL_MS)); @@ -525,7 +517,7 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () { const ensureHubReady = Effect.fn("LocalDeviceHost.ensureHubReady")(function* ( onPhase: (phase: "installing" | "starting") => Effect.Effect, - ): Effect.fn.Return { + ): Effect.fn.Return { const running = yield* Ref.get(runningRef); if (running) { const alive = yield* running.hub.child.isRunning.pipe(Effect.orElseSucceed(() => false)); @@ -543,10 +535,9 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () { Effect.provideService(ProcessRunner.ProcessRunner, runner), Effect.mapError( (cause) => - new DeviceHostError({ + new DeviceHost.DeviceHostError({ hostId, - step: `installing ${cause.tool}`, - detail: cause.message, + step: "installing device support", cause, }), ), @@ -572,14 +563,14 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () { return next; }); - const ensureReady: DeviceHost["ensureReady"] = (onPhase) => + const ensureReady: DeviceHost.DeviceHost["Service"]["ensureReady"] = (onPhase) => startLock.withPermits(1)(ensureHubReady(onPhase).pipe(Effect.map(toReady))); - const ensureAgentReady: DeviceHost["ensureAgentReady"] = (onPhase) => + const ensureAgentReady: DeviceHost.DeviceHost["Service"]["ensureAgentReady"] = (onPhase) => startLock.withPermits(1)( Effect.gen(function* (): Generator< - Effect.Effect, - DeviceHostAgentReady + Effect.Effect, + DeviceHost.DeviceHostAgentReady > { const running = yield* ensureHubReady(onPhase); if (running.agentDevice) return { ...toReady(running), agentDevice: running.agentDevice }; @@ -594,10 +585,9 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () { Effect.provideService(ProcessRunner.ProcessRunner, runner), Effect.mapError( (cause) => - new DeviceHostError({ + new DeviceHost.DeviceHostError({ hostId, - step: `installing ${cause.tool}`, - detail: cause.message, + step: "installing agent tools", cause, }), ), @@ -626,7 +616,7 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () { }; }; - const run: DeviceHostReady["run"] = (command, args, options) => + const run: DeviceHost.DeviceHostReady["run"] = (command, args, options) => runner .run({ command: @@ -652,17 +642,17 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () { Effect.catch((cause) => Effect.succeed({ stdout: "", stderr: String(cause), code: 127 })), ); - const toReady = (running: RunningHost): DeviceHostReady => ({ - hub: { origin: running.hub.origin } satisfies DeviceHubEndpoint, + const toReady = (running: RunningHost): DeviceHost.DeviceHostReady => ({ + hub: { origin: running.hub.origin } satisfies DeviceHost.DeviceHubEndpoint, run, helpers: running.helpers, }); - const current: DeviceHost["current"] = Ref.get(runningRef).pipe( + const current: DeviceHost.DeviceHost["Service"]["current"] = Ref.get(runningRef).pipe( Effect.map((running) => (running ? toReady(running) : null)), ); - const stopAgent: DeviceHost["stopAgent"] = startLock.withPermits(1)( + const stopAgent: DeviceHost.DeviceHost["Service"]["stopAgent"] = startLock.withPermits(1)( Effect.gen(function* () { yield* stopAgentDeviceDaemon(agentToolRef); yield* Ref.update(runningRef, (running) => @@ -671,7 +661,7 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () { }), ); - const stop: DeviceHost["stop"] = startLock.withPermits(1)( + const stop: DeviceHost.DeviceHost["Service"]["stop"] = startLock.withPermits(1)( Effect.gen(function* () { const running = yield* Ref.getAndSet(runningRef, null); yield* stopHub(running?.hub); @@ -683,7 +673,7 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () { // Never leave the hub or daemon behind when the server's scope closes. yield* Effect.addFinalizer(() => stop); - const host: DeviceHost = { + const host: DeviceHost.DeviceHost["Service"] = { id: hostId, summary, platformAvailability, @@ -696,6 +686,8 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () { return host; }); +export const layer = Layer.effect(DeviceHost.DeviceHost, make()); + /** Exposed for tests. */ export const __testing = { AgentDeviceDaemonFile, From 7ecbdbd8e7d8dde1448a73715d3ed444a314e67d Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Tue, 8 Sep 2026 14:31:49 -0700 Subject: [PATCH 25/28] refactor(devices): compose the local host at the service layer --- apps/server/src/device/DeviceService.ts | 4 +++- apps/server/src/device/LocalDeviceHost.ts | 3 --- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/apps/server/src/device/DeviceService.ts b/apps/server/src/device/DeviceService.ts index ad61d3ebbdca..ed1a11f85df4 100644 --- a/apps/server/src/device/DeviceService.ts +++ b/apps/server/src/device/DeviceService.ts @@ -713,7 +713,9 @@ export const make = Effect.gen(function* () { }); }); -export const layer = Layer.effect(DeviceService, make).pipe(Layer.provide(LocalDeviceHost.layer)); +export const layer = Layer.effect(DeviceService, make).pipe( + Layer.provide(Layer.effect(DeviceHost.DeviceHost, LocalDeviceHost.make())), +); /** State stream for WS subscribers: current snapshot first, then every change. */ export const stateStream = (service: DeviceService["Service"]): Stream.Stream => diff --git a/apps/server/src/device/LocalDeviceHost.ts b/apps/server/src/device/LocalDeviceHost.ts index b24ebfbd98b6..92622b7e24e3 100644 --- a/apps/server/src/device/LocalDeviceHost.ts +++ b/apps/server/src/device/LocalDeviceHost.ts @@ -25,7 +25,6 @@ import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; import * as FileSystem from "effect/FileSystem"; -import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; import * as Ref from "effect/Ref"; import * as Schema from "effect/Schema"; @@ -686,8 +685,6 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () { return host; }); -export const layer = Layer.effect(DeviceHost.DeviceHost, make()); - /** Exposed for tests. */ export const __testing = { AgentDeviceDaemonFile, From 064f3c95ca60937357152d192b072d44765019b7 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Tue, 8 Sep 2026 14:40:16 -0700 Subject: [PATCH 26/28] fix(devices): normalize public failures and preserve error causes --- apps/server/src/device/AgentDeviceShim.ts | 6 +- apps/server/src/device/DeviceActions.test.ts | 12 ++-- apps/server/src/device/DeviceActions.ts | 71 ++++++++++++++----- apps/server/src/device/DeviceHubProxy.test.ts | 44 +++++++++--- apps/server/src/device/DeviceHubProxy.ts | 26 +++---- apps/server/src/device/DeviceService.test.ts | 33 ++++++++- apps/server/src/device/DeviceService.ts | 53 +++++++++----- .../src/provider/Layers/ProviderService.ts | 8 ++- packages/contracts/src/device.ts | 47 ++++++++++-- 9 files changed, 231 insertions(+), 69 deletions(-) diff --git a/apps/server/src/device/AgentDeviceShim.ts b/apps/server/src/device/AgentDeviceShim.ts index d4f606d3841b..c2a28127e31d 100644 --- a/apps/server/src/device/AgentDeviceShim.ts +++ b/apps/server/src/device/AgentDeviceShim.ts @@ -14,10 +14,10 @@ const SHIM_DIR = "device/bin"; export const ensureAgentDeviceShim = Effect.fn("AgentDeviceShim.ensure")(function* (input: { readonly entryPath: string; readonly stateDir: string; - readonly fs: FileSystem.FileSystem; - readonly path: Path.Path; }) { - const { fs, path, entryPath } = input; + const { entryPath } = input; + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; const platform = yield* HostProcessPlatform; const shimDir = path.join(input.stateDir, SHIM_DIR); yield* fs.makeDirectory(shimDir, { recursive: true }); diff --git a/apps/server/src/device/DeviceActions.test.ts b/apps/server/src/device/DeviceActions.test.ts index a2719d8cc4b7..ed6b5a87335d 100644 --- a/apps/server/src/device/DeviceActions.test.ts +++ b/apps/server/src/device/DeviceActions.test.ts @@ -163,8 +163,8 @@ describe("runDeviceAction", () => { value: "grayscale", }), ); - expect(error._tag).toBe("DeviceOperationError"); - expect(error.detail).toContain("accessibility helper"); + expect(error._tag).toBe("DeviceActionUnavailableError"); + expect(error.message).toContain("requires a helper"); }), ); @@ -179,7 +179,7 @@ describe("runDeviceAction", () => { payload: "hi", }), ); - expect(error.detail).toContain("not supported on android"); + expect(error.message).toContain("not supported on android"); expect(calls).toEqual([]); }), ); @@ -191,7 +191,11 @@ describe("runDeviceAction", () => { runDeviceAction(ready, "ios", { type: "setAppearance", deviceId: udid, value: "dark" }), ); expect(error.operation).toBe("appearance"); - expect(error.detail).toBe("Invalid device: SIM-1"); + expect(error.message).toContain("exit code 1"); + expect(error.message).not.toContain("Invalid device: SIM-1"); + expect(error._tag === "DeviceOperationError" && error.cause).toMatchObject({ + stderr: "Invalid device: SIM-1", + }); }), ); diff --git a/apps/server/src/device/DeviceActions.ts b/apps/server/src/device/DeviceActions.ts index df284f4b1839..12781b31b4b5 100644 --- a/apps/server/src/device/DeviceActions.ts +++ b/apps/server/src/device/DeviceActions.ts @@ -17,6 +17,7 @@ import { type DeviceActionType, type DeviceForegroundApp, DeviceOperationError, + DeviceActionUnavailableError, type DeviceOrientation, type DevicePlatform, type DeviceSettings, @@ -83,13 +84,17 @@ export const supportsAction = (platform: DevicePlatform, input: DeviceActionInpu return true; }; -const fail = (operation: string, detail: string) => - new DeviceOperationError({ operation, detail: detail.trim() || "command failed" }); - const ok = (operation: string) => (result: { code: number; stderr: string; stdout: string }) => result.code === 0 ? Effect.succeed(result.stdout) - : Effect.fail(fail(operation, result.stderr || result.stdout)); + : Effect.fail( + new DeviceOperationError({ + operation, + reason: "command_failed", + exitCode: result.code, + cause: result, + }), + ); // iOS text-size categories in ascending order; the four shared steps index // into it. `default` is what a fresh simulator reports ("large"). @@ -181,7 +186,11 @@ export const runDeviceAction = Effect.fn("DeviceActions.run")(function* ( input: DeviceActionInput, ) { if (!supportsAction(platform, input)) { - return yield* fail(input.type, `${input.type} is not supported on ${platform}.`); + return yield* new DeviceActionUnavailableError({ + operation: input.type, + platform, + reason: "unsupported", + }); } if (platform === "ios") return yield* runIos(ready, input); return yield* runAndroid(ready.run, input); @@ -196,10 +205,11 @@ const axSettings = (ready: DeviceHostReady, udid: string, args: ReadonlyArray fail("push", String(cause))), + Effect.mapError( + (cause) => + new DeviceOperationError({ operation: "push", reason: "invalid_payload", cause }), + ), ); yield* run("xcrun", ["simctl", "push", udid, input.appId, "-"], { stdin: encoded }).pipe( Effect.flatMap(ok("push")), @@ -287,7 +305,11 @@ const runIos = Effect.fn("DeviceActions.runIos")(function* ( } case "shake": case "setOrientation": - return yield* fail(input.type, `${input.type} is not supported on iOS.`); + return yield* new DeviceActionUnavailableError({ + operation: input.type, + platform: "ios", + reason: "unsupported", + }); } }); @@ -298,7 +320,12 @@ const serveSimPermissions = ( ) => Effect.gen(function* () { const cli = ready.helpers.serveSimCli; - if (!cli) return yield* fail("permission", "serve-sim's CLI is missing from this install."); + if (!cli) + return yield* new DeviceActionUnavailableError({ + operation: "permission", + platform: "ios", + reason: "helper_missing", + }); yield* ready .run(process.execPath, [ cli, @@ -350,7 +377,11 @@ const runAndroid = Effect.fn("DeviceActions.runAndroid")(function* ( } return; } - return yield* fail(input.type, `${input.setting} is not supported on Android.`); + return yield* new DeviceActionUnavailableError({ + operation: input.type, + platform: "android", + reason: "unsupported", + }); case "setOrientation": { // `user-rotation lock` only rotates window content on recent images; // the display the encoder captures stays put. Tilting the emulator's @@ -388,7 +419,11 @@ const runAndroid = Effect.fn("DeviceActions.runAndroid")(function* ( case "setPermission": { const permissions = ANDROID_PERMISSIONS[input.permission]; if (!permissions) { - return yield* fail("permission", `${input.permission} has no Android equivalent.`); + return yield* new DeviceActionUnavailableError({ + operation: "permission", + platform: "android", + reason: "unsupported", + }); } const verb = input.decision === "grant" ? "grant" : "revoke"; for (const permission of permissions) { @@ -416,7 +451,11 @@ const runAndroid = Effect.fn("DeviceActions.runAndroid")(function* ( case "setColorFilter": case "shake": case "sendPush": - return yield* fail(input.type, `${input.type} is not supported on Android.`); + return yield* new DeviceActionUnavailableError({ + operation: input.type, + platform: "android", + reason: "unsupported", + }); } }); diff --git a/apps/server/src/device/DeviceHubProxy.test.ts b/apps/server/src/device/DeviceHubProxy.test.ts index 7ba51f079928..0f039274e207 100644 --- a/apps/server/src/device/DeviceHubProxy.test.ts +++ b/apps/server/src/device/DeviceHubProxy.test.ts @@ -9,7 +9,13 @@ import { import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import { HttpClient, HttpClientResponse, HttpRouter } from "effect/unstable/http"; -import { EnvironmentAuth } from "../auth/EnvironmentAuth.ts"; +import { + EnvironmentAuth, + ServerAuthMissingCredentialError, + ServerAuthSessionCredentialValidationError, + type ServerAuthCredentialError, + type ServerAuthInternalError, +} from "../auth/EnvironmentAuth.ts"; import { DeviceService } from "./DeviceService.ts"; import { deviceHubProxyRouteLayer } from "./DeviceHubProxy.ts"; @@ -18,7 +24,11 @@ afterEach(async () => { for (const dispose of disposers.splice(0)) await dispose(); }); -const fixture = (scopes: ReadonlyArray, fail = false) => { +const fixture = ( + scopes: ReadonlyArray, + fail = false, + authError?: ServerAuthCredentialError | ServerAuthInternalError, +) => { let finalized = 0; const requests: string[] = []; const client = HttpClient.make((request, _url, signal) => @@ -36,12 +46,14 @@ const fixture = (scopes: ReadonlyArray, fail = false) => { Layer.provideMerge( Layer.succeed(EnvironmentAuth, { authenticateWebSocketUpgrade: () => - Effect.succeed({ - sessionId: AuthSessionId.make("test"), - subject: "test", - method: "bearer-access-token", - scopes, - }), + authError + ? Effect.fail(authError) + : Effect.succeed({ + sessionId: AuthSessionId.make("test"), + subject: "test", + method: "bearer-access-token", + scopes, + }), } as unknown as EnvironmentAuth["Service"]), ), Layer.provideMerge( @@ -111,3 +123,19 @@ describe("device hub proxy", () => { expect(requests).toEqual([]); }); }); + +it.each([ + [new ServerAuthMissingCredentialError({}), 401], + [ + new ServerAuthSessionCredentialValidationError({ + cause: new Error("private credential diagnostic"), + }), + 500, + ], +] as const)("translates authentication failure to HTTP %s", async (error, status) => { + const { handler, requests } = fixture([], false, error); + const response = await handler(new Request("http://t3.test/api/device-hub/api/devices")); + expect(response.status).toBe(status); + expect(await response.text()).not.toContain("private credential diagnostic"); + expect(requests).toEqual([]); +}); diff --git a/apps/server/src/device/DeviceHubProxy.ts b/apps/server/src/device/DeviceHubProxy.ts index c18d077631df..f8685727c882 100644 --- a/apps/server/src/device/DeviceHubProxy.ts +++ b/apps/server/src/device/DeviceHubProxy.ts @@ -34,7 +34,7 @@ import { failEnvironmentInternal, failEnvironmentScopeRequired, } from "../auth/http.ts"; -import { DEVICE_HUB_ROUTE_PREFIX, DeviceService } from "./DeviceService.ts"; +import * as DeviceService from "./DeviceService.ts"; const ALLOWED_PATHS: ReadonlyArray = [ /^\/api\/devices$/, @@ -90,14 +90,16 @@ const authenticate = (requiredScope: AuthEnvironmentScope) => const request = yield* HttpServerRequest.HttpServerRequest; const serverAuth = yield* EnvironmentAuth.EnvironmentAuth; const session = yield* serverAuth.authenticateWebSocketUpgrade(request).pipe( - Effect.catchIf(EnvironmentAuth.isServerAuthCredentialError, (error) => - failEnvironmentAuthInvalid( - EnvironmentAuth.serverAuthCredentialReason(error), - EnvironmentAuth.serverAuthDpopFailureReason(error), - ), - ), - Effect.catchIf(EnvironmentAuth.isServerAuthInternalError, (error) => - failEnvironmentInternal("internal_error", error), + Effect.catch((error) => + Effect.gen(function* () { + if (EnvironmentAuth.isServerAuthCredentialError(error)) { + return yield* failEnvironmentAuthInvalid( + EnvironmentAuth.serverAuthCredentialReason(error), + EnvironmentAuth.serverAuthDpopFailureReason(error), + ); + } + return yield* failEnvironmentInternal("internal_error", error); + }), ), ); if (!session.scopes.includes(requiredScope)) { @@ -177,7 +179,7 @@ const handler = Effect.gen(function* () { if (Option.isNone(url)) { return HttpServerResponse.text("Bad Request", { status: 400 }); } - const hubPath = url.value.pathname.slice(DEVICE_HUB_ROUTE_PREFIX.length) || "/"; + const hubPath = url.value.pathname.slice(DeviceService.DEVICE_HUB_ROUTE_PREFIX.length) || "/"; const upgrade = isWebSocketUpgrade(request); const allowed = (upgrade ? ALLOWED_WS_PATHS : ALLOWED_PATHS).some((pattern) => pattern.test(hubPath), @@ -193,7 +195,7 @@ const handler = Effect.gen(function* () { (upgrade && hubPath !== "/api/devices/ws") || (!readOnly && /\/api\/stream-(mode|settings)$/.test(hubPath)); yield* authenticate(controlsDevice ? AuthOrchestrationOperateScope : AuthOrchestrationReadScope); - const devices = yield* DeviceService; + const devices = yield* DeviceService.DeviceService; const ready = yield* devices.currentReadiness(); if (!ready) { return HttpServerResponse.text("Device hub is not running", { status: 503 }); @@ -217,6 +219,6 @@ const handler = Effect.gen(function* () { export const deviceHubProxyRouteLayer = HttpRouter.add( "*", - `${DEVICE_HUB_ROUTE_PREFIX}/*`, + `${DeviceService.DEVICE_HUB_ROUTE_PREFIX}/*`, handler, ); diff --git a/apps/server/src/device/DeviceService.test.ts b/apps/server/src/device/DeviceService.test.ts index f7b7ece1d679..562e8a05a315 100644 --- a/apps/server/src/device/DeviceService.test.ts +++ b/apps/server/src/device/DeviceService.test.ts @@ -56,7 +56,10 @@ describe("DeviceService.stateStream", () => { ); }); -const fixture = Effect.fn("fixture")(function* (onBoot: Effect.Effect = Effect.void) { +const fixture = Effect.fn("fixture")(function* ( + onBoot: Effect.Effect = Effect.void, + bootError?: string, +) { const settings = yield* Ref.make(DEFAULT_SERVER_SETTINGS); const starts: string[] = []; const agentStarts: string[] = []; @@ -139,7 +142,9 @@ const fixture = Effect.fn("fixture")(function* (onBoot: Effect.Effect = Ef booted = true; return HttpClientResponse.fromWeb( request, - Response.json({ ok: true, serial: "emulator-5554" }), + Response.json( + bootError ? { ok: false, error: bootError } : { ok: true, serial: "emulator-5554" }, + ), ); } return HttpClientResponse.fromWeb( @@ -276,3 +281,27 @@ describe("device discovery after server restart", () => { }).pipe(Effect.scoped), ); }); + +for (const [diagnostic, reason, message] of [ + ["Insufficient disk space at /private/user/path", "disk_space", "not enough free disk space"], + ["Timed out spawning /private/user/command", "timeout", "did not become ready in time"], + ["Unexpected failure: secret-token", "launch_failed", "could not start"], +] as const) { + it.effect(`normalizes boot failure: ${reason}`, () => + Effect.gen(function* () { + const { service } = yield* fixture(Effect.void, diagnostic); + yield* service.configure({ enabled: true }); + const error = yield* service + .open({ + threadId: ThreadId.make("boot-failure"), + deviceId: "Pixel_API_35", + platform: "android", + }) + .pipe(Effect.flip); + expect(error._tag).toBe("DeviceBootError"); + expect(error.message).toContain(message); + expect(error.message).not.toContain(diagnostic); + expect((yield* service.state).bootingDevices).toEqual([]); + }).pipe(Effect.scoped), + ); +} diff --git a/apps/server/src/device/DeviceService.ts b/apps/server/src/device/DeviceService.ts index ed1a11f85df4..38abd11abff5 100644 --- a/apps/server/src/device/DeviceService.ts +++ b/apps/server/src/device/DeviceService.ts @@ -147,7 +147,8 @@ export const make = Effect.gen(function* () { onboardingCompleted: value.deviceOnboardingCompleted, })), Effect.mapError( - (cause) => new DeviceOperationError({ operation: "settings", detail: cause.message }), + (cause) => + new DeviceOperationError({ operation: "settings", reason: "settings_failed", cause }), ), ); const initialSettings = yield* readDeviceSettings; @@ -279,7 +280,6 @@ export const make = Effect.gen(function* () { ); const hubJson = ( - ready: DeviceReadiness, request: HttpClientRequest.HttpClientRequest, schema: Schema.Codec, operation: string, @@ -294,14 +294,14 @@ export const make = Effect.gen(function* () { (cause) => new DeviceOperationError({ operation, - detail: `${ready.hub.origin}: ${cause instanceof Error ? cause.message : String(cause)}`, + reason: "request_failed", + cause, }), ), ); const fetchDevices = Effect.fn("DeviceService.fetchDevices")(function* (ready: DeviceReadiness) { const list = yield* hubJson( - ready, HttpClientRequest.get(`${ready.hub.origin}/api/devices`), HubDeviceList, "list", @@ -322,7 +322,9 @@ export const make = Effect.gen(function* () { if (avds.code !== 0) { return yield* new DeviceOperationError({ operation: "list", - detail: `Could not list Android virtual devices: ${avds.stderr || avds.stdout}`, + reason: "command_failed", + exitCode: avds.code, + cause: avds, }); } for (const name of avds.stdout @@ -397,7 +399,11 @@ export const make = Effect.gen(function* () { .pipe( Effect.mapError( (cause) => - new DeviceOperationError({ operation: "configure", detail: cause.message }), + new DeviceOperationError({ + operation: "configure", + reason: "settings_failed", + cause, + }), ), ); if (!nextEnabled) { @@ -457,15 +463,23 @@ export const make = Effect.gen(function* () { const result = yield* HttpClientRequest.post(`${ready.hub.origin}/api/devices/boot`).pipe( HttpClientRequest.bodyJson({ platform: device.platform, id: device.id, name: device.name }), Effect.mapError( - (cause) => new DeviceOperationError({ operation: "boot", detail: String(cause) }), + (cause) => + new DeviceOperationError({ operation: "boot", reason: "invalid_payload", cause }), ), - Effect.flatMap((request) => hubJson(ready, request, HubActionResult, "boot", BOOT_TIMEOUT)), + Effect.flatMap((request) => hubJson(request, HubActionResult, "boot", BOOT_TIMEOUT)), ); if (!result.ok) { return yield* new DeviceBootError({ hostId: ready.hostId, deviceId: device.id, - detail: result.error ?? "The device hub reported a boot failure.", + reason: /insufficient.*(?:disk|space)|not enough.*(?:disk|space)|no space left/i.test( + result.error ?? "", + ) + ? "disk_space" + : /timed? out|timeout/i.test(result.error ?? "") + ? "timeout" + : "launch_failed", + cause: result, }); } if (device.platform === "ios") { @@ -476,10 +490,11 @@ export const make = Effect.gen(function* () { ).pipe( HttpClientRequest.bodyJson({ udid: device.id }), Effect.mapError( - (cause) => new DeviceOperationError({ operation: "boot", detail: String(cause) }), + (cause) => + new DeviceOperationError({ operation: "boot", reason: "invalid_payload", cause }), ), Effect.flatMap((request) => - hubJson(ready, request, HubActionResult, "attach stream", BOOT_TIMEOUT), + hubJson(request, HubActionResult, "attach stream", BOOT_TIMEOUT), ), ); } @@ -528,10 +543,11 @@ export const make = Effect.gen(function* () { ).pipe( HttpClientRequest.bodyJson({ udid: device.id }), Effect.mapError( - (cause) => new DeviceOperationError({ operation: "open", detail: String(cause) }), + (cause) => + new DeviceOperationError({ operation: "open", reason: "invalid_payload", cause }), ), Effect.flatMap((request) => - hubJson(ready, request, HubActionResult, "attach stream", BOOT_TIMEOUT), + hubJson(request, HubActionResult, "attach stream", BOOT_TIMEOUT), ), ); } @@ -578,16 +594,18 @@ export const make = Effect.gen(function* () { yield* HttpClientRequest.post(`${ready.hub.origin}/api/devices/shutdown`).pipe( HttpClientRequest.bodyJson({ platform, id: deviceId }), Effect.mapError( - (cause) => new DeviceOperationError({ operation: "shutdown", detail: String(cause) }), + (cause) => + new DeviceOperationError({ operation: "shutdown", reason: "invalid_payload", cause }), ), - Effect.flatMap((request) => hubJson(ready, request, HubActionResult, "shutdown")), + Effect.flatMap((request) => hubJson(request, HubActionResult, "shutdown")), Effect.flatMap((result) => result.ok ? Effect.void : Effect.fail( new DeviceOperationError({ operation: "shutdown", - detail: result.error ?? "The device hub reported a shutdown failure.", + reason: "hub_rejected", + cause: result, }), ), ), @@ -646,7 +664,8 @@ export const make = Effect.gen(function* () { (cause) => new DeviceOperationError({ operation: "screenshot", - detail: cause instanceof Error ? cause.message : String(cause), + reason: "request_failed", + cause, }), ), ); diff --git a/apps/server/src/provider/Layers/ProviderService.ts b/apps/server/src/provider/Layers/ProviderService.ts index 3b185b2d3574..2fb58568f08f 100644 --- a/apps/server/src/provider/Layers/ProviderService.ts +++ b/apps/server/src/provider/Layers/ProviderService.ts @@ -945,9 +945,11 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( const shimDir = yield* ensureAgentDeviceShim({ entryPath: readiness.agentDevice.entryPath, stateDir: serverConfig.stateDir, - fs: fileSystem, - path: pathService, - }).pipe(Effect.orElseSucceed(() => undefined)); + }).pipe( + Effect.provideService(FileSystem.FileSystem, fileSystem), + Effect.provideService(Path.Path, pathService), + Effect.orElseSucceed(() => undefined), + ); if (!shimDir) return undefined; return { PATH: shimDir, diff --git a/packages/contracts/src/device.ts b/packages/contracts/src/device.ts index be896ee45a99..309d156430fa 100644 --- a/packages/contracts/src/device.ts +++ b/packages/contracts/src/device.ts @@ -346,10 +346,17 @@ export class DeviceNotFoundError extends Schema.TaggedError export class DeviceBootError extends Schema.TaggedError()("DeviceBootError", { hostId: DeviceHostId, deviceId: DeviceId, - detail: Schema.String, + reason: Schema.Literals(["disk_space", "timeout", "launch_failed"]), + cause: Schema.Defect(), }) { override get message(): string { - return `Device ${this.deviceId} failed to boot: ${this.detail}`; + const explanation = { + disk_space: "There is not enough free disk space on the environment server.", + timeout: "The device did not become ready in time.", + launch_failed: + "The simulator or emulator could not start. Check its configuration on the environment server.", + }[this.reason]; + return `Device ${this.deviceId} failed to boot: ${explanation}`; } } @@ -357,11 +364,41 @@ export class DeviceOperationError extends Schema.TaggedError()( + "DeviceActionUnavailableError", + { + operation: Schema.String, + platform: DevicePlatform, + reason: Schema.Literals(["unsupported", "helper_missing"]), }, ) { override get message(): string { - return `Device ${this.operation} failed: ${this.detail}`; + return this.reason === "helper_missing" + ? `Device ${this.operation} requires a helper missing from this install. Set up device support again.` + : `Device ${this.operation} is not supported on ${this.platform}.`; } } @@ -371,6 +408,7 @@ export const DeviceError = Schema.Union([ DeviceNotFoundError, DeviceBootError, DeviceOperationError, + DeviceActionUnavailableError, ]); export type DeviceError = typeof DeviceError.Type; @@ -473,5 +511,6 @@ export const DeviceToolError = Schema.Union([ DeviceNotFoundError, DeviceBootError, DeviceOperationError, + DeviceActionUnavailableError, ]); export type DeviceToolError = typeof DeviceToolError.Type; From 2b221aff8d5a3da2766c1ca11b54ab860c4508c7 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Tue, 8 Sep 2026 14:45:30 -0700 Subject: [PATCH 27/28] fix(devices): preserve shutdown success when discovery is unavailable --- apps/server/src/device/DeviceService.test.ts | 30 +++++++++++++++++++ apps/server/src/device/DeviceService.ts | 23 ++++++++++++-- .../components/device/DeviceStreamView.tsx | 6 ++-- 3 files changed, 52 insertions(+), 7 deletions(-) diff --git a/apps/server/src/device/DeviceService.test.ts b/apps/server/src/device/DeviceService.test.ts index 562e8a05a315..ed42be64eb02 100644 --- a/apps/server/src/device/DeviceService.test.ts +++ b/apps/server/src/device/DeviceService.test.ts @@ -59,6 +59,7 @@ describe("DeviceService.stateStream", () => { const fixture = Effect.fn("fixture")(function* ( onBoot: Effect.Effect = Effect.void, bootError?: string, + failListAfterShutdown = false, ) { const settings = yield* Ref.make(DEFAULT_SERVER_SETTINGS); const starts: string[] = []; @@ -66,6 +67,7 @@ const fixture = Effect.fn("fixture")(function* ( const agentStops: string[] = []; const requests: string[] = []; let booted = false; + let shutDown = false; const ready: DeviceHost.DeviceHostReady = { hub: { origin: "http://device.test" }, helpers: { serveSimAxSettings: null, serveSimCli: null }, @@ -137,6 +139,17 @@ const fixture = Effect.fn("fixture")(function* ( new Response(new Uint8Array([137, 80, 78, 71])), ); } + if (request.url.endsWith("/shutdown")) { + shutDown = true; + booted = false; + return HttpClientResponse.fromWeb(request, Response.json({ ok: true })); + } + if (shutDown && failListAfterShutdown) { + return HttpClientResponse.fromWeb( + request, + new Response("Discovery busy", { status: 503 }), + ); + } if (request.url.endsWith("/boot")) { yield* onBoot; booted = true; @@ -305,3 +318,20 @@ for (const [diagnostic, reason, message] of [ }).pipe(Effect.scoped), ); } + +it.effect("keeps shutdown successful when subsequent discovery fails", () => + Effect.gen(function* () { + const { service } = yield* fixture(Effect.void, undefined, true); + yield* service.configure({ enabled: true }); + const threadId = ThreadId.make("shutdown-refresh"); + const session = yield* service.open({ + threadId, + deviceId: "Pixel_API_35", + platform: "android", + }); + yield* service.close({ threadId, deviceId: session.deviceId, shutdown: true }); + const state = yield* service.state; + expect(state.sessions).toEqual([]); + expect(state.devices.find((device) => device.id === session.deviceId)?.booted).toBe(false); + }).pipe(Effect.scoped), +); diff --git a/apps/server/src/device/DeviceService.ts b/apps/server/src/device/DeviceService.ts index 38abd11abff5..b998ec0fd143 100644 --- a/apps/server/src/device/DeviceService.ts +++ b/apps/server/src/device/DeviceService.ts @@ -46,7 +46,7 @@ import * as Stream from "effect/Stream"; import * as SynchronizedRef from "effect/SynchronizedRef"; import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"; -import { ServerSettingsService } from "../serverSettings.ts"; +import * as ServerSettings from "../serverSettings.ts"; import { readDeviceDetail, runDeviceAction } from "./DeviceActions.ts"; import * as DeviceHost from "./DeviceHost.ts"; @@ -138,7 +138,7 @@ const vendorPrefix = (platform: DevicePlatform) => export const make = Effect.gen(function* () { const localHost = yield* DeviceHost.DeviceHost; - const settings = yield* ServerSettingsService; + const settings = yield* ServerSettings.ServerSettingsService; const lifecycleLock = yield* Semaphore.make(1); const readDeviceSettings = settings.getSettings.pipe( Effect.map((value) => ({ @@ -610,7 +610,24 @@ export const make = Effect.gen(function* () { ), ), ); - yield* refresh(ready); + yield* publish((state) => ({ + ...state, + devices: state.devices.map((device) => + device.hostId === ready.hostId && device.id === deviceId + ? { ...device, booted: false } + : device, + ), + sessions: state.sessions.filter( + (session) => !(session.hostId === ready.hostId && session.deviceId === deviceId), + ), + })); + // Discovery can stall while an emulator saves its snapshot. A failed + // refresh must not turn an accepted shutdown into an action failure. + yield* refresh(ready).pipe( + Effect.catch((cause) => + Effect.logWarning("Device discovery unavailable after shutdown", { cause }), + ), + ); }); const close: DeviceService["Service"]["close"] = Effect.fn("DeviceService.close")( diff --git a/apps/web/src/components/device/DeviceStreamView.tsx b/apps/web/src/components/device/DeviceStreamView.tsx index cb70a98476d6..15fc96cbb2e7 100644 --- a/apps/web/src/components/device/DeviceStreamView.tsx +++ b/apps/web/src/components/device/DeviceStreamView.tsx @@ -185,8 +185,6 @@ export function DeviceStreamView(props: { : { width: frame.width, height: frame.height, - left: 0, - top: 0, ...(rotation ? { transform: `rotate(${rotation}deg)` } : {}), }; @@ -272,7 +270,7 @@ export function DeviceStreamView(props: { > {props.visible && access && mjpegUrl ? ( @@ -281,7 +279,7 @@ export function DeviceStreamView(props: { src={mjpegUrl} alt="" draggable={false} - className="absolute object-contain" + className="absolute top-0 left-0 object-contain" style={mediaStyle} /> ) : null} From 834988b6a93ca9672058b6de125fc3b44186c6a4 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Tue, 8 Sep 2026 14:55:47 -0700 Subject: [PATCH 28/28] refactor(devices): keep local host construction and layer together --- apps/server/src/device/DeviceService.ts | 4 +- .../server/src/device/LocalDeviceHost.test.ts | 45 +++++++++++++++++-- apps/server/src/device/LocalDeviceHost.ts | 3 ++ 3 files changed, 46 insertions(+), 6 deletions(-) diff --git a/apps/server/src/device/DeviceService.ts b/apps/server/src/device/DeviceService.ts index b998ec0fd143..6cca6631bd7e 100644 --- a/apps/server/src/device/DeviceService.ts +++ b/apps/server/src/device/DeviceService.ts @@ -749,9 +749,7 @@ export const make = Effect.gen(function* () { }); }); -export const layer = Layer.effect(DeviceService, make).pipe( - Layer.provide(Layer.effect(DeviceHost.DeviceHost, LocalDeviceHost.make())), -); +export const layer = Layer.effect(DeviceService, make).pipe(Layer.provide(LocalDeviceHost.layer)); /** State stream for WS subscribers: current snapshot first, then every change. */ export const stateStream = (service: DeviceService["Service"]): Stream.Stream => diff --git a/apps/server/src/device/LocalDeviceHost.test.ts b/apps/server/src/device/LocalDeviceHost.test.ts index c8ef28ed7e18..17d274e73529 100644 --- a/apps/server/src/device/LocalDeviceHost.test.ts +++ b/apps/server/src/device/LocalDeviceHost.test.ts @@ -2,13 +2,20 @@ import { describe, expect, it } from "@effect/vitest"; import * as NodePath from "@effect/platform-node/NodePath"; import { HostProcessEnvironment, HostProcessPlatform } from "@t3tools/shared/hostProcess"; import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; import * as FileSystem from "effect/FileSystem"; -import { __testing } from "./LocalDeviceHost.ts"; +import * as LocalDeviceHost from "./LocalDeviceHost.ts"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import * as ChildProcessSpawner from "effect/unstable/process/ChildProcessSpawner"; +import { HttpClient } from "effect/unstable/http"; +import * as NetService from "@t3tools/shared/Net"; +import * as ServerConfig from "../config.ts"; +import * as ProcessRunner from "../processRunner.ts"; const diagnose = (files: ReadonlyArray, environment: NodeJS.ProcessEnv) => - __testing.platformReason("android").pipe( + LocalDeviceHost.__testing.platformReason("android").pipe( Effect.provideService(HostProcessEnvironment, environment), Effect.provideService(HostProcessPlatform, "darwin"), Effect.provideService( @@ -62,7 +69,7 @@ describe("Android SDK availability", () => { it.effect("puts detected Android tools on the helper PATH without losing existing commands", () => Effect.gen(function* () { const path = yield* Path.Path; - const environment = __testing.deviceHostEnvironment( + const environment = LocalDeviceHost.__testing.deviceHostEnvironment( { PATH: "/usr/bin", HOME: "/test/home" }, "/sdk", "darwin", @@ -73,3 +80,35 @@ it.effect("puts detected Android tools on the helper PATH without losing existin expect(environment.HOME).toBe("/test/home"); }).pipe(Effect.provide(NodePath.layer)), ); + +it.effect( + "constructs and inspects an unconfigured host without installing or starting helpers", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-device-consent-" }); + const host = yield* LocalDeviceHost.make().pipe( + Effect.provide(Layer.mergeAll(ServerConfig.layerTest(baseDir, baseDir), NetService.layer)), + Effect.provideService(HostProcessEnvironment, { HOME: baseDir, PATH: "" }), + Effect.provideService(HostProcessPlatform, "linux"), + Effect.provideService( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make(() => + Effect.die(new Error("Host construction must not spawn processes")), + ), + ), + Effect.provideService(ProcessRunner.ProcessRunner, { + run: () => Effect.die(new Error("Host construction must not run commands")), + }), + Effect.provideService( + HttpClient.HttpClient, + HttpClient.make(() => + Effect.die(new Error("Host construction must not make network requests")), + ), + ), + ); + expect(yield* host.current).toBeNull(); + yield* host.stop; + expect(yield* fs.exists(`${baseDir}/tools`)).toBe(false); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), +); diff --git a/apps/server/src/device/LocalDeviceHost.ts b/apps/server/src/device/LocalDeviceHost.ts index 92622b7e24e3..b24ebfbd98b6 100644 --- a/apps/server/src/device/LocalDeviceHost.ts +++ b/apps/server/src/device/LocalDeviceHost.ts @@ -25,6 +25,7 @@ import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; import * as Ref from "effect/Ref"; import * as Schema from "effect/Schema"; @@ -685,6 +686,8 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () { return host; }); +export const layer = Layer.effect(DeviceHost.DeviceHost, make()); + /** Exposed for tests. */ export const __testing = { AgentDeviceDaemonFile,