diff --git a/apps/mobile/src/features/threads/thread-work-log.tsx b/apps/mobile/src/features/threads/thread-work-log.tsx index a104d2297cd9..f488e10cdd9b 100644 --- a/apps/mobile/src/features/threads/thread-work-log.tsx +++ b/apps/mobile/src/features/threads/thread-work-log.tsx @@ -76,7 +76,7 @@ export const THREAD_DISCLOSURE_TRANSITION_MS = 180; const WORK_LOG_LAYOUT_TRANSITION = LinearTransition.duration(THREAD_DISCLOSURE_TRANSITION_MS); const WORK_LOG_DETAIL_ENTER_TRANSITION = FadeIn.duration(140); const WORK_LOG_DETAIL_EXIT_TRANSITION = FadeOut.duration(120); -type WorkContentIcon = AppSymbolName | "browser" | "t3-code"; +type WorkContentIcon = AppSymbolName | "browser" | "device" | "t3-code"; function WorkLogIcon(props: { readonly icon: WorkContentIcon; @@ -92,7 +92,13 @@ function WorkLogIcon(props: { } return ( ; stdin?: string }; + +const makeReady = ( + respond: (call: Call) => { stdout?: string; stderr?: string; code?: number } = () => ({}), + helpers: DeviceHostReady["helpers"] = { + serveSimAxSettings: "/hub/simax/serve-sim-ax-settings", + serveSimCli: "/hub/serve-sim.js", + }, +) => { + const calls: Call[] = []; + const ready: DeviceHostReady = { + hub: { origin: "http://127.0.0.1:1" }, + helpers, + run: (command, args, options) => { + const call = { command, args, ...(options?.stdin ? { stdin: options.stdin } : {}) }; + calls.push(call); + const result = respond(call); + return Effect.succeed({ + stdout: result.stdout ?? "", + stderr: result.stderr ?? "", + code: result.code ?? 0, + }); + }, + }; + return { ready, calls }; +}; + +const udid = "SIM-1"; + +describe("supportsAction", () => { + it("advertises platform-specific toggles", () => { + const toggle = (setting: Extract["setting"]) => + ({ type: "setToggle", deviceId: udid, setting, value: true }) as const; + expect(supportsAction("ios", toggle("voiceOver"))).toBe(true); + expect(supportsAction("android", toggle("voiceOver"))).toBe(false); + expect(supportsAction("android", toggle("networkEnabled"))).toBe(true); + expect(supportsAction("ios", toggle("networkEnabled"))).toBe(false); + expect(supportsAction("ios", { type: "setLiquidGlass", deviceId: udid, value: "clear" })).toBe( + true, + ); + expect( + supportsAction("android", { type: "setLiquidGlass", deviceId: udid, value: "clear" }), + ).toBe(false); + expect( + supportsAction("android", { type: "setOrientation", deviceId: udid, value: "portrait" }), + ).toBe(true); + expect( + supportsAction("ios", { type: "setOrientation", deviceId: udid, value: "portrait" }), + ).toBe(false); + }); +}); + +describe("runDeviceAction", () => { + it.effect("maps shared text sizes onto simctl content-size categories", () => + Effect.gen(function* () { + const { ready, calls } = makeReady(); + yield* runDeviceAction(ready, "ios", { + type: "setTextSize", + deviceId: udid, + value: "extra-large", + }); + expect(calls).toEqual([ + { command: "xcrun", args: ["simctl", "ui", udid, "content_size", "accessibility-large"] }, + ]); + }), + ); + + it.effect("maps shared text sizes onto Android font_scale", () => + Effect.gen(function* () { + const { ready, calls } = makeReady(); + yield* runDeviceAction(ready, "android", { + type: "setTextSize", + deviceId: "emulator-5554", + value: "large", + }); + expect(calls[0]?.args).toEqual([ + "-s", + "emulator-5554", + "shell", + "settings", + "put", + "system", + "font_scale", + "1.15", + ]); + }), + ); + + it.effect( + "rotates emulators through the accelerometer and physical devices through the lock", + () => + Effect.gen(function* () { + const emulator = makeReady(); + yield* runDeviceAction(emulator.ready, "android", { + type: "setOrientation", + deviceId: "emulator-5554", + value: "landscape_left", + }); + expect(emulator.calls.at(-1)?.args).toEqual([ + "-s", + "emulator-5554", + "emu", + "sensor", + "set", + "acceleration", + "9.81:0:0", + ]); + const phone = makeReady(); + yield* runDeviceAction(phone.ready, "android", { + type: "setOrientation", + deviceId: "R5CT1234", + value: "landscape_right", + }); + expect(phone.calls).toEqual([ + { + command: "adb", + args: ["-s", "R5CT1234", "shell", "cmd", "window", "user-rotation", "lock", "3"], + }, + ]); + }), + ); + + it.effect("runs accessibility toggles through the bundled helper via simctl spawn", () => + Effect.gen(function* () { + const { ready, calls } = makeReady(); + yield* runDeviceAction(ready, "ios", { + type: "setToggle", + deviceId: udid, + setting: "voiceOver", + value: true, + }); + expect(calls).toEqual([ + { + command: "xcrun", + args: [ + "simctl", + "spawn", + udid, + "/hub/simax/serve-sim-ax-settings", + "set", + "voiceover", + "on", + ], + }, + ]); + }), + ); + + it.effect("fails clearly when the helper is missing", () => + Effect.gen(function* () { + const { ready } = makeReady(() => ({}), { serveSimAxSettings: null, serveSimCli: null }); + const error = yield* Effect.flip( + runDeviceAction(ready, "ios", { + type: "setColorFilter", + deviceId: udid, + value: "grayscale", + }), + ); + expect(error._tag).toBe("DeviceActionUnavailableError"); + expect(error.message).toContain("requires a helper"); + }), + ); + + it.effect("rejects actions the platform does not support without running anything", () => + Effect.gen(function* () { + const { ready, calls } = makeReady(); + const error = yield* Effect.flip( + runDeviceAction(ready, "android", { + type: "sendPush", + deviceId: "emulator-5554", + appId: "com.example", + payload: "hi", + }), + ); + expect(error.message).toContain("not supported on android"); + expect(calls).toEqual([]); + }), + ); + + it.effect("surfaces non-zero exit codes as operation errors", () => + Effect.gen(function* () { + const { ready } = makeReady(() => ({ code: 1, stderr: "Invalid device: SIM-1" })); + const error = yield* Effect.flip( + runDeviceAction(ready, "ios", { type: "setAppearance", deviceId: udid, value: "dark" }), + ); + expect(error.operation).toBe("appearance"); + expect(error.message).toContain("exit code 1"); + expect(error.message).not.toContain("Invalid device: SIM-1"); + expect(error._tag === "DeviceOperationError" && error.cause).toMatchObject({ + stderr: "Invalid device: SIM-1", + }); + }), + ); + + it.effect("wraps a bare push string in an APNs alert and feeds it on stdin", () => + Effect.gen(function* () { + const { ready, calls } = makeReady(); + yield* runDeviceAction(ready, "ios", { + type: "sendPush", + deviceId: udid, + appId: "com.example.app", + payload: "Hello", + }); + expect(calls[0]?.args).toEqual(["simctl", "push", udid, "com.example.app", "-"]); + expect(calls[0]?.stdin).toBe('{"aps":{"alert":"Hello"}}'); + }), + ); +}); + +describe("readDeviceDetail", () => { + it.effect("reads iOS settings from simctl and the accessibility helper", () => + Effect.gen(function* () { + const { ready } = makeReady((call) => { + const key = call.args.join(" "); + if (key.endsWith("ui SIM-1 appearance")) return { stdout: "dark\n" }; + if (key.endsWith("ui SIM-1 content_size")) return { stdout: "extra-extra-large\n" }; + if (key.endsWith("ui SIM-1 increase_contrast")) return { stdout: "enabled\n" }; + if (key.includes("serve-sim-ax-settings status")) { + return { + stdout: + '{"reduce-motion":"on","reduce-transparency":"off","show-borders":"off","voiceover":"off","liquid-glass":"tinted","color-filter":"grayscale"}', + }; + } + return { code: 1 }; + }); + const detail = yield* readDeviceDetail(ready, "ios", udid); + expect(detail.settings).toEqual({ + appearance: "dark", + textSize: "large", + increaseContrast: true, + reduceMotion: true, + reduceTransparency: false, + showBorders: false, + voiceOver: false, + liquidGlass: "tinted", + colorFilter: "grayscale", + }); + }), + ); + + it.effect("degrades unreadable values to unknown instead of failing", () => + Effect.gen(function* () { + const { ready } = makeReady(() => ({ code: 1, stderr: "boom" })); + const detail = yield* readDeviceDetail(ready, "ios", udid); + expect(detail.settings).toEqual({}); + expect(detail.foregroundApp).toBeNull(); + }), + ); + + it.effect("reads Android settings and the focused package", () => + Effect.gen(function* () { + const { ready } = makeReady((call) => { + const key = call.args.join(" "); + if (key.endsWith("cmd uimode night")) return { stdout: "Night mode: yes\n" }; + if (key.endsWith("font_scale")) return { stdout: "0.85\n" }; + if (key.endsWith("animator_duration_scale")) return { stdout: "0\n" }; + if (key.endsWith("wifi_on")) return { stdout: "1\n" }; + if (key.endsWith("dumpsys window")) { + return { + stdout: + " mFocusedApp=ActivityRecord{155579877 u0 com.example.app/.MainActivity t15}\n mCurrentFocus=Window{1a2b u0 com.example.app/com.example.app.MainActivity}\n", + }; + } + return { code: 1 }; + }); + const detail = yield* readDeviceDetail(ready, "android", "emulator-5554"); + expect(detail.settings).toEqual({ + appearance: "dark", + textSize: "small", + reduceMotion: true, + networkEnabled: true, + }); + expect(detail.foregroundApp).toEqual({ id: "com.example.app" }); + }), + ); +}); diff --git a/apps/server/src/device/DeviceActions.ts b/apps/server/src/device/DeviceActions.ts new file mode 100644 index 000000000000..12781b31b4b5 --- /dev/null +++ b/apps/server/src/device/DeviceActions.ts @@ -0,0 +1,568 @@ +/** + * Device settings and one-shot actions, run by the server against the host's + * toolchain instead of through serve-sim's shell-exec channel. + * + * serve-sim's preview drives its "Simulator" panel by sending shell commands + * over a token-gated socket. Proxying that would hand any environment + * session arbitrary command execution on the host, so T3 runs the same + * underlying commands itself, typed per action: `xcrun simctl ui` and + * `simctl privacy` for iOS, the `serve-sim-ax-settings` helper that serve-sim + * bundles for the accessibility toggles, and `adb shell` for Android. + * + * Each platform advertises which actions it supports; the panel hides the + * rest rather than showing controls that cannot work. + */ +import { + type DeviceActionInput, + type DeviceActionType, + type DeviceForegroundApp, + DeviceOperationError, + DeviceActionUnavailableError, + type DeviceOrientation, + type DevicePlatform, + type DeviceSettings, + type DeviceTextSize, +} from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Schema from "effect/Schema"; + +import type { DeviceHostReady } from "./DeviceHost.ts"; + +type Runner = DeviceHostReady["run"]; + +const decodeAxStatus = Schema.decodeUnknownEffect( + Schema.fromJsonString(Schema.Record(Schema.String, Schema.String)), +); +const encodePushPayload = Schema.encodeUnknownEffect( + Schema.fromJsonString(Schema.Record(Schema.String, Schema.Unknown)), +); + +const IOS_ACTIONS: ReadonlySet = new Set([ + "setAppearance", + "setTextSize", + "setToggle", + "setLiquidGlass", + "setColorFilter", + "setLocation", + "clearLocation", + "setPermission", + "openUrl", + "launchApp", + "terminateApp", + "sendPush", +]); + +const ANDROID_ACTIONS: ReadonlySet = new Set([ + "setAppearance", + "setTextSize", + "setToggle", + "setOrientation", + "setLocation", + "clearLocation", + "setPermission", + "openUrl", + "launchApp", + "terminateApp", +]); + +/** Toggle settings each platform can actually flip. */ +const IOS_TOGGLES = new Set([ + "reduceMotion", + "increaseContrast", + "reduceTransparency", + "showBorders", + "voiceOver", +]); +const ANDROID_TOGGLES = new Set(["reduceMotion", "networkEnabled"]); + +export const supportsAction = (platform: DevicePlatform, input: DeviceActionInput): boolean => { + const actions = platform === "ios" ? IOS_ACTIONS : ANDROID_ACTIONS; + if (!actions.has(input.type)) return false; + if (input.type === "setToggle") { + return (platform === "ios" ? IOS_TOGGLES : ANDROID_TOGGLES).has(input.setting); + } + return true; +}; + +const ok = (operation: string) => (result: { code: number; stderr: string; stdout: string }) => + result.code === 0 + ? Effect.succeed(result.stdout) + : Effect.fail( + new DeviceOperationError({ + operation, + reason: "command_failed", + exitCode: result.code, + cause: result, + }), + ); + +// iOS text-size categories in ascending order; the four shared steps index +// into it. `default` is what a fresh simulator reports ("large"). +const IOS_TEXT_SIZES: Record = { + small: "small", + default: "large", + large: "extra-extra-large", + "extra-large": "accessibility-large", +}; +const ANDROID_TEXT_SIZES: Record = { + small: "0.85", + default: "1.0", + large: "1.15", + "extra-large": "1.3", +}; + +const textSizeFromIos = (category: string): DeviceTextSize | undefined => { + const entry = (Object.entries(IOS_TEXT_SIZES) as Array<[DeviceTextSize, string]>).find( + ([, value]) => value === category, + ); + if (entry) return entry[0]; + if (category.startsWith("accessibility")) return "extra-large"; + if (category.includes("extra")) return "large"; + return category === "extra-small" || category === "small" || category === "medium" + ? "small" + : "default"; +}; + +const textSizeFromAndroid = (scale: number): DeviceTextSize => { + if (scale <= 0.9) return "small"; + if (scale >= 1.25) return "extra-large"; + if (scale >= 1.1) return "large"; + return "default"; +}; + +const IOS_TOGGLE_OPTIONS: Record = { + reduceMotion: "reduce-motion", + increaseContrast: "increase-contrast", + reduceTransparency: "reduce-transparency", + showBorders: "show-borders", + voiceOver: "voiceover", +}; + +// serve-sim permission names -> the TCC service or simctl privacy service. +const IOS_TCC_SERVICES: Record = { + camera: "camera", + microphone: "microphone", + photos: "photos", + contacts: "contacts", + calendar: "calendar", + reminders: "reminders", + motion: "motion", + "media-library": "media-library", + faceid: "faceid", +}; + +const ANDROID_PERMISSIONS: Record> = { + camera: ["android.permission.CAMERA"], + microphone: ["android.permission.RECORD_AUDIO"], + photos: ["android.permission.READ_MEDIA_IMAGES", "android.permission.READ_EXTERNAL_STORAGE"], + contacts: ["android.permission.READ_CONTACTS", "android.permission.WRITE_CONTACTS"], + calendar: ["android.permission.READ_CALENDAR", "android.permission.WRITE_CALENDAR"], + location: [ + "android.permission.ACCESS_FINE_LOCATION", + "android.permission.ACCESS_COARSE_LOCATION", + ], + notifications: ["android.permission.POST_NOTIFICATIONS"], + motion: ["android.permission.ACTIVITY_RECOGNITION"], +}; + +// Gravity vector (x:y:z) that makes the emulator report each orientation, +// and the window-manager rotation index for the same. +const ANDROID_GRAVITY: Record = { + portrait: "0:9.81:0", + landscape_left: "9.81:0:0", + portrait_upside_down: "0:-9.81:0", + landscape_right: "-9.81:0:0", +}; +const ANDROID_ROTATION: Record = { + portrait: "0", + landscape_left: "1", + portrait_upside_down: "2", + landscape_right: "3", +}; + +export const runDeviceAction = Effect.fn("DeviceActions.run")(function* ( + ready: DeviceHostReady, + platform: DevicePlatform, + input: DeviceActionInput, +) { + if (!supportsAction(platform, input)) { + return yield* new DeviceActionUnavailableError({ + operation: input.type, + platform, + reason: "unsupported", + }); + } + if (platform === "ios") return yield* runIos(ready, input); + return yield* runAndroid(ready.run, input); +}); + +const simctl = (run: Runner, udid: string, args: ReadonlyArray, operation: string) => + run("xcrun", ["simctl", ...args.slice(0, 1), udid, ...args.slice(1)]).pipe( + Effect.flatMap(ok(operation)), + ); + +const axSettings = (ready: DeviceHostReady, udid: string, args: ReadonlyArray) => + Effect.gen(function* () { + const helper = ready.helpers.serveSimAxSettings; + if (!helper) { + return yield* new DeviceActionUnavailableError({ + operation: "accessibility", + platform: "ios", + reason: "helper_missing", + }); + } + return yield* ready + .run("xcrun", ["simctl", "spawn", udid, helper, ...args]) + .pipe(Effect.flatMap(ok("accessibility"))); + }); + +const runIos = Effect.fn("DeviceActions.runIos")(function* ( + ready: DeviceHostReady, + input: DeviceActionInput, +) { + const udid = input.deviceId; + const { run } = ready; + switch (input.type) { + case "setAppearance": + yield* simctl(run, udid, ["ui", "appearance", input.value], "appearance"); + return; + case "setTextSize": + yield* simctl(run, udid, ["ui", "content_size", IOS_TEXT_SIZES[input.value]], "text size"); + return; + case "setToggle": + if (input.setting === "increaseContrast") { + yield* simctl( + run, + udid, + ["ui", "increase_contrast", input.value ? "enabled" : "disabled"], + "increase contrast", + ); + return; + } + yield* axSettings(ready, udid, [ + "set", + IOS_TOGGLE_OPTIONS[input.setting]!, + input.value ? "on" : "off", + ]); + return; + case "setLiquidGlass": + yield* axSettings(ready, udid, ["set", "liquid-glass", input.value]); + return; + case "setColorFilter": + yield* axSettings(ready, udid, ["set", "color-filter", input.value]); + return; + case "setLocation": + yield* simctl( + run, + udid, + ["location", "set", `${input.latitude},${input.longitude}`], + "location", + ); + return; + case "clearLocation": + yield* simctl(run, udid, ["location", "clear"], "location"); + return; + case "setPermission": { + if (input.permission === "notifications") { + // simctl has no notification permission verb; serve-sim's CLI edits + // the BulletinBoard plist for it. + yield* serveSimPermissions(ready, udid, input); + return; + } + const service = + input.permission === "location" ? "location" : IOS_TCC_SERVICES[input.permission]; + if (!service) + return yield* new DeviceActionUnavailableError({ + operation: "permission", + platform: "ios", + reason: "unsupported", + }); + yield* simctl(run, udid, ["privacy", input.decision, service, input.appId], "permission"); + return; + } + case "openUrl": + yield* simctl(run, udid, ["openurl", input.url], "open url"); + return; + case "launchApp": + yield* simctl(run, udid, ["launch", input.appId], "launch"); + return; + case "terminateApp": + yield* simctl(run, udid, ["terminate", input.appId], "terminate"); + return; + case "sendPush": { + const payload = + typeof input.payload === "string" ? { aps: { alert: input.payload } } : input.payload; + const encoded = yield* encodePushPayload(payload).pipe( + Effect.mapError( + (cause) => + new DeviceOperationError({ operation: "push", reason: "invalid_payload", cause }), + ), + ); + yield* run("xcrun", ["simctl", "push", udid, input.appId, "-"], { stdin: encoded }).pipe( + Effect.flatMap(ok("push")), + ); + return; + } + case "shake": + case "setOrientation": + return yield* new DeviceActionUnavailableError({ + operation: input.type, + platform: "ios", + reason: "unsupported", + }); + } +}); + +const serveSimPermissions = ( + ready: DeviceHostReady, + udid: string, + input: Extract, +) => + Effect.gen(function* () { + const cli = ready.helpers.serveSimCli; + if (!cli) + return yield* new DeviceActionUnavailableError({ + operation: "permission", + platform: "ios", + reason: "helper_missing", + }); + yield* ready + .run(process.execPath, [ + cli, + "permissions", + input.decision, + input.permission, + input.appId, + "-d", + udid, + ]) + .pipe(Effect.flatMap(ok("permission"))); + }); + +const adb = (run: Runner, serial: string, args: ReadonlyArray, operation: string) => + run("adb", ["-s", serial, ...args]).pipe(Effect.flatMap(ok(operation))); + +const runAndroid = Effect.fn("DeviceActions.runAndroid")(function* ( + run: Runner, + input: DeviceActionInput, +) { + const serial = input.deviceId; + const shell = (args: ReadonlyArray, operation: string) => + adb(run, serial, ["shell", ...args], operation); + switch (input.type) { + case "setAppearance": + yield* shell(["cmd", "uimode", "night", input.value === "dark" ? "yes" : "no"], "appearance"); + return; + case "setTextSize": + yield* shell( + ["settings", "put", "system", "font_scale", ANDROID_TEXT_SIZES[input.value]], + "text size", + ); + return; + case "setToggle": + if (input.setting === "networkEnabled") { + const state = input.value ? "enable" : "disable"; + yield* shell(["svc", "wifi", state], "network"); + yield* shell(["svc", "data", state], "network"); + return; + } + if (input.setting === "reduceMotion") { + const scale = input.value ? "0" : "1"; + for (const key of [ + "animator_duration_scale", + "transition_animation_scale", + "window_animation_scale", + ]) { + yield* shell(["settings", "put", "global", key, scale], "reduce motion"); + } + return; + } + return yield* new DeviceActionUnavailableError({ + operation: input.type, + platform: "android", + reason: "unsupported", + }); + case "setOrientation": { + // `user-rotation lock` only rotates window content on recent images; + // the display the encoder captures stays put. Tilting the emulator's + // accelerometer rotates it for real, so that is used whenever the + // target is an emulator. Physical devices get the lock. + if (serial.startsWith("emulator-")) { + yield* shell(["settings", "put", "system", "accelerometer_rotation", "1"], "orientation"); + yield* shell(["cmd", "window", "user-rotation", "free"], "orientation"); + yield* adb( + run, + serial, + ["emu", "sensor", "set", "acceleration", ANDROID_GRAVITY[input.value]], + "orientation", + ); + return; + } + yield* shell( + ["cmd", "window", "user-rotation", "lock", ANDROID_ROTATION[input.value]], + "orientation", + ); + return; + } + case "setLocation": + yield* adb( + run, + serial, + ["emu", "geo", "fix", String(input.longitude), String(input.latitude)], + "location", + ); + return; + case "clearLocation": + // The emulator has no "clear"; leaving the fix in place is the closest + // behavior, so this is a no-op that still refreshes the reading. + return; + case "setPermission": { + const permissions = ANDROID_PERMISSIONS[input.permission]; + if (!permissions) { + return yield* new DeviceActionUnavailableError({ + operation: "permission", + platform: "android", + reason: "unsupported", + }); + } + const verb = input.decision === "grant" ? "grant" : "revoke"; + for (const permission of permissions) { + // Not every app declares every permission in a group; ignore those. + yield* shell(["pm", verb, input.appId, permission], "permission").pipe(Effect.ignore); + } + return; + } + case "openUrl": + yield* shell( + ["am", "start", "-a", "android.intent.action.VIEW", "-d", input.url], + "open url", + ); + return; + case "launchApp": + yield* shell( + ["monkey", "-p", input.appId, "-c", "android.intent.category.LAUNCHER", "1"], + "launch", + ); + return; + case "terminateApp": + yield* shell(["am", "force-stop", input.appId], "terminate"); + return; + case "setLiquidGlass": + case "setColorFilter": + case "shake": + case "sendPush": + return yield* new DeviceActionUnavailableError({ + operation: input.type, + platform: "android", + reason: "unsupported", + }); + } +}); + +/** Read the current settings and foreground app. Errors degrade to unknowns. */ +export const readDeviceDetail = Effect.fn("DeviceActions.readDetail")(function* ( + ready: DeviceHostReady, + platform: DevicePlatform, + deviceId: string, +): Effect.fn.Return<{ settings: DeviceSettings; foregroundApp: DeviceForegroundApp | null }> { + return platform === "ios" + ? yield* readIos(ready, deviceId) + : yield* readAndroid(ready.run, deviceId); +}); + +const quiet = (effect: Effect.Effect) => + effect.pipe(Effect.orElseSucceed((): A | undefined => undefined)); + +const readIos = Effect.fn("DeviceActions.readIos")(function* ( + ready: DeviceHostReady, + udid: string, +) { + const { run } = ready; + const uiValue = (option: string) => + quiet( + simctl(run, udid, ["ui", option], option).pipe(Effect.map((out) => out.trim().toLowerCase())), + ); + const [appearance, contentSize, contrast, axStatus] = yield* Effect.all( + [ + uiValue("appearance"), + uiValue("content_size"), + uiValue("increase_contrast"), + quiet(axSettings(ready, udid, ["status"]).pipe(Effect.flatMap(decodeAxStatus))), + ], + { concurrency: 4 }, + ); + const onOff = (value: string | undefined) => + value === "on" ? true : value === "off" ? false : undefined; + const settings: DeviceSettings = { + ...(appearance === "light" || appearance === "dark" ? { appearance } : {}), + ...(contentSize ? { textSize: textSizeFromIos(contentSize) } : {}), + ...(contrast ? { increaseContrast: contrast === "enabled" } : {}), + ...(axStatus + ? { + ...(onOff(axStatus["reduce-motion"]) === undefined + ? {} + : { reduceMotion: onOff(axStatus["reduce-motion"]) }), + ...(onOff(axStatus["reduce-transparency"]) === undefined + ? {} + : { reduceTransparency: onOff(axStatus["reduce-transparency"]) }), + ...(onOff(axStatus["show-borders"]) === undefined + ? {} + : { showBorders: onOff(axStatus["show-borders"]) }), + ...(onOff(axStatus.voiceover) === undefined + ? {} + : { voiceOver: onOff(axStatus.voiceover) }), + ...(axStatus["liquid-glass"] === "clear" || axStatus["liquid-glass"] === "tinted" + ? { liquidGlass: axStatus["liquid-glass"] } + : {}), + ...(isColorFilter(axStatus["color-filter"]) + ? { colorFilter: axStatus["color-filter"] } + : {}), + } + : {}), + }; + return { settings, foregroundApp: null }; +}); + +const isColorFilter = (value: unknown): value is DeviceSettings["colorFilter"] & string => + value === "none" || + value === "grayscale" || + value === "red-green" || + value === "green-red" || + value === "blue-yellow"; + +const readAndroid = Effect.fn("DeviceActions.readAndroid")(function* (run: Runner, serial: string) { + const shell = (args: ReadonlyArray) => + quiet( + adb(run, serial, ["shell", ...args], args[0] ?? "shell").pipe(Effect.map((s) => s.trim())), + ); + const [night, fontScale, animator, wifi, focus] = yield* Effect.all( + [ + shell(["cmd", "uimode", "night"]), + shell(["settings", "get", "system", "font_scale"]), + shell(["settings", "get", "global", "animator_duration_scale"]), + shell(["settings", "get", "global", "wifi_on"]), + // `dumpsys window windows` stopped printing the focus on API 36; the + // unfiltered dump still does. + shell(["dumpsys", "window"]), + ], + { concurrency: 5 }, + ); + const scale = fontScale && fontScale !== "null" ? Number(fontScale) : Number.NaN; + const focused = focus?.match(/m(?:CurrentFocus|FocusedApp)=\w+\{[^ ]+ u\d+ ([^/ ]+)\//); + const settings: DeviceSettings = { + ...(night?.includes("yes") + ? { appearance: "dark" } + : night?.includes("no") + ? { appearance: "light" } + : {}), + ...(Number.isFinite(scale) ? { textSize: textSizeFromAndroid(scale) } : {}), + ...(animator !== undefined && animator !== "null" + ? { reduceMotion: Number(animator) === 0 } + : {}), + ...(wifi === "1" || wifi === "0" ? { networkEnabled: wifi === "1" } : {}), + }; + return { + settings, + foregroundApp: focused ? { id: focused[1]! } : null, + }; +}); diff --git a/apps/server/src/device/DeviceHost.ts b/apps/server/src/device/DeviceHost.ts new file mode 100644 index 000000000000..8d1bafbc5dfc --- /dev/null +++ b/apps/server/src/device/DeviceHost.ts @@ -0,0 +1,102 @@ +/** + * A device host is a machine with simulators or emulators on it. The service + * layer only ever talks to this interface, so a future SSH or cloud host slots + * in beside `LocalDeviceHost` without touching discovery, the proxy, or the + * MCP tools. + * + * Every ready host presents a loopback origin where expo-device-hub answers. + * Hosts add an agent-device daemon endpoint only after agent access is granted. + * For the local host both run on this machine; a remote host would forward + * them here. + */ +import type { + DeviceHostId, + DeviceHostSummary, + DevicePlatform, + DevicePlatformAvailability, +} from "@t3tools/contracts"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Schema from "effect/Schema"; + +export class DeviceHostError extends Schema.TaggedError()("DeviceHostError", { + hostId: Schema.String, + step: Schema.String, + cause: Schema.Defect(), +}) { + override get message(): string { + return `Device host ${this.hostId} failed while ${this.step}.`; + } +} + +export class DeviceHostTimeoutError extends Schema.TaggedError()( + "DeviceHostTimeoutError", + { hostId: Schema.String, timeoutMs: Schema.Number }, +) { + override get message(): string { + return `Device host ${this.hostId} did not start agent tools within ${this.timeoutMs} ms.`; + } +} + +export interface DeviceHubEndpoint { + /** Loopback origin of expo-device-hub, e.g. `http://127.0.0.1:3400`. */ + readonly origin: string; +} + +export interface AgentDeviceEndpoint { + readonly baseUrl: string; + readonly token: string; + /** Absolute path of the agent-device entry script for the provider PATH shim. */ + readonly entryPath: string; +} + +export interface DeviceHostReady { + readonly hub: DeviceHubEndpoint; + /** + * Runs a host command (`xcrun`, `adb`, or a helper bundled with the hub) + * where the devices live. On the local host this is a plain spawn; a + * remote host would run it over its transport. + */ + readonly run: ( + command: string, + args: ReadonlyArray, + options?: { readonly timeoutMs?: number; readonly stdin?: string }, + ) => Effect.Effect<{ readonly stdout: string; readonly stderr: string; readonly code: number }>; + /** Absolute paths of helper binaries vendored with the hub, when present. */ + readonly helpers: { + readonly serveSimAxSettings: string | null; + readonly serveSimCli: string | null; + }; +} + +export interface DeviceHostAgentReady extends DeviceHostReady { + readonly agentDevice: AgentDeviceEndpoint; +} + +export class DeviceHost extends Context.Service< + DeviceHost, + { + readonly id: DeviceHostId; + readonly summary: Effect.Effect; + readonly platformAvailability: ( + platform: DevicePlatform, + ) => Effect.Effect; + /** + * Installs tools on first use and starts the helper processes. Idempotent: + * concurrent callers share one start, and a ready host returns immediately. + */ + readonly ensureReady: ( + onPhase: (phase: "installing" | "starting") => Effect.Effect, + ) => Effect.Effect; + /** Installs and starts agent-device after the user grants agent access. */ + readonly ensureAgentReady: ( + onPhase: (phase: "installing" | "starting") => Effect.Effect, + ) => Effect.Effect; + /** Current endpoints when already running, without starting anything. */ + readonly current: Effect.Effect; + /** Stops only agent-device. Manual viewing through the hub stays available. */ + readonly stopAgent: Effect.Effect; + /** Stops helpers. Devices themselves keep running; the user owns those. */ + readonly stop: Effect.Effect; + } +>()("t3/device/DeviceHost") {} diff --git a/apps/server/src/device/DeviceHubProxy.test.ts b/apps/server/src/device/DeviceHubProxy.test.ts new file mode 100644 index 000000000000..0f039274e207 --- /dev/null +++ b/apps/server/src/device/DeviceHubProxy.test.ts @@ -0,0 +1,141 @@ +import { afterEach, describe, expect, it } from "vite-plus/test"; +import { + AuthOrchestrationReadScope, + AuthOrchestrationOperateScope, + AuthSessionId, + LOCAL_DEVICE_HOST_ID, + type AuthEnvironmentScope, +} from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import { HttpClient, HttpClientResponse, HttpRouter } from "effect/unstable/http"; +import { + EnvironmentAuth, + ServerAuthMissingCredentialError, + ServerAuthSessionCredentialValidationError, + type ServerAuthCredentialError, + type ServerAuthInternalError, +} from "../auth/EnvironmentAuth.ts"; +import { DeviceService } from "./DeviceService.ts"; +import { deviceHubProxyRouteLayer } from "./DeviceHubProxy.ts"; + +const disposers: Array<() => Promise> = []; +afterEach(async () => { + for (const dispose of disposers.splice(0)) await dispose(); +}); + +const fixture = ( + scopes: ReadonlyArray, + fail = false, + authError?: ServerAuthCredentialError | ServerAuthInternalError, +) => { + let finalized = 0; + const requests: string[] = []; + const client = HttpClient.make((request, _url, signal) => + Effect.gen(function* () { + requests.push(request.url); + signal.addEventListener("abort", () => { + finalized++; + }); + if (fail) return yield* Effect.die(new Error("upstream failed")); + return HttpClientResponse.fromWeb(request, new Response("frame")); + }), + ); + const { handler, dispose } = HttpRouter.toWebHandler( + deviceHubProxyRouteLayer.pipe( + Layer.provideMerge( + Layer.succeed(EnvironmentAuth, { + authenticateWebSocketUpgrade: () => + authError + ? Effect.fail(authError) + : Effect.succeed({ + sessionId: AuthSessionId.make("test"), + subject: "test", + method: "bearer-access-token", + scopes, + }), + } as unknown as EnvironmentAuth["Service"]), + ), + Layer.provideMerge( + Layer.succeed(DeviceService, { + currentReadiness: () => + Effect.succeed({ hostId: LOCAL_DEVICE_HOST_ID, hub: { origin: "http://hub.test" } }), + } as DeviceService["Service"]), + ), + Layer.provideMerge(Layer.succeed(HttpClient.HttpClient, client)), + ), + { disableLogger: true }, + ); + disposers.push(dispose); + return { handler, requests, finalized: () => finalized }; +}; + +describe("device hub proxy", () => { + it("releases the upstream response after forwarding its body and strips tickets", async () => { + const { handler, requests, finalized } = fixture([AuthOrchestrationReadScope]); + const response = await handler( + new Request("http://t3.test/api/device-hub/api/devices?wsTicket=secret"), + ); + expect(response.status).toBe(200); + expect(await response.text()).toBe("frame"); + expect(requests).toEqual(["http://hub.test/api/devices"]); + expect(finalized()).toBe(1); + }); + + it("releases resources when upstream acquisition fails", async () => { + const { handler, finalized } = fixture([AuthOrchestrationReadScope], true); + const response = await handler(new Request("http://t3.test/api/device-hub/api/devices")); + expect(response.status).toBe(500); + expect(finalized()).toBe(1); + }); + + it.each(["/vendor/serve-sim/helper/ws", "/vendor/serve-emu/ws"])( + "rejects input socket %s for a read-only session", + async (path) => { + const { handler, requests } = fixture([AuthOrchestrationReadScope]); + const response = await handler( + new Request(`http://t3.test/api/device-hub${path}`, { headers: { upgrade: "websocket" } }), + ); + expect(response.status).toBe(403); + expect(requests).toEqual([]); + }, + ); + + it("requires operate scope for stream tuning", async () => { + const readOnly = fixture([AuthOrchestrationReadScope]); + const path = "http://t3.test/api/device-hub/vendor/serve-emu/api/stream-settings"; + expect((await readOnly.handler(new Request(path, { method: "POST" }))).status).toBe(403); + const operator = fixture([AuthOrchestrationOperateScope]); + const response = await operator.handler(new Request(path, { method: "POST" })); + expect(response.status).toBe(200); + await response.text(); + }); + + it("never forwards the vendor shell endpoint", async () => { + const { handler, requests } = fixture([AuthOrchestrationOperateScope]); + expect( + ( + await handler( + new Request("http://t3.test/api/device-hub/vendor/serve-sim/exec", { method: "POST" }), + ) + ).status, + ).toBe(404); + expect(requests).toEqual([]); + }); +}); + +it.each([ + [new ServerAuthMissingCredentialError({}), 401], + [ + new ServerAuthSessionCredentialValidationError({ + cause: new Error("private credential diagnostic"), + }), + 500, + ], +] as const)("translates authentication failure to HTTP %s", async (error, status) => { + const { handler, requests } = fixture([], false, error); + const response = await handler(new Request("http://t3.test/api/device-hub/api/devices")); + expect(response.status).toBe(status); + expect(await response.text()).not.toContain("private credential diagnostic"); + expect(requests).toEqual([]); +}); diff --git a/apps/server/src/device/DeviceHubProxy.ts b/apps/server/src/device/DeviceHubProxy.ts new file mode 100644 index 000000000000..f8685727c882 --- /dev/null +++ b/apps/server/src/device/DeviceHubProxy.ts @@ -0,0 +1,224 @@ +/** + * Same-origin proxy in front of expo-device-hub. + * + * The hub binds loopback and is never reachable directly: serve-sim exposes a + * shell-exec route and serve-emu's action routes are unauthenticated, so the + * only way to a device stream is through this route, which requires an + * environment session with read scope (operate scope for input and tuning). Reusing the T3 + * origin is also what makes remote connections work unchanged — Tailscale and + * T3 Connect already carry `/api/*` and WebSocket upgrades for the app itself. + * + * Only the routes the Device panel needs are forwarded. Anything under the + * hub's dashboard, exec, or WebRTC surface is rejected here. + */ +import { + AuthOrchestrationReadScope, + AuthOrchestrationOperateScope, + type AuthEnvironmentScope, +} from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; +import { + HttpClient, + HttpClientRequest, + HttpRouter, + HttpServerRequest, + HttpServerResponse, +} from "effect/unstable/http"; +import * as Socket from "effect/unstable/socket/Socket"; +import * as NodeSocket from "@effect/platform-node/NodeSocket"; + +import * as EnvironmentAuth from "../auth/EnvironmentAuth.ts"; +import { + failEnvironmentAuthInvalid, + failEnvironmentInternal, + failEnvironmentScopeRequired, +} from "../auth/http.ts"; +import * as DeviceService from "./DeviceService.ts"; + +const ALLOWED_PATHS: ReadonlyArray = [ + /^\/api\/devices$/, + /^\/vendor\/serve-sim\/api$/, + /^\/vendor\/serve-sim\/api\/screenshot$/, + /^\/vendor\/serve-sim\/api\/event-log(\/events)?$/, + /^\/vendor\/serve-sim\/helper\/[^/]+\/(stream\.mjpeg|stream\.avcc|config|health|ax|foreground)$/, + /^\/vendor\/serve-sim\/appstate$/, + /^\/vendor\/serve-emu\/api\/(devices|screenshot|stream-mode|stream-settings|accessibility)$/, + /^\/vendor\/serve-emu\/health$/, +]; + +/** Read paths are GET-only; only these accept other methods (screenshot captures, stream tuning). */ +const MUTABLE_PATHS: ReadonlyArray = [ + /^\/vendor\/serve-sim\/api\/screenshot$/, + /^\/vendor\/serve-emu\/api\/(screenshot|stream-mode|stream-settings)$/, +]; + +const ALLOWED_WS_PATHS: ReadonlyArray = [ + /^\/api\/devices\/ws$/, + /^\/vendor\/serve-sim\/helper\/ws$/, + /^\/vendor\/serve-emu\/ws$/, +]; + +/** Hop-by-hop and origin headers that must not cross the proxy. */ +const DROPPED_REQUEST_HEADERS = new Set([ + "host", + "connection", + "upgrade", + "sec-websocket-key", + "sec-websocket-version", + "sec-websocket-extensions", + "sec-websocket-protocol", + "cookie", + "authorization", + "dpop", + "content-length", + "accept-encoding", +]); + +const isWebSocketUpgrade = (request: HttpServerRequest.HttpServerRequest) => + request.headers.upgrade?.toLowerCase() === "websocket"; + +/** + * `` and WebSocket cannot set headers, so every proxied request + * authenticates the way the `/ws` upgrade does: a cookie for browser + * sessions, or a short-lived `wsTicket` minted over authenticated HTTP for + * bearer and DPoP clients. The upgrade authenticator already implements that + * fallback order, so it is used for plain requests as well. + */ +const authenticate = (requiredScope: AuthEnvironmentScope) => + Effect.gen(function* () { + const request = yield* HttpServerRequest.HttpServerRequest; + const serverAuth = yield* EnvironmentAuth.EnvironmentAuth; + const session = yield* serverAuth.authenticateWebSocketUpgrade(request).pipe( + Effect.catch((error) => + Effect.gen(function* () { + if (EnvironmentAuth.isServerAuthCredentialError(error)) { + return yield* failEnvironmentAuthInvalid( + EnvironmentAuth.serverAuthCredentialReason(error), + EnvironmentAuth.serverAuthDpopFailureReason(error), + ); + } + return yield* failEnvironmentInternal("internal_error", error); + }), + ), + ); + if (!session.scopes.includes(requiredScope)) { + return yield* failEnvironmentScopeRequired(requiredScope); + } + }); + +const forwardHeaders = (request: HttpServerRequest.HttpServerRequest, origin: string) => { + const headers: Record = {}; + for (const [name, value] of Object.entries(request.headers)) { + if (DROPPED_REQUEST_HEADERS.has(name) || value === undefined) continue; + headers[name] = value; + } + // serve-emu refuses mutations whose Origin differs from the request origin. + if (request.headers.origin !== undefined) headers.origin = origin; + return headers; +}; + +/** + * Pipe a client WebSocket to the hub's with no framing changes. Frames are + * opaque: H.264 access units one way, input packets the other. + */ +const proxyWebSocket = Effect.fn("DeviceHubProxy.proxyWebSocket")(function* ( + request: HttpServerRequest.HttpServerRequest, + upstreamUrl: string, +) { + const client = yield* request.upgrade; + const upstream = yield* Socket.makeWebSocket(upstreamUrl, { + openTimeout: "10 seconds", + }).pipe(Effect.provide(NodeSocket.layerWebSocketConstructor)); + yield* Effect.scoped( + Effect.gen(function* () { + const writeToClient = yield* client.writer; + const writeToUpstream = yield* upstream.writer; + const downstream = upstream.runRaw((data) => writeToClient(data)); + const upstreamPump = client.runRaw((data) => writeToUpstream(data)); + // Whichever side closes first ends the other via scope teardown. + yield* Effect.raceFirst(downstream, upstreamPump); + }), + ).pipe(Effect.catchCause(() => Effect.void)); + return HttpServerResponse.empty(); +}); + +const proxyHttp = Effect.fn("DeviceHubProxy.proxyHttp")(function* ( + request: HttpServerRequest.HttpServerRequest, + upstreamUrl: string, + hubOrigin: string, +) { + const httpClient = HttpClient.withScope(yield* HttpClient.HttpClient); + const method = request.method; + const upstreamRequest = HttpClientRequest.make(method)(upstreamUrl).pipe( + HttpClientRequest.setHeaders(forwardHeaders(request, hubOrigin)), + method === "GET" || method === "HEAD" + ? (self) => self + : HttpClientRequest.bodyStream(request.stream), + ); + const response = yield* httpClient.execute(upstreamRequest); + const headers: Record = {}; + for (const [name, value] of Object.entries(response.headers)) { + if (name === "content-encoding" || name === "transfer-encoding" || name === "connection") { + continue; + } + if (value !== undefined) headers[name] = value; + } + // Long-lived MJPEG and AVCC responses must not be buffered by compression. + headers["cache-control"] = "no-store, no-transform"; + return HttpServerResponse.stream(response.stream, { + status: response.status, + headers, + ...(headers["content-type"] ? { contentType: headers["content-type"] } : {}), + }); +}); + +const handler = Effect.gen(function* () { + const request = yield* HttpServerRequest.HttpServerRequest; + const url = HttpServerRequest.toURL(request); + if (Option.isNone(url)) { + return HttpServerResponse.text("Bad Request", { status: 400 }); + } + const hubPath = url.value.pathname.slice(DeviceService.DEVICE_HUB_ROUTE_PREFIX.length) || "/"; + const upgrade = isWebSocketUpgrade(request); + const allowed = (upgrade ? ALLOWED_WS_PATHS : ALLOWED_PATHS).some((pattern) => + pattern.test(hubPath), + ); + if (!allowed) { + return HttpServerResponse.text("Not Found", { status: 404 }); + } + const readOnly = request.method === "GET" || request.method === "HEAD"; + if (!upgrade && !readOnly && !MUTABLE_PATHS.some((pattern) => pattern.test(hubPath))) { + return HttpServerResponse.text("Method Not Allowed", { status: 405 }); + } + const controlsDevice = + (upgrade && hubPath !== "/api/devices/ws") || + (!readOnly && /\/api\/stream-(mode|settings)$/.test(hubPath)); + yield* authenticate(controlsDevice ? AuthOrchestrationOperateScope : AuthOrchestrationReadScope); + const devices = yield* DeviceService.DeviceService; + const ready = yield* devices.currentReadiness(); + if (!ready) { + return HttpServerResponse.text("Device hub is not running", { status: 503 }); + } + // The hub runs in standalone mode at its origin root; the panel builds every + // stream and socket URL itself, so nothing depends on the hub knowing the + // T3 prefix. + // The ticket authenticates here and must not travel on to the hub. + const upstreamSearch = new URLSearchParams(url.value.search); + upstreamSearch.delete("wsTicket"); + const search = upstreamSearch.size > 0 ? `?${upstreamSearch.toString()}` : ""; + const upstreamPath = `${hubPath}${search}`; + if (upgrade) { + return yield* proxyWebSocket( + request, + `${ready.hub.origin.replace(/^http/, "ws")}${upstreamPath}`, + ); + } + return yield* proxyHttp(request, `${ready.hub.origin}${upstreamPath}`, ready.hub.origin); +}); + +export const deviceHubProxyRouteLayer = HttpRouter.add( + "*", + `${DeviceService.DEVICE_HUB_ROUTE_PREFIX}/*`, + handler, +); diff --git a/apps/server/src/device/DeviceService.test.ts b/apps/server/src/device/DeviceService.test.ts new file mode 100644 index 000000000000..ed42be64eb02 --- /dev/null +++ b/apps/server/src/device/DeviceService.test.ts @@ -0,0 +1,337 @@ +import { describe, expect, it } from "@effect/vitest"; +import { + DEFAULT_SERVER_SETTINGS, + DeviceId, + LOCAL_DEVICE_HOST_ID, + ThreadId, + type DeviceServiceState, +} from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Deferred from "effect/Deferred"; +import * as Fiber from "effect/Fiber"; +import * as PubSub from "effect/PubSub"; +import * as Ref from "effect/Ref"; +import * as Stream from "effect/Stream"; +import { HttpClient, HttpClientResponse } from "effect/unstable/http"; +import { ServerSettingsService } from "../serverSettings.ts"; +import * as DeviceHost from "./DeviceHost.ts"; + +import { type DeviceService, make, stateStream } from "./DeviceService.ts"; + +const baseState: DeviceServiceState = { + hosts: [], + hostStatus: "idle", + devices: [], + sessions: [], + onboardingCompleted: false, + agentAccessEnabled: false, + hubBasePath: "/api/device-hub", + revision: 0, +}; + +describe("DeviceService.stateStream", () => { + it.effect("emits the current snapshot and then every published change", () => + Effect.gen(function* () { + const pubsub = yield* PubSub.unbounded(); + const current = yield* Ref.make(baseState); + const service: Pick = { + state: Ref.get(current), + subscribe: PubSub.subscribe(pubsub), + }; + + const collected = yield* stateStream(service as DeviceService["Service"]).pipe( + Stream.take(3), + Stream.runCollect, + Effect.forkChild, + ); + yield* Effect.yieldNow; + for (const revision of [1, 2]) { + const next = { ...baseState, revision, hostStatus: "ready" as const }; + yield* Ref.set(current, next); + yield* PubSub.publish(pubsub, next); + } + const seen = yield* Fiber.join(collected); + expect(seen.map((state) => state.revision)).toEqual([0, 1, 2]); + }), + ); +}); + +const fixture = Effect.fn("fixture")(function* ( + onBoot: Effect.Effect = Effect.void, + bootError?: string, + failListAfterShutdown = false, +) { + const settings = yield* Ref.make(DEFAULT_SERVER_SETTINGS); + const starts: string[] = []; + const agentStarts: string[] = []; + const agentStops: string[] = []; + const requests: string[] = []; + let booted = false; + let shutDown = false; + const ready: DeviceHost.DeviceHostReady = { + hub: { origin: "http://device.test" }, + helpers: { serveSimAxSettings: null, serveSimCli: null }, + run: () => Effect.succeed({ code: 0, stdout: "Pixel_API_35\n", stderr: "" }), + }; + const host: DeviceHost.DeviceHost["Service"] = { + id: LOCAL_DEVICE_HOST_ID, + summary: Effect.succeed({ + id: LOCAL_DEVICE_HOST_ID, + kind: "local", + label: "Test server", + platforms: [{ platform: "android", available: true }], + hubInstalled: true, + agentDeviceInstalled: false, + }), + platformAvailability: (platform) => Effect.succeed({ platform, available: true }), + ensureReady: (onPhase) => + Effect.gen(function* () { + starts.push("start"); + yield* onPhase("starting"); + return ready; + }), + ensureAgentReady: (onPhase) => + Effect.gen(function* () { + agentStarts.push("start"); + yield* onPhase("starting"); + return { + ...ready, + agentDevice: { baseUrl: "http://agent.test", token: "test", entryPath: "/agent" }, + }; + }), + current: Effect.succeed(null), + stopAgent: Effect.sync(() => { + agentStops.push("stop"); + }), + stop: Effect.sync(() => { + starts.push("stop"); + }), + }; + const service = yield* make.pipe( + Effect.provideService(DeviceHost.DeviceHost, host), + Effect.provideService( + ServerSettingsService, + ServerSettingsService.of({ + start: Effect.void, + ready: Effect.void, + getSettings: Ref.get(settings), + updateSettings: (patch) => + Ref.updateAndGet(settings, (current) => ({ + ...current, + enableDeviceSupport: patch.enableDeviceSupport ?? current.enableDeviceSupport, + enableAgentDeviceAccess: + patch.enableAgentDeviceAccess ?? current.enableAgentDeviceAccess, + deviceOnboardingCompleted: + patch.deviceOnboardingCompleted ?? current.deviceOnboardingCompleted, + })), + streamChanges: Stream.empty, + subscribeChanges: Effect.succeed(Stream.empty), + }), + ), + Effect.provideService( + HttpClient.HttpClient, + HttpClient.make((request) => + Effect.gen(function* () { + requests.push(request.url); + if (request.url.includes("/api/screenshot")) { + return HttpClientResponse.fromWeb( + request, + new Response(new Uint8Array([137, 80, 78, 71])), + ); + } + if (request.url.endsWith("/shutdown")) { + shutDown = true; + booted = false; + return HttpClientResponse.fromWeb(request, Response.json({ ok: true })); + } + if (shutDown && failListAfterShutdown) { + return HttpClientResponse.fromWeb( + request, + new Response("Discovery busy", { status: 503 }), + ); + } + if (request.url.endsWith("/boot")) { + yield* onBoot; + booted = true; + return HttpClientResponse.fromWeb( + request, + Response.json( + bootError ? { ok: false, error: bootError } : { ok: true, serial: "emulator-5554" }, + ), + ); + } + return HttpClientResponse.fromWeb( + request, + Response.json({ + simulators: [], + emulators: booted + ? [ + { + id: "emulator-5554", + name: "Pixel_API_35", + platform: "android", + version: "Android 15", + booted: true, + physical: false, + }, + ] + : [], + }), + ); + }), + ), + ), + ); + return { service, starts, agentStarts, agentStops, requests, settings }; +}); + +describe("device setup consent", () => { + it.effect("listing and provider startup do not start helpers before consent", () => + Effect.gen(function* () { + const { service, starts, requests } = yield* fixture(); + expect((yield* service.list).hostStatus).toBe("disabled"); + expect(yield* service.readinessIfSupported()).toBeNull(); + const readiness = yield* service.readiness().pipe(Effect.result); + expect(readiness._tag).toBe("Failure"); + expect(starts).toEqual([]); + expect(requests).toEqual([]); + }).pipe(Effect.scoped), + ); + + it.effect( + "explicit setup discovers never-booted AVDs; disabling stops helpers and blocks agents", + () => + Effect.gen(function* () { + const { service, starts, settings } = yield* fixture(); + const state = yield* service.configure({ enabled: true }); + expect((yield* Ref.get(settings)).enableDeviceSupport).toBe(true); + expect(state.devices.map((device) => [device.id, device.booted])).toEqual([ + ["Pixel_API_35", false], + ]); + expect(starts).toEqual(["start"]); + const disabled = yield* service.configure({ enabled: false }); + expect(disabled.hostStatus).toBe("disabled"); + expect(disabled.devices).toEqual([]); + expect((yield* Ref.get(settings)).enableDeviceSupport).toBe(false); + expect(yield* service.readinessIfSupported()).toBeNull(); + expect(starts).toEqual(["start", "stop"]); + }).pipe(Effect.scoped), + ); + + it.effect("boots a stopped Android AVD and uses its emulator serial without duplicating it", () => + Effect.gen(function* () { + const { service, requests } = yield* fixture(); + yield* service.configure({ enabled: true }); + const session = yield* service.open({ + threadId: ThreadId.make("thread-1"), + deviceId: "Pixel_API_35", + platform: "android", + }); + expect(session.deviceId).toBe("emulator-5554"); + expect(requests.filter((url) => url.endsWith("/boot"))).toHaveLength(1); + const state = yield* service.state; + expect(state.devices.map((device) => device.id)).toEqual(["emulator-5554"]); + expect(state.bootingDevices).toEqual([]); + }).pipe(Effect.scoped), + ); + + it.effect("installs agent support only after the separate agent permission", () => + Effect.gen(function* () { + const { service, agentStarts, agentStops, settings } = yield* fixture(); + yield* service.configure({ enabled: true }); + expect(agentStarts).toEqual([]); + expect(yield* service.agentReadinessIfSupported()).toBeNull(); + + yield* service.configure({ agentAccessEnabled: true }); + expect(agentStarts).toEqual(["start"]); + expect((yield* Ref.get(settings)).enableAgentDeviceAccess).toBe(true); + expect((yield* service.state).agentAccessEnabled).toBe(true); + + yield* service.configure({ agentAccessEnabled: false, onboardingCompleted: true }); + expect(agentStops).toEqual(["stop"]); + expect((yield* service.state).onboardingCompleted).toBe(true); + expect((yield* Ref.get(settings)).deviceOnboardingCompleted).toBe(true); + }).pipe(Effect.scoped), + ); +}); + +it.effect("publishes boot progress and does not restore sessions after support is disabled", () => + Effect.gen(function* () { + const started = yield* Deferred.make(); + const finish = yield* Deferred.make(); + const { service } = yield* fixture( + Deferred.succeed(started, undefined).pipe(Effect.andThen(Deferred.await(finish))), + ); + yield* service.configure({ enabled: true }); + const opening = yield* service + .open({ threadId: ThreadId.make("thread-1"), deviceId: "Pixel_API_35", platform: "android" }) + .pipe(Effect.result, Effect.forkChild); + yield* Deferred.await(started); + expect((yield* service.state).bootingDevices?.map((device) => device.name)).toEqual([ + "Pixel_API_35", + ]); + yield* service.configure({ enabled: false }); + yield* Deferred.succeed(finish, undefined); + expect((yield* Fiber.join(opening))._tag).toBe("Failure"); + const state = yield* service.state; + expect(state.hostStatus).toBe("disabled"); + expect(state.devices).toEqual([]); + expect(state.sessions).toEqual([]); + expect(state.bootingDevices).toEqual([]); + }).pipe(Effect.scoped), +); + +describe("device discovery after server restart", () => { + it.effect("captures an explicit device before any client lists devices", () => + Effect.gen(function* () { + const { service, settings, requests } = yield* fixture(); + yield* Ref.update(settings, (current) => ({ ...current, enableDeviceSupport: true })); + expect((yield* service.state).devices).toEqual([]); + const capture = yield* service.screenshot({ deviceId: DeviceId.make("Pixel_API_35") }); + expect(capture.device.id).toBe("Pixel_API_35"); + expect(Array.from(capture.png)).toEqual([137, 80, 78, 71]); + expect(requests.some((url) => url.endsWith("/api/devices"))).toBe(true); + }).pipe(Effect.scoped), + ); +}); + +for (const [diagnostic, reason, message] of [ + ["Insufficient disk space at /private/user/path", "disk_space", "not enough free disk space"], + ["Timed out spawning /private/user/command", "timeout", "did not become ready in time"], + ["Unexpected failure: secret-token", "launch_failed", "could not start"], +] as const) { + it.effect(`normalizes boot failure: ${reason}`, () => + Effect.gen(function* () { + const { service } = yield* fixture(Effect.void, diagnostic); + yield* service.configure({ enabled: true }); + const error = yield* service + .open({ + threadId: ThreadId.make("boot-failure"), + deviceId: "Pixel_API_35", + platform: "android", + }) + .pipe(Effect.flip); + expect(error._tag).toBe("DeviceBootError"); + expect(error.message).toContain(message); + expect(error.message).not.toContain(diagnostic); + expect((yield* service.state).bootingDevices).toEqual([]); + }).pipe(Effect.scoped), + ); +} + +it.effect("keeps shutdown successful when subsequent discovery fails", () => + Effect.gen(function* () { + const { service } = yield* fixture(Effect.void, undefined, true); + yield* service.configure({ enabled: true }); + const threadId = ThreadId.make("shutdown-refresh"); + const session = yield* service.open({ + threadId, + deviceId: "Pixel_API_35", + platform: "android", + }); + yield* service.close({ threadId, deviceId: session.deviceId, shutdown: true }); + const state = yield* service.state; + expect(state.sessions).toEqual([]); + expect(state.devices.find((device) => device.id === session.deviceId)?.booted).toBe(false); + }).pipe(Effect.scoped), +); diff --git a/apps/server/src/device/DeviceService.ts b/apps/server/src/device/DeviceService.ts new file mode 100644 index 000000000000..6cca6631bd7e --- /dev/null +++ b/apps/server/src/device/DeviceService.ts @@ -0,0 +1,764 @@ +/** + * Device discovery, per-thread device sessions, and the state stream clients + * render the Device panel from. + * + * Discovery and boot go through expo-device-hub's JSON API rather than + * shelling out to simctl and adb here: the hub already normalizes both + * platforms into one device shape and is the process that has to know a + * device is booted before it can stream it. Sessions are the server's own + * bookkeeping — which thread is looking at which device — so the panel and + * the `device_*` tools agree, and so a `device_open` from an agent surfaces in + * every connected client the way `preview_open` does. + */ +import { + type DeviceActionInput, + type DeviceCloseInput, + type DeviceConfigureInput, + type DeviceDetail, + type DeviceDetailInput, + type DeviceError, + type DeviceHostId, + type DeviceId, + DeviceBootError, + DeviceHostUnavailableError, + DeviceNotFoundError, + DeviceOperationError, + type DeviceOpenInput, + type DevicePlatform, + DevicePlatformUnavailableError, + type DeviceServiceState, + type DeviceSession, + type DeviceShutdownInput, + type DeviceSummary, + LOCAL_DEVICE_HOST_ID, + type ThreadId, +} from "@t3tools/contracts"; +import * as Context from "effect/Context"; +import * as DateTime from "effect/DateTime"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as PubSub from "effect/PubSub"; +import * as Schema from "effect/Schema"; +import * as Scope from "effect/Scope"; +import * as Semaphore from "effect/Semaphore"; +import * as Stream from "effect/Stream"; +import * as SynchronizedRef from "effect/SynchronizedRef"; +import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"; + +import * as ServerSettings from "../serverSettings.ts"; + +import { readDeviceDetail, runDeviceAction } from "./DeviceActions.ts"; +import * as DeviceHost from "./DeviceHost.ts"; +import * as LocalDeviceHost from "./LocalDeviceHost.ts"; + +/** Origin-relative prefix the hub is proxied under. See DeviceHubProxy. */ +export const DEVICE_HUB_ROUTE_PREFIX = "/api/device-hub"; + +const BOOT_TIMEOUT = Duration.minutes(3); +const SCREENSHOT_TIMEOUT = Duration.seconds(20); + +const HubDevice = Schema.Struct({ + id: Schema.String, + name: Schema.String, + version: Schema.String, + platform: Schema.Literals(["ios", "android"]), + booted: Schema.Boolean, + physical: Schema.Boolean, +}); +const HubDeviceList = Schema.Struct({ + simulators: Schema.Array(HubDevice), + emulators: Schema.Array(HubDevice), + errors: Schema.optional(Schema.Array(Schema.Struct({ message: Schema.String }))), +}); +const HubActionResult = Schema.Struct({ + ok: Schema.Boolean, + id: Schema.optional(Schema.String), + serial: Schema.optional(Schema.String), + error: Schema.optional(Schema.String), +}); + +export interface DeviceScreenshot { + readonly device: DeviceSummary; + readonly png: Uint8Array; +} + +export interface DeviceReadiness extends DeviceHost.DeviceHostReady { + readonly hostId: DeviceHostId; +} + +export interface DeviceAgentReadiness extends DeviceReadiness { + readonly agentDevice: DeviceHost.AgentDeviceEndpoint; +} + +export class DeviceService extends Context.Service< + DeviceService, + { + readonly state: Effect.Effect; + readonly subscribe: Effect.Effect, never, Scope.Scope>; + readonly configure: ( + input: DeviceConfigureInput, + ) => Effect.Effect; + /** Refreshes devices only after device support has been enabled. */ + readonly list: Effect.Effect; + readonly open: (input: DeviceOpenInput) => Effect.Effect; + readonly close: (input: DeviceCloseInput) => Effect.Effect; + readonly shutdown: (input: DeviceShutdownInput) => Effect.Effect; + /** Current settings and foreground app for one device. */ + readonly detail: (input: DeviceDetailInput) => Effect.Effect; + /** Runs one action, then returns the refreshed detail. */ + readonly action: (input: DeviceActionInput) => Effect.Effect; + readonly screenshot: (input: { + readonly hostId?: DeviceHostId | undefined; + readonly deviceId: DeviceId; + }) => Effect.Effect; + /** Host endpoints for the proxy and the provider environment. */ + readonly readiness: (hostId?: DeviceHostId) => Effect.Effect; + /** + * `readiness` only when the host can run at least one platform; a machine + * with no simulator toolchain never installs or starts anything. + */ + readonly readinessIfSupported: ( + hostId?: DeviceHostId, + ) => Effect.Effect; + readonly agentReadinessIfSupported: ( + hostId?: DeviceHostId, + ) => Effect.Effect; + readonly currentReadiness: (hostId?: DeviceHostId) => Effect.Effect; + readonly sessionsForThread: (threadId: ThreadId) => Effect.Effect>; + } +>()("t3/device/DeviceService") {} + +interface ServiceState { + readonly state: DeviceServiceState; +} + +const vendorPrefix = (platform: DevicePlatform) => + platform === "ios" ? "/vendor/serve-sim" : "/vendor/serve-emu"; + +export const make = Effect.gen(function* () { + const localHost = yield* DeviceHost.DeviceHost; + const settings = yield* ServerSettings.ServerSettingsService; + const lifecycleLock = yield* Semaphore.make(1); + const readDeviceSettings = settings.getSettings.pipe( + Effect.map((value) => ({ + enabled: value.enableDeviceSupport, + agentAccessEnabled: value.enableAgentDeviceAccess, + onboardingCompleted: value.deviceOnboardingCompleted, + })), + Effect.mapError( + (cause) => + new DeviceOperationError({ operation: "settings", reason: "settings_failed", cause }), + ), + ); + const initialSettings = yield* readDeviceSettings; + const hosts: ReadonlyMap = new Map([ + [localHost.id, localHost], + ]); + const httpClient = (yield* HttpClient.HttpClient).pipe(HttpClient.withScope); + const statePubSub = yield* PubSub.unbounded(); + const initialHosts = yield* Effect.forEach(hosts.values(), (host) => host.summary); + const stateRef = yield* SynchronizedRef.make({ + state: { + hosts: initialHosts, + hostStatus: initialSettings.enabled ? "idle" : "disabled", + devices: [], + sessions: [], + onboardingCompleted: initialSettings.onboardingCompleted, + agentAccessEnabled: initialSettings.agentAccessEnabled, + hubBasePath: DEVICE_HUB_ROUTE_PREFIX, + revision: 0, + }, + }); + + const publish = (update: (state: DeviceServiceState) => DeviceServiceState) => + SynchronizedRef.updateAndGetEffect(stateRef, ({ state }) => { + const next = { ...update(state), revision: state.revision + 1 }; + return PubSub.publish(statePubSub, next).pipe(Effect.as({ state: next })); + }).pipe(Effect.map(({ state }) => state)); + + const resolveHost = (hostId: DeviceHostId | undefined) => + Effect.gen(function* () { + const id = hostId ?? LOCAL_DEVICE_HOST_ID; + const host = hosts.get(id); + if (!host) { + return yield* new DeviceHostUnavailableError({ hostId: id, reason: "Unknown host." }); + } + return host; + }); + + const readiness: DeviceService["Service"]["readiness"] = Effect.fn("DeviceService.readiness")( + function* (hostId) { + const host = yield* resolveHost(hostId); + if (!(yield* readDeviceSettings).enabled) { + return yield* new DeviceHostUnavailableError({ + hostId: host.id, + reason: + "Device support is off. Enable it in the Device panel before installing or starting device tools.", + }); + } + const ready = yield* host + .ensureReady((phase) => + publish((state) => ({ ...state, hostStatus: phase, hostStatusDetail: undefined })).pipe( + Effect.asVoid, + ), + ) + .pipe( + Effect.tapError((error) => + publish((state) => ({ + ...state, + hostStatus: "failed", + hostStatusDetail: error.message, + })), + ), + Effect.mapError( + (error) => new DeviceHostUnavailableError({ hostId: host.id, reason: error.message }), + ), + ); + yield* SynchronizedRef.get(stateRef).pipe( + Effect.flatMap(({ state }) => + state.hostStatus === "ready" + ? Effect.void + : publish((current) => ({ + ...current, + hostStatus: "ready", + hostStatusDetail: undefined, + })), + ), + ); + return { hostId: host.id, ...ready }; + }, + lifecycleLock.withPermit, + ); + + const readinessIfSupported: DeviceService["Service"]["readinessIfSupported"] = Effect.fn( + "DeviceService.readinessIfSupported", + )(function* (hostId) { + if (!(yield* readDeviceSettings).enabled) return null; + const host = yield* resolveHost(hostId); + const summary = yield* host.summary; + if (!summary.platforms.some((platform) => platform.available)) return null; + return yield* readiness(host.id); + }); + + const agentReadinessIfSupported: DeviceService["Service"]["agentReadinessIfSupported"] = + Effect.fn("DeviceService.agentReadinessIfSupported")(function* (hostId) { + const deviceSettings = yield* readDeviceSettings; + if (!deviceSettings.enabled || !deviceSettings.agentAccessEnabled) return null; + const host = yield* resolveHost(hostId); + const summary = yield* host.summary; + if (!summary.platforms.some((platform) => platform.available)) return null; + const ready = yield* host + .ensureAgentReady((phase) => + publish((state) => ({ ...state, hostStatus: phase, hostStatusDetail: undefined })).pipe( + Effect.asVoid, + ), + ) + .pipe( + Effect.tapError((error) => + publish((state) => ({ + ...state, + hostStatus: "failed", + hostStatusDetail: error.message, + })), + ), + Effect.mapError( + (error) => new DeviceHostUnavailableError({ hostId: host.id, reason: error.message }), + ), + ); + const hostSummaries = yield* Effect.forEach(hosts.values(), (candidate) => candidate.summary); + yield* publish((state) => ({ ...state, hosts: hostSummaries, hostStatus: "ready" })); + return { hostId: host.id, ...ready }; + }, lifecycleLock.withPermit); + + const currentReadiness: DeviceService["Service"]["currentReadiness"] = (hostId) => + resolveHost(hostId).pipe( + Effect.flatMap((host) => + host.current.pipe(Effect.map((ready) => (ready ? { hostId: host.id, ...ready } : null))), + ), + Effect.orElseSucceed(() => null), + ); + + const hubJson = ( + request: HttpClientRequest.HttpClientRequest, + schema: Schema.Codec, + operation: string, + timeout: Duration.Input = Duration.seconds(15), + ) => + httpClient.execute(request).pipe( + Effect.flatMap(HttpClientResponse.filterStatusOk), + Effect.flatMap(HttpClientResponse.schemaBodyJson(schema)), + Effect.scoped, + Effect.timeout(timeout), + Effect.mapError( + (cause) => + new DeviceOperationError({ + operation, + reason: "request_failed", + cause, + }), + ), + ); + + const fetchDevices = Effect.fn("DeviceService.fetchDevices")(function* (ready: DeviceReadiness) { + const list = yield* hubJson( + HttpClientRequest.get(`${ready.hub.origin}/api/devices`), + HubDeviceList, + "list", + ); + const toSummary = (device: typeof HubDevice.Type): DeviceSummary => ({ + hostId: ready.hostId, + id: device.id, + platform: device.platform, + name: device.name, + version: device.version, + booted: device.booted, + physical: device.physical, + }); + const devices = [...list.simulators, ...list.emulators].map(toSummary); + const host = yield* resolveHost(ready.hostId); + if ((yield* host.platformAvailability("android")).available) { + const avds = yield* ready.run("emulator", ["-list-avds"]); + if (avds.code !== 0) { + return yield* new DeviceOperationError({ + operation: "list", + reason: "command_failed", + exitCode: avds.code, + cause: avds, + }); + } + for (const name of avds.stdout + .split(/\r?\n/) + .map((line) => line.trim()) + .filter(Boolean)) { + if (!devices.some((device) => device.platform === "android" && device.name === name)) { + devices.push({ + hostId: ready.hostId, + id: name, + name, + platform: "android", + version: "Android", + booted: false, + physical: false, + }); + } + } + } + return { devices, detail: list.errors?.map((error) => error.message).join("\n") || undefined }; + }); + + const refresh = Effect.fn("DeviceService.refresh")(function* (ready: DeviceReadiness) { + const { devices, detail } = yield* fetchDevices(ready); + const hostSummaries = yield* Effect.forEach(hosts.values(), (host) => host.summary); + return yield* lifecycleLock.withPermit( + Effect.gen(function* () { + if (!(yield* readDeviceSettings).enabled) + return (yield* SynchronizedRef.get(stateRef)).state; + return yield* publish((state) => ({ + ...state, + hosts: hostSummaries, + devices, + hostStatusDetail: detail, + })); + }), + ); + }); + + const list: DeviceService["Service"]["list"] = Effect.gen(function* () { + if (!(yield* readDeviceSettings).enabled) return (yield* SynchronizedRef.get(stateRef)).state; + const ready = yield* readiness(); + return yield* refresh(ready); + }).pipe( + Effect.tapError((error) => + publish((state) => + state.hostStatus === "disabled" + ? state + : { ...state, hostStatus: "failed", hostStatusDetail: error.message }, + ), + ), + Effect.withSpan("DeviceService.list"), + ); + + const configure: DeviceService["Service"]["configure"] = Effect.fn("DeviceService.configure")( + function* (input) { + const currentSettings = yield* readDeviceSettings; + const nextEnabled = input.enabled ?? currentSettings.enabled; + const nextAgentAccess = input.agentAccessEnabled ?? currentSettings.agentAccessEnabled; + yield* lifecycleLock.withPermit( + Effect.gen(function* () { + yield* settings + .updateSettings({ + ...(input.enabled === undefined ? {} : { enableDeviceSupport: input.enabled }), + ...(input.agentAccessEnabled === undefined + ? {} + : { enableAgentDeviceAccess: input.agentAccessEnabled }), + ...(input.onboardingCompleted === undefined + ? {} + : { deviceOnboardingCompleted: input.onboardingCompleted }), + }) + .pipe( + Effect.mapError( + (cause) => + new DeviceOperationError({ + operation: "configure", + reason: "settings_failed", + cause, + }), + ), + ); + if (!nextEnabled) { + yield* Effect.forEach(hosts.values(), (host) => host.stop, { discard: true }); + } else if (input.agentAccessEnabled === false) { + yield* Effect.forEach(hosts.values(), (host) => host.stopAgent, { discard: true }); + } + yield* publish((state) => ({ + ...state, + hostStatus: nextEnabled ? "idle" : "disabled", + hostStatusDetail: undefined, + devices: nextEnabled ? state.devices : [], + sessions: nextEnabled ? state.sessions : [], + bootingDevices: nextEnabled ? state.bootingDevices : [], + agentAccessEnabled: nextAgentAccess, + onboardingCompleted: input.onboardingCompleted ?? state.onboardingCompleted, + })); + }), + ); + if (nextEnabled && nextAgentAccess && input.agentAccessEnabled === true) { + yield* agentReadinessIfSupported(); + } + return yield* list; + }, + ); + + const findDevice = ( + state: DeviceServiceState, + hostId: DeviceHostId, + deviceId: DeviceId, + ): DeviceSummary | undefined => + state.devices.find((device) => device.hostId === hostId && device.id === deviceId); + + const ensurePlatform = Effect.fn("DeviceService.ensurePlatform")(function* ( + host: DeviceHost.DeviceHost["Service"], + platform: DevicePlatform, + ) { + const availability = yield* host.platformAvailability(platform); + if (!availability.available) { + return yield* new DevicePlatformUnavailableError({ + hostId: host.id, + platform, + reason: availability.reason ?? "Platform toolchain missing.", + }); + } + }); + + /** + * Boot through the hub so its device list and the streaming helper both see + * the device come up. Android AVDs change id when they boot (AVD name to + * emulator serial), so the returned id is authoritative. + */ + const boot = Effect.fn("DeviceService.boot")(function* ( + ready: DeviceReadiness, + device: DeviceSummary, + ) { + const result = yield* HttpClientRequest.post(`${ready.hub.origin}/api/devices/boot`).pipe( + HttpClientRequest.bodyJson({ platform: device.platform, id: device.id, name: device.name }), + Effect.mapError( + (cause) => + new DeviceOperationError({ operation: "boot", reason: "invalid_payload", cause }), + ), + Effect.flatMap((request) => hubJson(request, HubActionResult, "boot", BOOT_TIMEOUT)), + ); + if (!result.ok) { + return yield* new DeviceBootError({ + hostId: ready.hostId, + deviceId: device.id, + reason: /insufficient.*(?:disk|space)|not enough.*(?:disk|space)|no space left/i.test( + result.error ?? "", + ) + ? "disk_space" + : /timed? out|timeout/i.test(result.error ?? "") + ? "timeout" + : "launch_failed", + cause: result, + }); + } + if (device.platform === "ios") { + // Booting alone does not attach a serve-sim helper; the grid start + // does both and is idempotent for a booted simulator. + yield* HttpClientRequest.post( + `${ready.hub.origin}${vendorPrefix("ios")}/grid/api/start`, + ).pipe( + HttpClientRequest.bodyJson({ udid: device.id }), + Effect.mapError( + (cause) => + new DeviceOperationError({ operation: "boot", reason: "invalid_payload", cause }), + ), + Effect.flatMap((request) => + hubJson(request, HubActionResult, "attach stream", BOOT_TIMEOUT), + ), + ); + } + return result.serial ?? result.id ?? device.id; + }); + + const open: DeviceService["Service"]["open"] = Effect.fn("DeviceService.open")(function* (input) { + const host = yield* resolveHost(input.hostId); + yield* ensurePlatform(host, input.platform); + const ready = yield* readiness(host.id); + let state = yield* refresh(ready); + let device = findDevice(state, host.id, input.deviceId); + if (!device) { + return yield* new DeviceNotFoundError({ hostId: host.id, deviceId: input.deviceId }); + } + if (!device.booted && input.boot !== false) { + const booting = { ...device, threadId: input.threadId }; + yield* publish((current) => ({ + ...current, + bootingDevices: [ + ...(current.bootingDevices ?? []).filter( + (entry) => entry.hostId !== booting.hostId || entry.id !== booting.id, + ), + booting, + ], + })); + const bootedId = yield* boot(ready, device).pipe( + Effect.ensuring( + publish((current) => ({ + ...current, + bootingDevices: (current.bootingDevices ?? []).filter( + (entry) => entry.hostId !== booting.hostId || entry.id !== booting.id, + ), + })), + ), + ); + state = yield* refresh(ready); + device = findDevice(state, host.id, bootedId) ?? findDevice(state, host.id, device.id); + if (!device) { + return yield* new DeviceNotFoundError({ hostId: host.id, deviceId: bootedId }); + } + } else if (device.platform === "ios" && device.booted) { + // A simulator booted outside T3 has no helper attached yet. + yield* HttpClientRequest.post( + `${ready.hub.origin}${vendorPrefix("ios")}/grid/api/start`, + ).pipe( + HttpClientRequest.bodyJson({ udid: device.id }), + Effect.mapError( + (cause) => + new DeviceOperationError({ operation: "open", reason: "invalid_payload", cause }), + ), + Effect.flatMap((request) => + hubJson(request, HubActionResult, "attach stream", BOOT_TIMEOUT), + ), + ); + } + const openedAt = DateTime.formatIso(yield* DateTime.now); + const session: DeviceSession = { + threadId: input.threadId, + hostId: host.id, + deviceId: device.id, + platform: device.platform, + openedAt, + }; + yield* lifecycleLock.withPermit( + Effect.gen(function* () { + if (!(yield* readDeviceSettings).enabled) + return yield* new DeviceHostUnavailableError({ + hostId: host.id, + reason: "Device support was turned off while the device was opening.", + }); + yield* publish((current) => ({ + ...current, + sessions: [ + ...current.sessions.filter( + (existing) => + !( + existing.threadId === session.threadId && + existing.hostId === session.hostId && + existing.deviceId === session.deviceId + ), + ), + session, + ], + })); + }), + ); + return session; + }); + + const shutdownDevice = Effect.fn("DeviceService.shutdownDevice")(function* ( + hostId: DeviceHostId, + deviceId: DeviceId, + platform: DevicePlatform, + ) { + const ready = yield* readiness(hostId); + yield* HttpClientRequest.post(`${ready.hub.origin}/api/devices/shutdown`).pipe( + HttpClientRequest.bodyJson({ platform, id: deviceId }), + Effect.mapError( + (cause) => + new DeviceOperationError({ operation: "shutdown", reason: "invalid_payload", cause }), + ), + Effect.flatMap((request) => hubJson(request, HubActionResult, "shutdown")), + Effect.flatMap((result) => + result.ok + ? Effect.void + : Effect.fail( + new DeviceOperationError({ + operation: "shutdown", + reason: "hub_rejected", + cause: result, + }), + ), + ), + ); + yield* publish((state) => ({ + ...state, + devices: state.devices.map((device) => + device.hostId === ready.hostId && device.id === deviceId + ? { ...device, booted: false } + : device, + ), + sessions: state.sessions.filter( + (session) => !(session.hostId === ready.hostId && session.deviceId === deviceId), + ), + })); + // Discovery can stall while an emulator saves its snapshot. A failed + // refresh must not turn an accepted shutdown into an action failure. + yield* refresh(ready).pipe( + Effect.catch((cause) => + Effect.logWarning("Device discovery unavailable after shutdown", { cause }), + ), + ); + }); + + const close: DeviceService["Service"]["close"] = Effect.fn("DeviceService.close")( + function* (input) { + const { state } = yield* SynchronizedRef.get(stateRef); + const closing = state.sessions.filter( + (session) => + session.threadId === input.threadId && + (input.deviceId === undefined || session.deviceId === input.deviceId), + ); + if (closing.length === 0) return; + yield* publish((current) => ({ + ...current, + sessions: current.sessions.filter((session) => !closing.includes(session)), + })); + if (input.shutdown) { + yield* Effect.forEach( + closing, + (session) => shutdownDevice(session.hostId, session.deviceId, session.platform), + { discard: true }, + ); + } + }, + ); + + const shutdown: DeviceService["Service"]["shutdown"] = Effect.fn("DeviceService.shutdown")( + function* (input) { + const host = yield* resolveHost(input.hostId); + yield* shutdownDevice(host.id, input.deviceId, input.platform); + // Sessions on a powered-off device are stale in every thread. + yield* publish((current) => ({ + ...current, + sessions: current.sessions.filter( + (session) => !(session.hostId === host.id && session.deviceId === input.deviceId), + ), + })); + }, + ); + + const screenshot: DeviceService["Service"]["screenshot"] = Effect.fn("DeviceService.screenshot")( + function* (input) { + const { ready, device } = yield* resolveDevice(input.hostId, input.deviceId); + const url = `${ready.hub.origin}${vendorPrefix(device.platform)}/api/screenshot?device=${encodeURIComponent(device.id)}`; + const png = yield* httpClient.execute(HttpClientRequest.post(url)).pipe( + Effect.flatMap(HttpClientResponse.filterStatusOk), + Effect.flatMap((response) => response.arrayBuffer), + Effect.map((buffer) => new Uint8Array(buffer)), + Effect.scoped, + Effect.timeout(SCREENSHOT_TIMEOUT), + Effect.mapError( + (cause) => + new DeviceOperationError({ + operation: "screenshot", + reason: "request_failed", + cause, + }), + ), + ); + return { device, png }; + }, + ); + + const resolveDevice = Effect.fn("DeviceService.resolveDevice")(function* ( + hostId: DeviceHostId | undefined, + deviceId: DeviceId, + ) { + const host = yield* resolveHost(hostId); + const ready = yield* readiness(host.id); + const { state } = yield* SynchronizedRef.get(stateRef); + const device = + findDevice(state, host.id, deviceId) ?? findDevice(yield* refresh(ready), host.id, deviceId); + if (!device) return yield* new DeviceNotFoundError({ hostId: host.id, deviceId }); + return { ready, device }; + }); + + const detail: DeviceService["Service"]["detail"] = Effect.fn("DeviceService.detail")( + function* (input) { + const { ready, device } = yield* resolveDevice(input.hostId, input.deviceId); + const read = yield* readDeviceDetail(ready, device.platform, device.id); + return { + hostId: ready.hostId, + deviceId: device.id, + settings: read.settings, + foregroundApp: read.foregroundApp, + readAt: DateTime.formatIso(yield* DateTime.now), + }; + }, + ); + + const action: DeviceService["Service"]["action"] = Effect.fn("DeviceService.action")( + function* (input) { + const { ready, device } = yield* resolveDevice(input.hostId, input.deviceId); + yield* runDeviceAction(ready, device.platform, input); + return yield* detail({ hostId: ready.hostId, deviceId: device.id }); + }, + ); + + const sessionsForThread: DeviceService["Service"]["sessionsForThread"] = (threadId) => + SynchronizedRef.get(stateRef).pipe( + Effect.map(({ state }) => state.sessions.filter((session) => session.threadId === threadId)), + ); + + return DeviceService.of({ + state: SynchronizedRef.get(stateRef).pipe(Effect.map(({ state }) => state)), + subscribe: PubSub.subscribe(statePubSub), + configure, + list, + open, + close, + shutdown, + detail, + action, + screenshot, + readiness, + readinessIfSupported, + agentReadinessIfSupported, + currentReadiness, + sessionsForThread, + }); +}); + +export const layer = Layer.effect(DeviceService, make).pipe(Layer.provide(LocalDeviceHost.layer)); + +/** State stream for WS subscribers: current snapshot first, then every change. */ +export const stateStream = (service: DeviceService["Service"]): Stream.Stream => + Stream.unwrap( + Effect.gen(function* () { + // Subscribe before reading the snapshot so no change between the two + // is lost; the scope lives as long as the stream does. + const subscription = yield* service.subscribe; + const initial = yield* service.state; + return Stream.concat(Stream.make(initial), Stream.fromSubscription(subscription)); + }), + ).pipe(Stream.scoped); diff --git a/apps/server/src/device/DeviceToolchain.test.ts b/apps/server/src/device/DeviceToolchain.test.ts new file mode 100644 index 000000000000..9f70ca91a8e1 --- /dev/null +++ b/apps/server/src/device/DeviceToolchain.test.ts @@ -0,0 +1,39 @@ +import { expect, it } from "@effect/vitest"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; +import * as ChildProcessSpawner from "effect/unstable/process/ChildProcessSpawner"; + +import * as ProcessRunner from "../processRunner.ts"; +import { ensureDeviceHub, isDeviceHubInstalled } from "./DeviceToolchain.ts"; + +it.effect("failed installation cleans staging and exposes only a safe failure message", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-device-install-" }); + const result = { + code: ChildProcessSpawner.ExitCode(1), + stdout: "", + stderr: "registry rejected https://private:credential@example.test/package", + timedOut: false, + stdoutTruncated: false, + stderrTruncated: false, + stdoutInvalidUtf8: false, + stderrInvalidUtf8: false, + }; + const error = yield* ensureDeviceHub(baseDir).pipe( + Effect.provideService(ProcessRunner.ProcessRunner, { + run: () => Effect.succeed(result), + }), + Effect.flip, + ); + expect(error.message).toBe( + "Installing expo-device-hub failed while running npm install (exit code 1).", + ); + expect(error.cause).toBe(result); + expect(yield* isDeviceHubInstalled(baseDir)).toBe(false); + expect(yield* fs.readDirectory(path.join(baseDir, "tools", "expo-device-hub"))).toEqual([]); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), +); diff --git a/apps/server/src/device/DeviceToolchain.ts b/apps/server/src/device/DeviceToolchain.ts new file mode 100644 index 000000000000..e6b43cd81ee8 --- /dev/null +++ b/apps/server/src/device/DeviceToolchain.ts @@ -0,0 +1,223 @@ +/** + * Pinned installs of the two external tools device support is built on. + * + * `expo-device-hub` streams simulator and emulator screens and `agent-device` + * drives them. Each is npm-installed separately after its matching consent + * step into `/tools//` and executed from there with the + * server's own Node, never `npx`: an ephemeral + * npx cache would make every first `device_open` after a reboot depend on the + * registry, and the pinned versions are part of the contract the injected + * agent instructions describe. + * + * Install follows the pinned-runtime recipe: stage into a temp sibling, write a + * sentinel only after npm exits 0, then rename into place. npm extracts files + * before it finishes, so an entry file alone does not prove a usable tree. + */ +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Option from "effect/Option"; +import * as Path from "effect/Path"; +import * as PlatformError from "effect/PlatformError"; +import * as Schema from "effect/Schema"; +import * as Semaphore from "effect/Semaphore"; + +import * as ProcessRunner from "../processRunner.ts"; + +const DEVICE_HUB_PACKAGE = "expo-device-hub"; +const DEVICE_HUB_VERSION = "0.9.0"; +const AGENT_DEVICE_PACKAGE = "agent-device"; +const AGENT_DEVICE_VERSION = "0.20.10"; + +const INSTALL_TIMEOUT = Duration.minutes(10); +const installLock = Semaphore.makeUnsafe(1); + +export interface DeviceToolPaths { + readonly installDir: string; + /** Absolute path of the tool's entry script, run with the server's Node. */ + readonly entryPath: string; + readonly sentinelPath: string; +} + +export interface DeviceToolchainPaths { + readonly hub: DeviceToolPaths; + readonly agentDevice: DeviceToolPaths; +} + +export class DeviceToolchainInstallError extends Schema.TaggedError()( + "DeviceToolchainInstallError", + { + tool: Schema.String, + step: Schema.String, + exitCode: Schema.optional(Schema.Number), + cause: Schema.optional(Schema.Defect()), + }, +) { + override get message(): string { + const suffix = this.exitCode === undefined ? "" : ` (exit code ${this.exitCode})`; + return `Installing ${this.tool} failed while ${this.step}${suffix}.`; + } +} + +interface ToolSpec { + readonly name: string; + readonly version: string; + readonly entry: ReadonlyArray; +} + +const HUB_SPEC: ToolSpec = { + name: DEVICE_HUB_PACKAGE, + version: DEVICE_HUB_VERSION, + entry: ["dist", "server", "cli.mjs"], +}; + +const AGENT_DEVICE_SPEC: ToolSpec = { + name: AGENT_DEVICE_PACKAGE, + version: AGENT_DEVICE_VERSION, + entry: ["bin", "agent-device.mjs"], +}; + +const toolPaths = (path: Path.Path, baseDir: string, spec: ToolSpec): DeviceToolPaths => { + const installDir = path.join(baseDir, "tools", spec.name, spec.version); + return { + installDir, + entryPath: path.join(installDir, "node_modules", spec.name, ...spec.entry), + sentinelPath: path.join(installDir, ".install-complete"), + }; +}; + +const deviceToolchainPaths = (path: Path.Path, baseDir: string): DeviceToolchainPaths => ({ + hub: toolPaths(path, baseDir, HUB_SPEC), + agentDevice: toolPaths(path, baseDir, AGENT_DEVICE_SPEC), +}); + +/** Keep daemon state (daemon.json, sessions) in userdata, separate from tool installs. */ +export const agentDeviceStateDir = (path: Path.Path, stateDir: string): string => + path.join(stateDir, "device", "agent-device"); + +const isInstalled = Effect.fn("DeviceToolchain.isInstalled")(function* ( + fs: FileSystem.FileSystem, + paths: DeviceToolPaths, + version: string, +) { + const [entryExists, sentinel] = yield* Effect.all([ + fs.exists(paths.entryPath), + fs.readFileString(paths.sentinelPath).pipe(Effect.option), + ]).pipe(Effect.orElseSucceed(() => [false, Option.none()] as const)); + return entryExists && Option.isSome(sentinel) && sentinel.value.trim() === version; +}); + +const installTool = Effect.fn("DeviceToolchain.installTool")(function* ( + spec: ToolSpec, + paths: DeviceToolPaths, +) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const runner = yield* ProcessRunner.ProcessRunner; + const fail = (step: string) => (cause: unknown) => + new DeviceToolchainInstallError({ tool: spec.name, step, cause }); + + if (yield* isInstalled(fs, paths, spec.version)) return paths; + + const parentDir = path.dirname(paths.installDir); + yield* fs + .remove(paths.installDir, { recursive: true, force: true }) + .pipe(Effect.mapError(fail("removing an incomplete install"))); + yield* fs + .makeDirectory(parentDir, { recursive: true }) + .pipe(Effect.mapError(fail("preparing the install directory"))); + const stagingDir = yield* fs + .makeTempDirectory({ directory: parentDir, prefix: ".staging-" }) + .pipe(Effect.mapError(fail("preparing the install directory"))); + + return yield* Effect.gen(function* () { + const installArgs = [ + "install", + "--prefix", + stagingDir, + "--no-fund", + "--no-audit", + `${spec.name}@${spec.version}`, + ]; + const result = yield* runner + .run({ command: "npm", args: installArgs, timeout: INSTALL_TIMEOUT }) + .pipe( + Effect.catchTags({ + ProcessSpawnError: (error) => + error.cause instanceof PlatformError.PlatformError && + error.cause.reason._tag === "NotFound" + ? runner.run({ + command: "pnpm", + args: ["--package=npm@11", "dlx", "npm", ...installArgs], + timeout: INSTALL_TIMEOUT, + }) + : Effect.fail(error), + }), + Effect.mapError(fail("running npm install")), + ); + if (result.code !== 0) { + return yield* new DeviceToolchainInstallError({ + tool: spec.name, + step: "running npm install", + exitCode: Number(result.code), + cause: result, + }); + } + const stagedEntry = path.join(stagingDir, "node_modules", spec.name, ...spec.entry); + if (!(yield* fs.exists(stagedEntry).pipe(Effect.orElseSucceed(() => false)))) { + return yield* new DeviceToolchainInstallError({ + tool: spec.name, + step: "verifying the installed entry point", + }); + } + yield* fs + .writeFileString(path.join(stagingDir, ".install-complete"), `${spec.version}\n`) + .pipe(Effect.mapError(fail("recording the completed install"))); + yield* fs.rename(stagingDir, paths.installDir).pipe( + Effect.catch((cause) => + // A concurrent server may have published the same version first. + isInstalled(fs, paths, spec.version).pipe( + Effect.flatMap((published) => + published ? Effect.void : Effect.fail(fail("publishing the install")(cause)), + ), + ), + ), + ); + return paths; + }).pipe( + Effect.ensuring(fs.remove(stagingDir, { recursive: true, force: true }).pipe(Effect.ignore)), + ); +}); + +const ensureTool = Effect.fn("DeviceToolchain.ensureTool")(function* ( + baseDir: string, + spec: ToolSpec, + select: (paths: DeviceToolchainPaths) => DeviceToolPaths, +) { + const path = yield* Path.Path; + const paths = deviceToolchainPaths(path, baseDir); + return yield* installLock.withPermit(installTool(spec, select(paths))); +}); + +export const ensureDeviceHub = (baseDir: string) => + ensureTool(baseDir, HUB_SPEC, (paths) => paths.hub); + +export const ensureAgentDevice = (baseDir: string) => + ensureTool(baseDir, AGENT_DEVICE_SPEC, (paths) => paths.agentDevice); + +const isToolInstalled = Effect.fn("DeviceToolchain.isToolInstalled")(function* ( + baseDir: string, + spec: ToolSpec, + select: (paths: DeviceToolchainPaths) => DeviceToolPaths, +) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const paths = deviceToolchainPaths(path, baseDir); + return yield* isInstalled(fs, select(paths), spec.version); +}); + +export const isDeviceHubInstalled = (baseDir: string) => + isToolInstalled(baseDir, HUB_SPEC, (paths) => paths.hub); + +export const isAgentDeviceInstalled = (baseDir: string) => + isToolInstalled(baseDir, AGENT_DEVICE_SPEC, (paths) => paths.agentDevice); diff --git a/apps/server/src/device/LocalDeviceHost.test.ts b/apps/server/src/device/LocalDeviceHost.test.ts new file mode 100644 index 000000000000..17d274e73529 --- /dev/null +++ b/apps/server/src/device/LocalDeviceHost.test.ts @@ -0,0 +1,114 @@ +import { describe, expect, it } from "@effect/vitest"; +import * as NodePath from "@effect/platform-node/NodePath"; +import { HostProcessEnvironment, HostProcessPlatform } from "@t3tools/shared/hostProcess"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Path from "effect/Path"; +import * as FileSystem from "effect/FileSystem"; + +import * as LocalDeviceHost from "./LocalDeviceHost.ts"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import * as ChildProcessSpawner from "effect/unstable/process/ChildProcessSpawner"; +import { HttpClient } from "effect/unstable/http"; +import * as NetService from "@t3tools/shared/Net"; +import * as ServerConfig from "../config.ts"; +import * as ProcessRunner from "../processRunner.ts"; + +const diagnose = (files: ReadonlyArray, environment: NodeJS.ProcessEnv) => + LocalDeviceHost.__testing.platformReason("android").pipe( + Effect.provideService(HostProcessEnvironment, environment), + Effect.provideService(HostProcessPlatform, "darwin"), + Effect.provideService( + FileSystem.FileSystem, + FileSystem.makeNoop({ + exists: (file) => Effect.succeed(files.includes(file)), + }), + ), + Effect.provide(NodePath.layer), + ); + +describe("Android SDK availability", () => { + it.effect("explains that adb alone is insufficient to launch an emulator", () => + Effect.gen(function* () { + const reason = yield* diagnose(["/sdk/platform-tools/adb"], { ANDROID_HOME: "/sdk" }); + expect(reason).toContain("Android Emulator is missing"); + }), + ); + + it.effect("identifies command-line tools required by the device hub", () => + Effect.gen(function* () { + const reason = yield* diagnose(["/sdk/platform-tools/adb", "/sdk/emulator/emulator"], { + ANDROID_HOME: "/sdk", + }); + expect(reason).toContain("Command-line Tools (latest)"); + }), + ); + + it.effect("discovers the standard macOS SDK without ANDROID_HOME", () => + Effect.gen(function* () { + const root = "/test/home/Library/Android/sdk"; + const reason = yield* diagnose( + [ + `${root}/platform-tools/adb`, + `${root}/emulator/emulator`, + `${root}/cmdline-tools/latest/bin/avdmanager`, + ], + { HOME: "/test/home" }, + ); + expect(reason).toBeNull(); + }), + ); + + it.effect("reports an absent SDK without running or installing tools", () => + Effect.gen(function* () { + expect(yield* diagnose([], { HOME: "/test/home" })).toContain("Android SDK was not found"); + }), + ); +}); + +it.effect("puts detected Android tools on the helper PATH without losing existing commands", () => + Effect.gen(function* () { + const path = yield* Path.Path; + const environment = LocalDeviceHost.__testing.deviceHostEnvironment( + { PATH: "/usr/bin", HOME: "/test/home" }, + "/sdk", + "darwin", + path, + ); + expect(environment.PATH).toBe("/sdk/platform-tools:/sdk/emulator:/usr/bin"); + expect(environment.ANDROID_HOME).toBe("/sdk"); + expect(environment.HOME).toBe("/test/home"); + }).pipe(Effect.provide(NodePath.layer)), +); + +it.effect( + "constructs and inspects an unconfigured host without installing or starting helpers", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-device-consent-" }); + const host = yield* LocalDeviceHost.make().pipe( + Effect.provide(Layer.mergeAll(ServerConfig.layerTest(baseDir, baseDir), NetService.layer)), + Effect.provideService(HostProcessEnvironment, { HOME: baseDir, PATH: "" }), + Effect.provideService(HostProcessPlatform, "linux"), + Effect.provideService( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make(() => + Effect.die(new Error("Host construction must not spawn processes")), + ), + ), + Effect.provideService(ProcessRunner.ProcessRunner, { + run: () => Effect.die(new Error("Host construction must not run commands")), + }), + Effect.provideService( + HttpClient.HttpClient, + HttpClient.make(() => + Effect.die(new Error("Host construction must not make network requests")), + ), + ), + ); + expect(yield* host.current).toBeNull(); + yield* host.stop; + expect(yield* fs.exists(`${baseDir}/tools`)).toBe(false); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), +); diff --git a/apps/server/src/device/LocalDeviceHost.ts b/apps/server/src/device/LocalDeviceHost.ts new file mode 100644 index 000000000000..b24ebfbd98b6 --- /dev/null +++ b/apps/server/src/device/LocalDeviceHost.ts @@ -0,0 +1,697 @@ +/** + * The device host that is this machine. + * + * Runs expo-device-hub as a supervised child on a loopback port and starts the + * agent-device daemon in HTTP mode under a T3-owned state directory. Both are + * lazy: the device service requires explicit setup consent before it calls + * ensureReady to install tools or start helper processes. + * + * The hub runs in its standalone mode (origin root). The T3 proxy strips its + * own prefix, and the Device panel derives stream and socket URLs from the + * prefix itself rather than from anything the hub prints. + */ +import { + type DeviceHostSummary, + type DevicePlatform, + type DevicePlatformAvailability, + LOCAL_DEVICE_HOST_ID, +} from "@t3tools/contracts"; +import { waitForHttpReady } from "@t3tools/shared/httpReadiness"; +import { HostProcessEnvironment, HostProcessPlatform } from "@t3tools/shared/hostProcess"; +import * as NetService from "@t3tools/shared/Net"; +import { isCommandAvailable } from "@t3tools/shared/shell"; +import * as Clock from "effect/Clock"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Path from "effect/Path"; +import * as Ref from "effect/Ref"; +import * as Schema from "effect/Schema"; +import * as Scope from "effect/Scope"; +import * as Semaphore from "effect/Semaphore"; +import * as Stream from "effect/Stream"; +import { HttpClient } from "effect/unstable/http"; +import * as ChildProcess from "effect/unstable/process/ChildProcess"; +import * as ChildProcessSpawner from "effect/unstable/process/ChildProcessSpawner"; + +import * as ServerConfig from "../config.ts"; +import * as ProcessRunner from "../processRunner.ts"; +import * as DeviceHost from "./DeviceHost.ts"; +import { + agentDeviceStateDir, + type DeviceToolPaths, + ensureAgentDevice, + ensureDeviceHub, + isAgentDeviceInstalled, + isDeviceHubInstalled, +} from "./DeviceToolchain.ts"; + +const HUB_READY_TIMEOUT_MS = 30_000; +const DAEMON_READY_TIMEOUT_MS = 30_000; +const DAEMON_POLL_MS = 100; +const HUB_RESTART_STABLE_UPTIME_MS = 60_000; +const HUB_RESTART_MAX_DELAY_MS = 30_000; + +/** + * Written beside the agent-device state so a server that dies without running + * its finalizers (SIGKILL, dev-runner restarts) does not leave a hub bound to + * a loopback port forever. The next start reads it, kills only a process that + * is still that hub, and replaces the file. + */ +const HubStateFile = Schema.Struct({ + pid: Schema.Int, + port: Schema.Int, + entryPath: Schema.String, +}); +const decodeHubStateFile = Schema.decodeUnknownEffect(Schema.fromJsonString(HubStateFile)); +const encodeHubStateFile = Schema.encodeUnknownEffect(Schema.fromJsonString(HubStateFile)); + +const AgentDeviceDaemonFile = Schema.Struct({ + httpPort: Schema.Int, + token: Schema.String, + pid: Schema.optional(Schema.Int), +}); +const decodeDaemonFile = Schema.decodeUnknownEffect(Schema.fromJsonString(AgentDeviceDaemonFile)); + +interface HubProcess { + readonly child: ChildProcessSpawner.ChildProcessHandle; + readonly scope: Scope.Closeable; + readonly origin: string; + readonly startedAtMillis: number; +} + +interface RunningHost { + readonly hub: HubProcess; + readonly agentDevice: DeviceHost.AgentDeviceEndpoint | null; + readonly helpers: DeviceHost.DeviceHostReady["helpers"]; +} + +const platformReason = Effect.fn("LocalDeviceHost.platformReason")(function* ( + platform: DevicePlatform, +): Effect.fn.Return { + const hostPlatform = yield* HostProcessPlatform; + if (platform === "ios") { + if (hostPlatform !== "darwin") return "iOS Simulators need macOS with Xcode."; + if (!(yield* isCommandAvailable("xcrun"))) return "Xcode command line tools were not found."; + return null; + } + const sdk = yield* androidSdk; + if (!sdk.root) + return "Android SDK was not found. Install it with Android Studio or set ANDROID_HOME to your SDK directory."; + if (!sdk.adb) + return `Android SDK Platform-Tools are missing from ${sdk.root}. Install them in Android Studio's SDK Manager.`; + if (!sdk.emulator) + return `Android Emulator is missing from ${sdk.root}. Install it in Android Studio's SDK Manager.`; + if (!sdk.avdmanager) + return `Android SDK Command-line Tools (latest) are missing from ${sdk.root}. Install them in Android Studio's SDK Manager.`; + return null; +}); + +/** Resolve the SDK once for both diagnostics and the environment passed to helpers. */ +const androidSdk = Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const environment = yield* HostProcessEnvironment; + const platform = yield* HostProcessPlatform; + const home = environment.HOME ?? environment.USERPROFILE ?? ""; + const explicit = environment.ANDROID_HOME?.trim() || environment.ANDROID_SDK_ROOT?.trim(); + const candidates = explicit + ? [explicit] + : [ + path.join(home, "Library", "Android", "sdk"), + path.join(home, "Android", "Sdk"), + path.join( + environment.LOCALAPPDATA ?? path.join(home, "AppData", "Local"), + "Android", + "Sdk", + ), + ]; + if (!explicit) { + for (const directory of (environment.PATH ?? "").split(platform === "win32" ? ";" : ":")) { + if (!directory) continue; + const resolved = yield* fs + .realPath(path.join(directory, platform === "win32" ? "adb.exe" : "adb")) + .pipe(Effect.option); + if (resolved._tag === "Some") candidates.push(path.dirname(path.dirname(resolved.value))); + } + } + const exists = (file: string) => fs.exists(file).pipe(Effect.orElseSucceed(() => false)); + for (const root of candidates) { + const adb = yield* exists( + path.join(root, "platform-tools", platform === "win32" ? "adb.exe" : "adb"), + ); + const emulator = yield* exists( + path.join(root, "emulator", platform === "win32" ? "emulator.exe" : "emulator"), + ); + if (explicit || adb || emulator) { + const avdmanager = yield* exists( + path.join( + root, + "cmdline-tools", + "latest", + "bin", + platform === "win32" ? "avdmanager.bat" : "avdmanager", + ), + ); + return { root, adb, emulator, avdmanager }; + } + } + return { root: null, adb: false, emulator: false, avdmanager: false }; +}); + +const deviceHostEnvironment = ( + environment: NodeJS.ProcessEnv, + sdkRoot: string | null, + hostPlatform: NodeJS.Platform, + path: Path.Path, +): NodeJS.ProcessEnv => { + return sdkRoot + ? { + ...environment, + ANDROID_HOME: sdkRoot, + PATH: [ + path.join(sdkRoot, "platform-tools"), + path.join(sdkRoot, "emulator"), + environment.PATH ?? environment.Path ?? "", + ].join(hostPlatform === "win32" ? ";" : ":"), + } + : environment; +}; + +export const make = Effect.fn("LocalDeviceHost.make")(function* () { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const config = yield* ServerConfig.ServerConfig; + const path = yield* Path.Path; + const fs = yield* FileSystem.FileSystem; + const net = yield* NetService.NetService; + const runner = yield* ProcessRunner.ProcessRunner; + const httpClient = yield* HttpClient.HttpClient; + const environment = yield* HostProcessEnvironment; + const hostPlatform = yield* HostProcessPlatform; + const sdk = yield* androidSdk; + const hostEnvironment = deviceHostEnvironment(environment, sdk.root, hostPlatform, path); + const startLock = yield* Semaphore.make(1); + const runningRef = yield* Ref.make(null); + const restartDelayRef = yield* Ref.make(0); + const hostId = LOCAL_DEVICE_HOST_ID; + + const platformAvailability = Effect.fn("LocalDeviceHost.platformAvailability")(function* ( + platform: DevicePlatform, + ): Effect.fn.Return { + const reason = yield* platformReason(platform).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Path.Path, path), + ); + return reason === null ? { platform, available: true } : { platform, available: false, reason }; + }); + + const summary: Effect.Effect = Effect.gen(function* () { + const [platforms, hubInstalled, agentDeviceInstalled] = yield* Effect.all([ + Effect.all([platformAvailability("ios"), platformAvailability("android")]), + isDeviceHubInstalled(config.baseDir).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Path.Path, path), + ), + isAgentDeviceInstalled(config.baseDir).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Path.Path, path), + ), + ]); + return { + id: hostId, + kind: "local", + label: "This machine", + platforms, + hubInstalled, + agentDeviceInstalled, + }; + }); + + const hubEnvironment = (): NodeJS.ProcessEnv => ({ + ...hostEnvironment, + FORCE_COLOR: "0", + NO_COLOR: "1", + }); + + const stopHub = (hub: HubProcess | undefined) => + hub ? Scope.close(hub.scope, Exit.void).pipe(Effect.ignore) : Effect.void; + + const hubStatePath = () => path.join(agentDeviceStateDir(path, config.stateDir), "hub.json"); + + const isProcessAlive = (pid: number) => + Effect.sync(() => { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } + }); + + /** + * A hub left behind by a previous server is identified by pid plus the + * command line's entry path, so a recycled pid belonging to something else + * is never touched. + */ + const reapStaleHub = Effect.gen(function* () { + const previous = yield* fs + .readFileString(hubStatePath()) + .pipe(Effect.flatMap(decodeHubStateFile), Effect.option); + if (previous._tag === "None") return; + const alive = yield* isProcessAlive(previous.value.pid); + if (alive) { + const commandLine = yield* runner + .run({ + command: "ps", + args: ["-o", "command=", "-p", String(previous.value.pid)], + timeout: Duration.seconds(5), + timeoutBehavior: "timedOutResult", + }) + .pipe( + Effect.map((result) => result.stdout), + Effect.orElseSucceed(() => ""), + ); + if (commandLine.includes(previous.value.entryPath)) { + yield* Effect.logWarning("Stopping a device hub left behind by a previous server", { + pid: previous.value.pid, + port: previous.value.port, + }); + yield* Effect.sync(() => { + try { + process.kill(previous.value.pid, "SIGTERM"); + } catch { + // Already gone. + } + }); + } + } + yield* fs.remove(hubStatePath(), { force: true }).pipe(Effect.ignore); + }).pipe(Effect.catchCause(() => Effect.void)); + + const recordHub = (hub: HubProcess, hubTool: DeviceToolPaths) => + encodeHubStateFile({ + pid: Number(hub.child.pid), + port: Number(new URL(hub.origin).port), + entryPath: hubTool.entryPath, + }).pipe( + Effect.flatMap((json) => fs.writeFileString(hubStatePath(), json)), + Effect.ignore, + ); + + const spawnHub = Effect.fn("LocalDeviceHost.spawnHub")(function* ( + hubTool: DeviceToolPaths, + ): Effect.fn.Return { + yield* reapStaleHub; + yield* fs + .makeDirectory(agentDeviceStateDir(path, config.stateDir), { recursive: true }) + .pipe(Effect.ignore); + const port = yield* net.reserveLoopbackPort("127.0.0.1").pipe( + Effect.mapError( + (cause) => + new DeviceHost.DeviceHostError({ + hostId, + step: "reserving a port for the device hub", + cause, + }), + ), + ); + const origin = `http://127.0.0.1:${port}`; + const scope = yield* Scope.make("sequential"); + const child = yield* spawner + .spawn( + ChildProcess.make( + process.execPath, + [ + hubTool.entryPath, + "--port", + String(port), + "--host", + "127.0.0.1", + "--hide-sidebar", + "--hide-boot-device", + ], + { + detached: false, + shell: false, + stdout: "pipe", + stderr: "pipe", + env: hubEnvironment(), + }, + ), + ) + .pipe( + Effect.provideService(Scope.Scope, scope), + Effect.mapError( + (cause) => + new DeviceHost.DeviceHostError({ + hostId, + step: "starting the device hub", + cause, + }), + ), + ); + const startedAtMillis = yield* Clock.currentTimeMillis; + const hub: HubProcess = { child, scope, origin, startedAtMillis }; + yield* Effect.forkIn(observeHubOutput(hub), scope); + yield* waitForHttpReady({ + baseUrl: origin, + path: "/readyz", + timeoutMs: HUB_READY_TIMEOUT_MS, + makeError: (info) => + new DeviceHost.DeviceHostError({ + hostId, + step: "waiting for the device hub to answer", + cause: info.cause, + }), + }).pipe( + Effect.provideService(HttpClient.HttpClient, httpClient), + Effect.tapError(() => stopHub(hub)), + ); + yield* recordHub(hub, hubTool); + yield* Effect.logInfo("Device hub started", { pid: Number(child.pid), port }); + return hub; + }); + + const observeHubOutput = (hub: HubProcess) => + hub.child.all.pipe( + Stream.decodeText(), + Stream.splitLines, + Stream.map((line) => line.trim()), + Stream.filter((line) => line.length > 0), + Stream.runForEach((line) => + Effect.logDebug("Device hub output", { pid: Number(hub.child.pid), output: line }), + ), + Effect.catchCause(() => Effect.void), + ); + + /** + * Restart the hub when it dies under us, with the same doubling backoff the + * relay connector uses so a hub that crashes on boot cannot spin. + */ + const superviseHub = (hub: HubProcess, hubTool: DeviceToolPaths): Effect.Effect => + Effect.gen(function* () { + yield* Effect.result(hub.child.exitCode); + const running = yield* Ref.get(runningRef); + if (running?.hub.child.pid !== hub.child.pid) return; + const uptime = (yield* Clock.currentTimeMillis) - hub.startedAtMillis; + const delay = yield* Ref.modify(restartDelayRef, (current) => { + if (uptime >= HUB_RESTART_STABLE_UPTIME_MS) return [0, 0]; + const next = current === 0 ? 1_000 : Math.min(current * 2, HUB_RESTART_MAX_DELAY_MS); + return [current, next]; + }); + yield* Effect.logWarning("Device hub exited; restarting", { + pid: Number(hub.child.pid), + delayMs: delay, + }); + yield* Effect.sleep(Duration.millis(delay)); + yield* startLock.withPermits(1)( + Effect.gen(function* () { + const current = yield* Ref.get(runningRef); + if (current?.hub.child.pid !== hub.child.pid) return; + const replacement = yield* spawnHub(hubTool); + yield* Ref.set(runningRef, { ...current, hub: replacement }); + yield* Effect.forkDetach(superviseHub(replacement, hubTool)); + }), + ); + }).pipe( + Effect.catchCause((cause) => Effect.logWarning("Device hub supervisor failed", { cause })), + ); + + const daemonFilePath = () => path.join(agentDeviceStateDir(path, config.stateDir), "daemon.json"); + + const readDaemonFile = Effect.fn("LocalDeviceHost.readDaemonFile")(function* () { + const raw = yield* fs.readFileString(daemonFilePath()); + return yield* decodeDaemonFile(raw); + }); + + /** + * agent-device auto-starts its daemon on any command. A trivial `devices` + * call in HTTP mode is the documented way to bring it up; its output is the + * daemon.json this reads back. + */ + const startAgentDeviceDaemon = Effect.fn("LocalDeviceHost.startAgentDeviceDaemon")(function* ( + agentTool: DeviceToolPaths, + ): Effect.fn.Return { + const stateDir = agentDeviceStateDir(path, config.stateDir); + yield* fs.makeDirectory(stateDir, { recursive: true }).pipe(Effect.ignore); + const existing = yield* readDaemonFile().pipe(Effect.option); + const daemonEnvironment: NodeJS.ProcessEnv = { + ...hostEnvironment, + AGENT_DEVICE_STATE_DIR: stateDir, + AGENT_DEVICE_DAEMON_SERVER_MODE: "http", + // The daemon idles out after five minutes by default; the server owns + // its lifetime here and stops it explicitly. + AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: "0", + AGENT_DEVICE_NO_UPDATE_NOTIFIER: "1", + FORCE_COLOR: "0", + NO_COLOR: "1", + }; + const toEndpoint = ( + file: typeof AgentDeviceDaemonFile.Type, + ): DeviceHost.AgentDeviceEndpoint => ({ + baseUrl: `http://127.0.0.1:${file.httpPort}`, + token: file.token, + entryPath: agentTool.entryPath, + }); + if (existing._tag === "Some") { + const alive = yield* HttpClient.withScope(httpClient) + .get(`http://127.0.0.1:${existing.value.httpPort}/health`) + .pipe( + Effect.timeout(Duration.seconds(2)), + Effect.flatMap((response) => + response.arrayBuffer.pipe(Effect.as(response.status === 200)), + ), + Effect.scoped, + Effect.orElseSucceed(() => false), + ); + if (alive) return toEndpoint(existing.value); + yield* fs.remove(daemonFilePath(), { force: true }).pipe(Effect.ignore); + } + // There is no `daemon start`; the first command in a state dir spawns the + // daemon and blocks until it answers. `devices` is the cheapest one. + yield* runner + .run({ + command: process.execPath, + args: [agentTool.entryPath, "devices", "--json"], + env: daemonEnvironment, + timeout: Duration.millis(DAEMON_READY_TIMEOUT_MS), + timeoutBehavior: "timedOutResult", + }) + .pipe(Effect.ignore); + const deadline = (yield* Clock.currentTimeMillis) + DAEMON_READY_TIMEOUT_MS; + while (true) { + const file = yield* readDaemonFile().pipe(Effect.option); + if (file._tag === "Some") return toEndpoint(file.value); + if ((yield* Clock.currentTimeMillis) > deadline) { + return yield* new DeviceHost.DeviceHostTimeoutError({ + hostId, + timeoutMs: DAEMON_READY_TIMEOUT_MS, + }); + } + yield* Effect.sleep(Duration.millis(DAEMON_POLL_MS)); + } + }); + + const stopAgentDeviceDaemon = (agentTool: DeviceToolPaths | null) => + agentTool + ? runner + .run({ + command: process.execPath, + args: [ + agentTool.entryPath, + "daemon", + "stop", + "--state-dir", + agentDeviceStateDir(path, config.stateDir), + ], + env: { ...hostEnvironment, AGENT_DEVICE_NO_UPDATE_NOTIFIER: "1" }, + timeout: Duration.seconds(10), + timeoutBehavior: "timedOutResult", + }) + .pipe(Effect.ignore) + : Effect.void; + + let agentToolRef: DeviceToolPaths | null = null; + + const ensureHubReady = Effect.fn("LocalDeviceHost.ensureHubReady")(function* ( + onPhase: (phase: "installing" | "starting") => Effect.Effect, + ): Effect.fn.Return { + const running = yield* Ref.get(runningRef); + if (running) { + const alive = yield* running.hub.child.isRunning.pipe(Effect.orElseSucceed(() => false)); + if (alive) return running; + yield* Ref.set(runningRef, null); + } + const installed = yield* isDeviceHubInstalled(config.baseDir).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Path.Path, path), + ); + if (!installed) yield* onPhase("installing"); + const hubTool = yield* ensureDeviceHub(config.baseDir).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Path.Path, path), + Effect.provideService(ProcessRunner.ProcessRunner, runner), + Effect.mapError( + (cause) => + new DeviceHost.DeviceHostError({ + hostId, + step: "installing device support", + cause, + }), + ), + ); + yield* onPhase("starting"); + const hub = yield* spawnHub(hubTool); + const candidate = helperPaths(hubTool); + const [axExists, cliExists] = yield* Effect.all([ + fs.exists(candidate.serveSimAxSettings).pipe(Effect.orElseSucceed(() => false)), + fs.exists(candidate.serveSimCli).pipe(Effect.orElseSucceed(() => false)), + ]); + const next: RunningHost = { + hub, + agentDevice: null, + helpers: { + serveSimAxSettings: axExists ? candidate.serveSimAxSettings : null, + serveSimCli: cliExists ? candidate.serveSimCli : null, + }, + }; + yield* Ref.set(runningRef, next); + yield* Ref.set(restartDelayRef, 0); + yield* Effect.forkDetach(superviseHub(hub, hubTool)); + return next; + }); + + const ensureReady: DeviceHost.DeviceHost["Service"]["ensureReady"] = (onPhase) => + startLock.withPermits(1)(ensureHubReady(onPhase).pipe(Effect.map(toReady))); + + const ensureAgentReady: DeviceHost.DeviceHost["Service"]["ensureAgentReady"] = (onPhase) => + startLock.withPermits(1)( + Effect.gen(function* (): Generator< + Effect.Effect, + DeviceHost.DeviceHostAgentReady + > { + const running = yield* ensureHubReady(onPhase); + if (running.agentDevice) return { ...toReady(running), agentDevice: running.agentDevice }; + const installed = yield* isAgentDeviceInstalled(config.baseDir).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Path.Path, path), + ); + if (!installed) yield* onPhase("installing"); + const agentTool = yield* ensureAgentDevice(config.baseDir).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Path.Path, path), + Effect.provideService(ProcessRunner.ProcessRunner, runner), + Effect.mapError( + (cause) => + new DeviceHost.DeviceHostError({ + hostId, + step: "installing agent tools", + cause, + }), + ), + ); + agentToolRef = agentTool; + yield* onPhase("starting"); + const agentDevice = yield* startAgentDeviceDaemon(agentTool); + const next = { ...running, agentDevice }; + yield* Ref.set(runningRef, next); + return { ...toReady(next), agentDevice }; + }), + ); + + const helperPaths = (hubTool: DeviceToolPaths) => { + const serveSimDist = path.join( + hubTool.installDir, + "node_modules", + "expo-device-hub", + "vendor", + "serve-sim", + "dist", + ); + return { + serveSimAxSettings: path.join(serveSimDist, "simax", "serve-sim-ax-settings"), + serveSimCli: path.join(serveSimDist, "serve-sim.js"), + }; + }; + + const run: DeviceHost.DeviceHostReady["run"] = (command, args, options) => + runner + .run({ + command: + command === "emulator" && sdk.root + ? path.join( + sdk.root, + "emulator", + hostPlatform === "win32" ? "emulator.exe" : "emulator", + ) + : command, + args, + env: hostEnvironment, + timeout: Duration.millis(options?.timeoutMs ?? 20_000), + timeoutBehavior: "timedOutResult", + ...(options?.stdin === undefined ? {} : { stdin: options.stdin }), + }) + .pipe( + Effect.map((result) => ({ + stdout: result.stdout, + stderr: result.stderr, + code: Number(result.code), + })), + Effect.catch((cause) => Effect.succeed({ stdout: "", stderr: String(cause), code: 127 })), + ); + + const toReady = (running: RunningHost): DeviceHost.DeviceHostReady => ({ + hub: { origin: running.hub.origin } satisfies DeviceHost.DeviceHubEndpoint, + run, + helpers: running.helpers, + }); + + const current: DeviceHost.DeviceHost["Service"]["current"] = Ref.get(runningRef).pipe( + Effect.map((running) => (running ? toReady(running) : null)), + ); + + const stopAgent: DeviceHost.DeviceHost["Service"]["stopAgent"] = startLock.withPermits(1)( + Effect.gen(function* () { + yield* stopAgentDeviceDaemon(agentToolRef); + yield* Ref.update(runningRef, (running) => + running ? { ...running, agentDevice: null } : running, + ); + }), + ); + + const stop: DeviceHost.DeviceHost["Service"]["stop"] = startLock.withPermits(1)( + Effect.gen(function* () { + const running = yield* Ref.getAndSet(runningRef, null); + yield* stopHub(running?.hub); + yield* fs.remove(hubStatePath(), { force: true }).pipe(Effect.ignore); + yield* stopAgentDeviceDaemon(agentToolRef); + }), + ); + + // Never leave the hub or daemon behind when the server's scope closes. + yield* Effect.addFinalizer(() => stop); + + const host: DeviceHost.DeviceHost["Service"] = { + id: hostId, + summary, + platformAvailability, + ensureReady, + ensureAgentReady, + current, + stopAgent, + stop, + }; + return host; +}); + +export const layer = Layer.effect(DeviceHost.DeviceHost, make()); + +/** Exposed for tests. */ +export const __testing = { + AgentDeviceDaemonFile, + androidSdk, + platformReason, + deviceHostEnvironment, +}; diff --git a/apps/server/src/mcp/McpDeviceToolkit.test.ts b/apps/server/src/mcp/McpDeviceToolkit.test.ts new file mode 100644 index 000000000000..083ac0c034fe --- /dev/null +++ b/apps/server/src/mcp/McpDeviceToolkit.test.ts @@ -0,0 +1,127 @@ +import { expect, it } from "@effect/vitest"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { EnvironmentId, ProviderInstanceId, ThreadId } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import { McpSchema, McpServer } from "effect/unstable/ai"; + +import * as DeviceService from "../device/DeviceService.ts"; +import * as McpHttpServer from "./McpHttpServer.ts"; +import * as McpInvocationContext from "./McpInvocationContext.ts"; + +const environmentId = EnvironmentId.make("environment-device-test"); +const threadId = ThreadId.make("thread-device-test"); +const invocation = (capabilities: ReadonlyArray) => ({ + environmentId, + threadId, + providerSessionId: "provider-session-device-test", + providerInstanceId: ProviderInstanceId.make("codex"), + capabilities: new Set(capabilities), + issuedAt: 1, +}); +const client = McpSchema.McpServerClient.of({ + clientId: 1, + clientCapabilities: {}, + clientInfo: { name: "mcp-test", version: "1.0.0" }, + protocolVersion: "2025-06-18", + initializePayload: { + protocolVersion: "2025-06-18", + capabilities: {}, + clientInfo: { name: "mcp-test", version: "1.0.0" }, + }, + getClient: Effect.die("unused"), +}); + +const device = { + hostId: "local", + id: "UDID-1", + platform: "ios" as const, + name: "iPhone 17 Pro", + version: "iOS 27.0", + booted: true, + physical: false, +}; +const state = { + hosts: [ + { + id: "local", + kind: "local" as const, + label: "This machine", + platforms: [ + { platform: "ios" as const, available: true }, + { platform: "android" as const, available: false, reason: "No SDK" }, + ], + hubInstalled: true, + agentDeviceInstalled: true, + }, + ], + hostStatus: "ready" as const, + devices: [device], + sessions: [], + onboardingCompleted: true, + agentAccessEnabled: true, + hubBasePath: "/api/device-hub", + revision: 1, +}; +const png = new Uint8Array(24); +new DataView(png.buffer).setUint32(0, 0x89504e47); +new DataView(png.buffer).setUint32(4, 0x0d0a1a0a); +new DataView(png.buffer).setUint32(12, 0x49484452); +new DataView(png.buffer).setUint32(16, 1206); +new DataView(png.buffer).setUint32(20, 2622); + +const DeviceServiceMock = Layer.mock(DeviceService.DeviceService)({ + state: Effect.succeed(state), + list: Effect.succeed(state), + open: (input) => + Effect.succeed({ + threadId: input.threadId, + hostId: "local", + deviceId: input.deviceId, + platform: input.platform, + openedAt: "2026-09-08T00:00:00.000Z", + }), + sessionsForThread: () => Effect.succeed([]), + screenshot: () => Effect.succeed({ device, png }), + close: () => Effect.void, +}); + +const TestLayer = McpHttpServer.DeviceToolkitRegistrationLive.pipe( + Layer.provideMerge(McpServer.McpServer.layer), + Layer.provideMerge(DeviceServiceMock), + Layer.provide(NodeServices.layer), +); + +it.effect("registers the device tools and returns the screenshot as image content", () => + Effect.scoped( + Effect.gen(function* () { + const server = yield* McpServer.McpServer; + const names = server.tools.map(({ tool }) => tool.name).toSorted(); + expect(names).toEqual(["device_close", "device_list", "device_open", "device_screenshot"]); + + const callWith = (capabilities: ReadonlyArray) => + Effect.provideService(McpInvocationContext.McpInvocationContext, invocation(capabilities)); + + const opened = yield* server + .callTool({ name: "device_open", arguments: { platform: "ios" } }) + .pipe(callWith(["device"]), Effect.provideService(McpSchema.McpServerClient, client)); + expect(opened.isError).toBe(false); + const openedContent = opened.structuredContent as { quickStart: string }; + expect(openedContent.quickStart).toContain("--udid UDID-1"); + + const shot = yield* server + .callTool({ name: "device_screenshot", arguments: { deviceId: "UDID-1" } }) + .pipe(callWith(["device"]), Effect.provideService(McpSchema.McpServerClient, client)); + expect(shot.isError).toBe(false); + expect(shot.content.map((entry) => entry.type)).toEqual(["text", "image"]); + expect(shot.structuredContent).toMatchObject({ + screenshot: { mimeType: "image/png", width: 1206, height: 2622 }, + }); + + const denied = yield* server + .callTool({ name: "device_list", arguments: {} }) + .pipe(callWith(["preview"]), Effect.provideService(McpSchema.McpServerClient, client)); + expect(denied.isError).toBe(true); + }), + ).pipe(Effect.provide(TestLayer)), +); diff --git a/apps/server/src/mcp/McpHttpServer.ts b/apps/server/src/mcp/McpHttpServer.ts index 3f3e48ebe4b2..747f785bbd7d 100644 --- a/apps/server/src/mcp/McpHttpServer.ts +++ b/apps/server/src/mcp/McpHttpServer.ts @@ -16,6 +16,7 @@ import { HttpRouter, HttpServerRequest, HttpServerResponse } from "effect/unstab import packageJson from "../../package.json" with { type: "json" }; import * as ServerConfig from "../config.ts"; +import * as DeviceService from "../device/DeviceService.ts"; import * as McpInvocationContext from "./McpInvocationContext.ts"; import * as McpSessionRegistry from "./McpSessionRegistry.ts"; import * as PreviewAutomationBroker from "./PreviewAutomationBroker.ts"; @@ -28,6 +29,15 @@ import { PreviewSnapshotToolkit, PreviewStandardToolkit, } from "./toolkits/preview/tools.ts"; +import { + DeviceScreenshotToolkitHandlersLive, + DeviceStandardToolkitHandlersLive, +} from "./toolkits/device/handlers.ts"; +import { + DeviceScreenshotTool, + DeviceScreenshotToolkit, + DeviceStandardToolkit, +} from "./toolkits/device/tools.ts"; const unauthorized = HttpServerResponse.jsonUnsafe( { @@ -424,6 +434,154 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot }); }); +interface ImageToolResult { + readonly screenshot: { + readonly mimeType: "image/png"; + readonly data: string; + readonly width: number; + readonly height: number; + }; + readonly [key: string]: unknown; +} + +/** + * Failures surface only their tag: the remote message may carry renderer or + * device output the agent should not see, and the tag is what it can act on. + */ +const imageToolFailure = + (toolName: string, operation: string, failureText: string) => + (cause: Cause.Cause) => { + if (Cause.hasInterrupts(cause) || cause.reasons.some(Cause.isDieReason)) { + return Effect.failCause(cause).pipe(Effect.orDie); + } + const failures = cause.reasons.filter(Cause.isFailReason); + const firstFailure = failures[0]?.error; + const errorTag = + typeof firstFailure === "object" && + firstFailure !== null && + "_tag" in firstFailure && + typeof firstFailure._tag === "string" + ? firstFailure._tag + : `${toolName}Error`; + const result = new McpSchema.CallToolResult({ + isError: true, + structuredContent: { + error: { + _tag: errorTag, + operation, + failureCount: failures.length, + }, + }, + content: [{ type: "text", text: failureText }], + }); + return Effect.logWarning(`${toolName} failed`, { + operation, + errorTag, + failureCount: failures.length, + }).pipe(Effect.as(result)); + }; + +/** + * `McpServer.toolkit` serializes every result as JSON text, which is the + * wrong shape for a screenshot: the model needs image content. Tools whose + * result carries a `screenshot` field are registered by hand so the PNG goes + * out as an image block and the rest of the payload as JSON metadata. + */ +const registerImageTool = ( + tool: T, + handle: (payload: Tool.Parameters) => Effect.Effect<{ readonly encodedResult: unknown }, E, R>, + provide: ( + effect: Effect.Effect<{ readonly encodedResult: unknown }, E, R>, + ) => Effect.Effect< + { readonly encodedResult: unknown }, + E, + McpInvocationContext.McpInvocationContext + >, + operation: string, + failureText: string, +) => + Effect.gen(function* () { + const server = yield* McpServer.McpServer; + yield* server.addTool({ + tool: new McpSchema.Tool({ + name: tool.name, + description: Tool.getDescription(tool), + inputSchema: Tool.getJsonSchema(tool), + annotations: { + ...Context.getOption(tool.annotations, Tool.Title).pipe( + Option.map((title) => ({ title })), + Option.getOrUndefined, + ), + readOnlyHint: Context.get(tool.annotations, Tool.Readonly), + destructiveHint: Context.get(tool.annotations, Tool.Destructive), + idempotentHint: Context.get(tool.annotations, Tool.Idempotent), + openWorldHint: Context.get(tool.annotations, Tool.OpenWorld), + }, + }), + annotations: tool.annotations, + handle: (payload) => + Effect.withFiber((fiber) => { + const invocation = Context.getUnsafe( + fiber.context, + McpInvocationContext.McpInvocationContext, + ); + return provide(handle(payload as Tool.Parameters)).pipe( + Effect.provideService(McpInvocationContext.McpInvocationContext, invocation), + Effect.matchCauseEffect({ + onFailure: imageToolFailure(tool.name, operation, failureText), + onSuccess: ({ encodedResult }) => { + const { screenshot, ...rest } = encodedResult as ImageToolResult; + const includeImage = + (payload as { readonly includeImage?: boolean } | undefined)?.includeImage !== + false; + const metadata = { + ...rest, + screenshot: { + mimeType: screenshot.mimeType, + width: screenshot.width, + height: screenshot.height, + }, + }; + return Effect.succeed( + new McpSchema.CallToolResult({ + isError: false, + structuredContent: metadata, + content: [ + { type: "text", text: JSON.stringify(metadata) }, + ...(includeImage + ? [ + { + type: "image" as const, + data: new Uint8Array(Buffer.from(screenshot.data, "base64")), + mimeType: screenshot.mimeType, + }, + ] + : []), + ], + }), + ); + }, + }), + ); + }), + }); + }); + +const registerDeviceScreenshot = Effect.fn("McpHttpServer.registerDeviceScreenshot")(function* () { + const devices = yield* DeviceService.DeviceService; + const built = yield* DeviceScreenshotToolkit; + yield* registerImageTool( + DeviceScreenshotTool, + (payload) => + built + .handle("device_screenshot", payload) + .pipe(Stream.unwrap, Stream.run(Sink.last()), Effect.flatMap(Effect.fromOption)), + (effect) => effect.pipe(Effect.provideService(DeviceService.DeviceService, devices)), + "screenshot", + "Device screenshot failed.", + ); +}); + const PreviewStandardToolkitRegistrationLive = McpServer.toolkit(PreviewStandardToolkit).pipe( Layer.provide(PreviewStandardToolkitHandlersLive), ); @@ -437,6 +595,19 @@ export const PreviewToolkitRegistrationLive = Layer.mergeAll( PreviewSnapshotRegistrationLive, ); +const DeviceStandardToolkitRegistrationLive = McpServer.toolkit(DeviceStandardToolkit).pipe( + Layer.provide(DeviceStandardToolkitHandlersLive), +); + +const DeviceScreenshotRegistrationLive = Layer.effectDiscard(registerDeviceScreenshot()).pipe( + Layer.provide(DeviceScreenshotToolkitHandlersLive), +); + +export const DeviceToolkitRegistrationLive = Layer.mergeAll( + DeviceStandardToolkitRegistrationLive, + DeviceScreenshotRegistrationLive, +); + const McpTransportLive = McpServer.layerHttp({ name: "T3 Code", version: packageJson.version, @@ -444,4 +615,7 @@ const McpTransportLive = McpServer.layerHttp({ protocols: [McpProtocol.v2025_06_18], }).pipe(Layer.provide(McpAuthMiddlewareLive)); -export const layer = PreviewToolkitRegistrationLive.pipe(Layer.provideMerge(McpTransportLive)); +export const layer = Layer.mergeAll( + PreviewToolkitRegistrationLive, + DeviceToolkitRegistrationLive, +).pipe(Layer.provideMerge(McpTransportLive)); diff --git a/apps/server/src/mcp/McpInvocationContext.ts b/apps/server/src/mcp/McpInvocationContext.ts index 49273485a44d..f508c3d0cccd 100644 --- a/apps/server/src/mcp/McpInvocationContext.ts +++ b/apps/server/src/mcp/McpInvocationContext.ts @@ -7,7 +7,7 @@ import { import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; -export type McpCapability = "preview"; +export type McpCapability = "preview" | "device"; export interface McpInvocationScope { readonly environmentId: EnvironmentId; @@ -28,8 +28,11 @@ export const requireMcpCapability = Effect.fn("mcp.requireCapability")(function* ) { const invocation = yield* McpInvocationContext; if (!invocation.capabilities.has(capability)) { + // The error shape predates the second capability; "preview" is the + // wire-compatible tag, and device handlers remap it before it reaches + // the agent. return yield* new PreviewAutomationUnavailableError({ - capability, + capability: "preview", environmentId: invocation.environmentId, threadId: invocation.threadId, providerSessionId: invocation.providerSessionId, diff --git a/apps/server/src/mcp/McpProviderSession.test.ts b/apps/server/src/mcp/McpProviderSession.test.ts new file mode 100644 index 000000000000..1c3c3af5091e --- /dev/null +++ b/apps/server/src/mcp/McpProviderSession.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, it } from "vite-plus/test"; +import { withAgentDeviceEnvironment } from "./McpProviderSession.ts"; + +describe("device CLI environment", () => { + it("preserves provider credentials and commands while routing devices to the owned daemon", () => { + const environment = withAgentDeviceEnvironment( + { PATH: "/provider/bin:/usr/bin", PROVIDER_KEY: "fixture" }, + { + agentDeviceEnvironment: { + PATH: "/t3/device/bin", + PATH_SEPARATOR: ":", + AGENT_DEVICE_DAEMON_BASE_URL: "http://127.0.0.1:9000", + AGENT_DEVICE_DAEMON_AUTH_TOKEN: "fixture-device", + }, + }, + ); + expect(environment).toEqual({ + PATH: "/t3/device/bin:/provider/bin:/usr/bin", + PROVIDER_KEY: "fixture", + AGENT_DEVICE_DAEMON_BASE_URL: "http://127.0.0.1:9000", + AGENT_DEVICE_DAEMON_AUTH_TOKEN: "fixture-device", + }); + }); + + it("does not grant CLI access when device access was not supplied", () => { + const environment = { PATH: "/usr/bin", PROVIDER_KEY: "fixture" }; + expect(withAgentDeviceEnvironment(environment, undefined)).toBe(environment); + expect(withAgentDeviceEnvironment(environment, {})).toBe(environment); + }); +}); diff --git a/apps/server/src/mcp/McpProviderSession.ts b/apps/server/src/mcp/McpProviderSession.ts index d5dc582046c1..4019cb37bd04 100644 --- a/apps/server/src/mcp/McpProviderSession.ts +++ b/apps/server/src/mcp/McpProviderSession.ts @@ -7,6 +7,31 @@ export interface McpProviderSessionConfig { readonly providerInstanceId: ProviderInstanceId; readonly endpoint: string; readonly authorizationHeader: string; + /** Capabilities the credential grants ("preview", "device"). */ + readonly capabilities: ReadonlySet; + /** + * Set when the session may drive devices. Adapters spread this into the + * provider subprocess environment so the `agent-device` CLI is on PATH and + * already pointed at the server's daemon; the agent never handles a token. + */ + readonly agentDeviceEnvironment?: Readonly>; +} + +/** Provider env with the device variables applied over `base`, or `base` untouched. */ +export function withAgentDeviceEnvironment( + base: NodeJS.ProcessEnv, + config: Pick | undefined, +): NodeJS.ProcessEnv { + const extra = config?.agentDeviceEnvironment; + if (!extra) return base; + const separator = extra.PATH_SEPARATOR ?? ":"; + const basePath = base.PATH ?? base.Path; + const { PATH: shimDir, PATH_SEPARATOR: _separator, ...rest } = extra; + return { + ...base, + ...rest, + ...(shimDir ? { PATH: basePath ? `${shimDir}${separator}${basePath}` : shimDir } : {}), + }; } const sessionsByThread = new Map(); diff --git a/apps/server/src/mcp/McpSessionRegistry.test.ts b/apps/server/src/mcp/McpSessionRegistry.test.ts index 1d8aead99d0d..1a4062f9e660 100644 --- a/apps/server/src/mcp/McpSessionRegistry.test.ts +++ b/apps/server/src/mcp/McpSessionRegistry.test.ts @@ -38,6 +38,7 @@ it.effect("stores only a token hash, resolves the bearer token, and revokes by t const threadId = ThreadId.make("thread-1"); const issued = yield* registry.issue({ threadId, + capabilities: new Set(["preview"]), providerInstanceId: ProviderInstanceId.make("codex"), }); expect(issued.config.endpoint).toBe("http://127.0.0.1:43123/mcp"); @@ -67,6 +68,7 @@ it.effect("builds MCP endpoints from the bound server host", () => const registry = yield* makeRegistry(() => 1_000, makeFakeHttpServer(hostname)); const issued = yield* registry.issue({ threadId: ThreadId.make(`thread-${hostname}`), + capabilities: new Set(["preview"]), providerInstanceId: ProviderInstanceId.make("codex"), }); expect(issued.config.endpoint).toBe(expectedEndpoint); @@ -80,6 +82,7 @@ it.effect("expires credentials once their session stops showing signs of life", const registry = yield* makeRegistry(() => timestamp); const issued = yield* registry.issue({ threadId: ThreadId.make("thread-2"), + capabilities: new Set(["preview"]), providerInstanceId: ProviderInstanceId.make("claude"), }); const token = issued.config.authorizationHeader.replace(/^Bearer\s+/, ""); @@ -95,6 +98,7 @@ it.effect("keeps a credential alive across turns that never touch an MCP tool", const threadId = ThreadId.make("thread-3"); const issued = yield* registry.issue({ threadId, + capabilities: new Set(["preview"]), providerInstanceId: ProviderInstanceId.make("claude"), }); const token = issued.config.authorizationHeader.replace(/^Bearer\s+/, ""); @@ -116,6 +120,7 @@ it.effect("does not keep credentials of other threads alive", () => const registry = yield* makeRegistry(() => timestamp); const issued = yield* registry.issue({ threadId: ThreadId.make("thread-4"), + capabilities: new Set(["preview"]), providerInstanceId: ProviderInstanceId.make("codex"), }); const token = issued.config.authorizationHeader.replace(/^Bearer\s+/, ""); diff --git a/apps/server/src/mcp/McpSessionRegistry.ts b/apps/server/src/mcp/McpSessionRegistry.ts index f19a4f4e8c49..abeb74043db7 100644 --- a/apps/server/src/mcp/McpSessionRegistry.ts +++ b/apps/server/src/mcp/McpSessionRegistry.ts @@ -14,6 +14,7 @@ import * as McpProviderSession from "./McpProviderSession.ts"; export interface McpCredentialRequest { readonly threadId: ThreadId; readonly providerInstanceId: ProviderInstanceId; + readonly capabilities: ReadonlySet; } export interface McpIssuedCredential { @@ -128,7 +129,7 @@ const makeWithOptions = Effect.fn("McpSessionRegistry.make")(function* ( threadId: ThreadId.make(request.threadId), providerSessionId, providerInstanceId: ProviderInstanceId.make(request.providerInstanceId), - capabilities: new Set(["preview"]), + capabilities: new Set(request.capabilities), issuedAt, }; yield* SynchronizedRef.update(state, ({ records }) => { @@ -144,6 +145,7 @@ const makeWithOptions = Effect.fn("McpSessionRegistry.make")(function* ( providerInstanceId: scope.providerInstanceId, endpoint, authorizationHeader: `Bearer ${rawToken}`, + capabilities: scope.capabilities, }, }; }, diff --git a/apps/server/src/mcp/toolkits/device/handlers.test.ts b/apps/server/src/mcp/toolkits/device/handlers.test.ts new file mode 100644 index 000000000000..e37b2a8b22c2 --- /dev/null +++ b/apps/server/src/mcp/toolkits/device/handlers.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { agentDeviceQuickStart, agentDeviceTargetArgs, pngDimensions } from "./handlers.ts"; + +const device = { + hostId: "local", + id: "ABCD-1234", + platform: "ios" as const, + name: "iPhone 17 Pro", + version: "iOS 27.0", + booted: true, + physical: false, +}; + +describe("device tool helpers", () => { + it("pins agent-device commands to the device by platform-specific flag", () => { + expect(agentDeviceTargetArgs(device)).toEqual(["--platform", "ios", "--udid", "ABCD-1234"]); + expect(agentDeviceTargetArgs({ ...device, platform: "android", id: "emulator-5554" })).toEqual([ + "--platform", + "android", + "--serial", + "emulator-5554", + ]); + }); + + it("writes the quick start around the pinned target", () => { + const text = agentDeviceQuickStart(device); + expect(text).toContain("agent-device snapshot -i --platform ios --udid ABCD-1234"); + expect(text).toContain("iPhone 17 Pro (iOS 27.0)"); + expect(text).toContain("XCTest runner"); + }); + + it("reads PNG dimensions from the IHDR chunk", () => { + const png = new Uint8Array(24); + new DataView(png.buffer).setUint32(0, 0x89504e47); + new DataView(png.buffer).setUint32(4, 0x0d0a1a0a); + new DataView(png.buffer).setUint32(12, 0x49484452); + new DataView(png.buffer).setUint32(16, 1179); + new DataView(png.buffer).setUint32(20, 2556); + expect(pngDimensions(png)).toEqual({ width: 1179, height: 2556 }); + expect(pngDimensions(new Uint8Array([1, 2, 3]))).toEqual({ width: 0, height: 0 }); + }); +}); diff --git a/apps/server/src/mcp/toolkits/device/handlers.ts b/apps/server/src/mcp/toolkits/device/handlers.ts new file mode 100644 index 000000000000..bab7391af24d --- /dev/null +++ b/apps/server/src/mcp/toolkits/device/handlers.ts @@ -0,0 +1,212 @@ +import { + type DeviceError, + type DeviceHostId, + type DeviceId, + type DevicePlatform, + type DeviceSummary, + DeviceToolUnavailableError, + LOCAL_DEVICE_HOST_ID, +} from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; + +import * as DeviceService from "../../../device/DeviceService.ts"; +import * as McpInvocationContext from "../../McpInvocationContext.ts"; +import { DeviceScreenshotToolkit, DeviceStandardToolkit, DeviceToolkit } from "./tools.ts"; + +/** The flags that pin every agent-device command to one device. */ +export function agentDeviceTargetArgs(device: DeviceSummary): ReadonlyArray { + return device.platform === "ios" + ? ["--platform", "ios", "--udid", device.id] + : ["--platform", "android", "--serial", device.id]; +} + +/** + * Just-in-time guidance returned from `device_open`. This is the one place + * the agent learns how to drive the device, so it lives with the tool result + * rather than in the always-on prompt block; threads that never open a device + * never pay for it. + */ +export function agentDeviceQuickStart(device: DeviceSummary): string { + const target = agentDeviceTargetArgs(device).join(" "); + const platformNotes = + device.platform === "ios" + ? "First use builds an XCTest runner and can take a couple of minutes; later commands are fast." + : "The Android snapshot helper installs itself on first use."; + return [ + `The user is watching ${device.name} (${device.version}) in the Device panel.`, + `Drive it with the agent-device CLI, which is on PATH and already connected to this environment. Always pass ${target}.`, + "Typical loop:", + ` agent-device open ${target} # or: open `, + ` agent-device snapshot -i ${target} # accessibility tree with @eN refs`, + ` agent-device click @e3 ${target}`, + ` agent-device fill @e5 "text" ${target}`, + ` agent-device screenshot /tmp/shot.png ${target} # or call device_screenshot`, + ` agent-device install ${target}`, + "Prefer snapshot refs over coordinates. Run `agent-device help` for workflow guides and `agent-device --help` for flags.", + "Do not call simctl, adb, xcrun, or serve-sim directly while these tools are attached; use agent-device.", + platformNotes, + ].join("\n"); +} + +const requireDeviceAccess = McpInvocationContext.requireMcpCapability("device").pipe( + Effect.mapError( + () => + new DeviceToolUnavailableError({ + reason: "Agent device access is turned off for this environment.", + }), + ), +); + +const pickDevice = ( + devices: ReadonlyArray, + input: { + readonly deviceId?: DeviceId | undefined; + readonly platform?: DevicePlatform | undefined; + readonly hostId?: DeviceHostId | undefined; + }, +): Effect.Effect => + Effect.gen(function* () { + const hostId = input.hostId ?? LOCAL_DEVICE_HOST_ID; + if (input.deviceId !== undefined) { + const match = devices.find( + (device) => device.hostId === hostId && device.id === input.deviceId, + ); + if (match) return match; + return yield* new DeviceToolUnavailableError({ + reason: `No device ${input.deviceId} on host ${hostId}. Call device_list for current ids.`, + }); + } + const candidates = devices.filter( + (device) => + device.hostId === hostId && + (input.platform === undefined || device.platform === input.platform), + ); + if (candidates.length === 0) { + return yield* new DeviceToolUnavailableError({ + reason: + input.platform === undefined + ? "No simulators or emulators were found. Call device_list to see why." + : `No ${input.platform} devices were found on host ${hostId}. Call device_list to see why.`, + }); + } + const platforms = new Set(candidates.map((device) => device.platform)); + if (input.platform === undefined && platforms.size > 1) { + return yield* new DeviceToolUnavailableError({ + reason: "Both iOS and Android devices are available; pass platform or deviceId.", + }); + } + return candidates.find((device) => device.booted) ?? candidates[0]!; + }); + +const toolError = (error: DeviceError | DeviceToolUnavailableError) => error; + +const handlers = { + device_list: (input) => + Effect.gen(function* () { + const scope = yield* requireDeviceAccess; + const devices = yield* DeviceService.DeviceService; + const state = yield* devices.list; + if (state.hostStatus === "disabled") { + return yield* new DeviceToolUnavailableError({ + reason: + "Device support is off. Ask the user to enable it in the Device panel before installing or starting device tools.", + }); + } + const hostId = input?.hostId; + const open = state.sessions + .filter((session) => session.threadId === scope.threadId) + .map((session) => ({ hostId: session.hostId, deviceId: session.deviceId })); + return { + hosts: hostId ? state.hosts.filter((host) => host.id === hostId) : state.hosts, + devices: hostId + ? state.devices.filter((device) => device.hostId === hostId) + : state.devices, + open, + }; + }).pipe(Effect.mapError(toolError)), + device_open: (input) => + Effect.gen(function* () { + const scope = yield* requireDeviceAccess; + const devices = yield* DeviceService.DeviceService; + const state = yield* devices.list; + if (state.hostStatus === "disabled") { + return yield* new DeviceToolUnavailableError({ + reason: + "Device support is off. Ask the user to enable it in the Device panel before installing or starting device tools.", + }); + } + const target = yield* pickDevice(state.devices, input); + const session = yield* devices.open({ + threadId: scope.threadId, + hostId: target.hostId, + deviceId: target.id, + platform: target.platform, + }); + const after = yield* devices.state; + const device = + after.devices.find( + (candidate) => candidate.hostId === session.hostId && candidate.id === session.deviceId, + ) ?? target; + return { + device, + agentDevice: { command: "agent-device", targetArgs: agentDeviceTargetArgs(device) }, + quickStart: agentDeviceQuickStart(device), + }; + }).pipe(Effect.mapError(toolError)), + device_screenshot: (input) => + Effect.gen(function* () { + const scope = yield* requireDeviceAccess; + const devices = yield* DeviceService.DeviceService; + const sessions = yield* devices.sessionsForThread(scope.threadId); + const target = + input.deviceId !== undefined + ? { hostId: input.hostId ?? LOCAL_DEVICE_HOST_ID, deviceId: input.deviceId } + : sessions.at(-1); + if (!target) { + return yield* new DeviceToolUnavailableError({ + reason: "No device is open in this thread. Call device_open first.", + }); + } + const shot = yield* devices.screenshot(target); + return { + device: shot.device, + screenshot: { + mimeType: "image/png" as const, + data: Buffer.from(shot.png).toString("base64"), + ...pngDimensions(shot.png), + }, + }; + }).pipe(Effect.mapError(toolError)), + device_close: (input) => + Effect.gen(function* () { + const scope = yield* requireDeviceAccess; + const devices = yield* DeviceService.DeviceService; + yield* devices.close({ + threadId: scope.threadId, + ...(input.deviceId === undefined ? {} : { deviceId: input.deviceId }), + ...(input.shutdown === undefined ? {} : { shutdown: input.shutdown }), + }); + return {}; + }).pipe(Effect.mapError(toolError)), +} satisfies Parameters[0]; + +/** Width and height from the IHDR chunk; a PNG that lacks one reports 0×0. */ +export function pngDimensions(png: Uint8Array): { width: number; height: number } { + if (png.length < 24) return { width: 0, height: 0 }; + const view = new DataView(png.buffer, png.byteOffset, png.byteLength); + const isPng = + view.getUint32(0) === 0x89504e47 && + view.getUint32(4) === 0x0d0a1a0a && + view.getUint32(12) === 0x49484452; + return isPng + ? { width: view.getUint32(16), height: view.getUint32(20) } + : { width: 0, height: 0 }; +} + +const { device_screenshot, ...standardHandlers } = handlers; + +export const DeviceStandardToolkitHandlersLive = DeviceStandardToolkit.toLayer(standardHandlers); + +export const DeviceScreenshotToolkitHandlersLive = DeviceScreenshotToolkit.toLayer({ + device_screenshot, +}); diff --git a/apps/server/src/mcp/toolkits/device/tools.ts b/apps/server/src/mcp/toolkits/device/tools.ts new file mode 100644 index 000000000000..e98d6ab64829 --- /dev/null +++ b/apps/server/src/mcp/toolkits/device/tools.ts @@ -0,0 +1,98 @@ +import { + DeviceToolCloseInput, + DeviceToolError, + DeviceToolListResult, + DeviceToolOpenInput, + DeviceToolOpenResult, + DeviceToolScreenshotResult, + DeviceToolTargetInput, +} from "@t3tools/contracts"; +import * as Schema from "effect/Schema"; +import { Tool, Toolkit } from "effect/unstable/ai"; + +import * as McpInvocationContext from "../../McpInvocationContext.ts"; +import * as DeviceService from "../../../device/DeviceService.ts"; + +const dependencies = [McpInvocationContext.McpInvocationContext, DeviceService.DeviceService]; + +/** + * Deliberately a small surface: lifecycle, visibility for the user, and one + * image-returning verb. Driving the device (taps, typing, install, logs) + * happens through the preconfigured `agent-device` CLI, which has the + * semantic snapshot model agents need and stays current with its own + * releases. Wrapping its commands here would only lag behind it. + */ +const DeviceListTool = Tool.make("device_list", { + description: + "List iOS Simulators and Android Emulators on this environment's device hosts, which platforms each host can run, and which devices are already open in this thread's Device panel. Call this before device_open when you do not know a device id.", + // An empty struct serializes as `anyOf [object, array]`, which some + // providers reject and then drop every tool on the server with it. + parameters: Schema.Struct({ + hostId: Schema.optional( + Schema.String.annotate({ description: "Limit to one device host. Defaults to all hosts." }), + ), + }), + success: DeviceToolListResult, + failure: DeviceToolError, + dependencies, +}) + .annotate(Tool.Title, "List devices") + .annotate(Tool.Readonly, true) + .annotate(Tool.Destructive, false) + .annotate(Tool.Idempotent, true) + .annotate(Tool.OpenWorld, false); + +const DeviceOpenTool = Tool.make("device_open", { + description: + "Open a simulator or emulator for this thread: boots it if needed, starts its live stream, and shows it in the user's Device panel so they can watch. Returns the agent-device CLI invocation pinned to the device; drive the device with that CLI afterwards.", + parameters: DeviceToolOpenInput, + success: DeviceToolOpenResult, + failure: DeviceToolError, + dependencies, +}) + .annotate(Tool.Title, "Open device") + .annotate(Tool.Readonly, false) + .annotate(Tool.Destructive, false) + .annotate(Tool.Idempotent, true) + .annotate(Tool.OpenWorld, true); + +export const DeviceScreenshotTool = Tool.make("device_screenshot", { + description: + "Capture the current screen of an open device as a PNG image. Use it to see what the user sees; for taps and text use the agent-device CLI.", + parameters: DeviceToolTargetInput, + success: DeviceToolScreenshotResult, + failure: DeviceToolError, + dependencies, +}) + .annotate(Tool.Title, "Screenshot device") + .annotate(Tool.Readonly, true) + .annotate(Tool.Destructive, false) + .annotate(Tool.Idempotent, true) + .annotate(Tool.OpenWorld, true); + +const DeviceCloseTool = Tool.make("device_close", { + description: + "Remove a device from this thread's Device panel. Pass shutdown=true to also power the simulator or emulator off.", + parameters: DeviceToolCloseInput, + success: Schema.Record(Schema.String, Schema.Never).annotate({ + description: "The device was closed.", + }), + failure: DeviceToolError, + dependencies, +}) + .annotate(Tool.Title, "Close device") + .annotate(Tool.Readonly, false) + .annotate(Tool.Destructive, true) + .annotate(Tool.Idempotent, true) + .annotate(Tool.OpenWorld, true); + +export const DeviceStandardToolkit = Toolkit.make(DeviceListTool, DeviceOpenTool, DeviceCloseTool); + +export const DeviceScreenshotToolkit = Toolkit.make(DeviceScreenshotTool); + +export const DeviceToolkit = Toolkit.make( + DeviceListTool, + DeviceOpenTool, + DeviceScreenshotTool, + DeviceCloseTool, +); diff --git a/apps/server/src/provider/CodexDeveloperInstructions.ts b/apps/server/src/provider/CodexDeveloperInstructions.ts index 1c2439a9ad9a..85784d21ca4b 100644 --- a/apps/server/src/provider/CodexDeveloperInstructions.ts +++ b/apps/server/src/provider/CodexDeveloperInstructions.ts @@ -12,18 +12,39 @@ For browser work, first call \`preview_status\`. If no automation-capable previe Do not switch to global browser skills, Chrome, Node REPL browser automation, standalone Playwright, or agent-browser merely because the preview is initially closed or a first call fails. Use an alternative browser system only when the T3 preview tools are absent, the user explicitly requests another browser, or \`preview_open\` returns an explicit unsupported/unavailable error. A failed T3 preview tool call should be inspected and retried with corrected arguments when the error is actionable. `; +const T3_CODE_DEVICE_TOOL_INSTRUCTIONS = ` + +## T3 Code devices + +The \`t3-code\` MCP server also exposes \`device_*\` tools for iOS Simulators and Android Emulators on this environment. For mobile verification, call \`device_list\`, then \`device_open\` so the user can watch the device in their Device panel; its result explains how to drive the device. Driving happens through the \`agent-device\` CLI, which is on PATH and already connected: prefer \`agent-device snapshot -i\` refs over coordinates, and use \`device_screenshot\` when you need to see the screen. Do not call simctl, adb, xcrun, or serve-sim directly while these tools are present. If \`device_list\` reports a platform as unavailable, say so instead of trying another route. +`; + +export interface T3CodeToolAvailability { + readonly browser: boolean; + readonly device: boolean; +} + +const normalizeAvailability = ( + availability: boolean | T3CodeToolAvailability, +): T3CodeToolAvailability => + typeof availability === "boolean" ? { browser: availability, device: false } : availability; + /** - * The browser block is omitted entirely when the preview tools aren't attached. - * Describing `preview_*` tools that aren't in the turn's tool list would be + * Each block is omitted entirely when its tools aren't attached. Describing + * `preview_*` or `device_*` tools that aren't in the turn's tool list would be * worse than saying nothing: the instructions actively steer the model away - * from Playwright and agent-browser, so leaving them in would talk it out of - * the only browser automation it still has. + * from Playwright, agent-browser, and raw simctl/adb, so leaving them in would + * talk it out of the only automation it still has. */ -const browserToolInstructions = (browserToolsAvailable: boolean): string => - browserToolsAvailable ? T3_CODE_BROWSER_TOOL_INSTRUCTIONS : ""; +const browserToolInstructions = (availability: boolean | T3CodeToolAvailability): string => { + const tools = normalizeAvailability(availability); + return `${tools.browser ? T3_CODE_BROWSER_TOOL_INSTRUCTIONS : ""}${ + tools.device ? T3_CODE_DEVICE_TOOL_INSTRUCTIONS : "" + }`; +}; const codexPlanModeDeveloperInstructions = ( - browserToolsAvailable: boolean, + browserToolsAvailable: boolean | T3CodeToolAvailability, ): string => `# Plan Mode (Conversational) You work in 3 phases, and you should *chat your way* to a great plan before finalizing it. A great plan is very detailed-intent- and implementation-wise-so that it can be handed to another engineer or agent to be implemented right away. It must be **decision complete**, where the implementer does not need to make any decisions. @@ -156,7 +177,7 @@ ${browserToolInstructions(browserToolsAvailable)} `; const codexDefaultModeDeveloperInstructions = ( - browserToolsAvailable: boolean, + browserToolsAvailable: boolean | T3CodeToolAvailability, ): string => `# Collaboration Mode: Default You are now in Default mode. Any previous instructions for other modes (e.g. Plan mode) are no longer active. @@ -184,7 +205,7 @@ export function buildCodexDeveloperInstructions( * it from the session's actual MCP configuration rather than re-reading the * setting, so the prompt cannot claim tools the turn doesn't have. */ - browserToolsAvailable = true, + browserToolsAvailable: boolean | T3CodeToolAvailability = true, ): string { const base = interactionMode === "plan" diff --git a/apps/server/src/provider/Drivers/AntigravityDriver.ts b/apps/server/src/provider/Drivers/AntigravityDriver.ts index 65a8c97fe668..0082f3cbdc30 100644 --- a/apps/server/src/provider/Drivers/AntigravityDriver.ts +++ b/apps/server/src/provider/Drivers/AntigravityDriver.ts @@ -1,3 +1,4 @@ +import { withAgentDeviceEnvironment } from "../../mcp/McpProviderSession.ts"; import { AntigravitySettings, ProviderDriverKind, ProviderSetupError } from "@t3tools/contracts"; import { HostProcessPlatform } from "@t3tools/shared/hostProcess"; import * as Crypto from "effect/Crypto"; @@ -162,7 +163,7 @@ export const AntigravityDriver: ProviderDriver 0 ? { extraArgs } : {}), ...(mcpSession diff --git a/apps/server/src/provider/Layers/CodexAdapter.ts b/apps/server/src/provider/Layers/CodexAdapter.ts index 2d88e58dc1fb..b43755736ca3 100644 --- a/apps/server/src/provider/Layers/CodexAdapter.ts +++ b/apps/server/src/provider/Layers/CodexAdapter.ts @@ -2274,7 +2274,10 @@ export const makeCodexAdapter = Effect.fn("makeCodexAdapter")(function* ( ...(mcpSession ? { environment: { - ...(options?.environment ?? process.env), + ...McpProviderSession.withAgentDeviceEnvironment( + options?.environment ?? process.env, + mcpSession, + ), T3_MCP_BEARER_TOKEN: mcpSession.authorizationHeader.replace(/^Bearer\s+/, ""), }, appServerArgs: [ @@ -2283,6 +2286,7 @@ export const makeCodexAdapter = Effect.fn("makeCodexAdapter")(function* ( "-c", 'mcp_servers.t3-code.bearer_token_env_var="T3_MCP_BEARER_TOKEN"', ], + mcpCapabilities: mcpSession.capabilities, } : {}), }; diff --git a/apps/server/src/provider/Layers/CodexSessionRuntime.ts b/apps/server/src/provider/Layers/CodexSessionRuntime.ts index 78f4b9aa8e50..d41cfd73d76a 100644 --- a/apps/server/src/provider/Layers/CodexSessionRuntime.ts +++ b/apps/server/src/provider/Layers/CodexSessionRuntime.ts @@ -39,7 +39,10 @@ import * as EffectCodexSchema from "effect-codex-app-server/schema"; import { buildCodexInitializeParams } from "./CodexProvider.ts"; import { codexSessionAppServerArgs } from "./codexLaunchArgs.ts"; import { expandHomePath } from "../../pathExpansion.ts"; -import { buildCodexDeveloperInstructions } from "../CodexDeveloperInstructions.ts"; +import { + buildCodexDeveloperInstructions, + type T3CodeToolAvailability, +} from "../CodexDeveloperInstructions.ts"; const decodeV2TurnStartResponse = Schema.decodeUnknownEffect(EffectCodexSchema.V2TurnStartResponse); const PROVIDER = ProviderDriverKind.make("codex"); @@ -66,6 +69,16 @@ export function hasConfiguredMcpServer(appServerArgs: ReadonlyArray | un return appServerArgs?.some((argument) => argument.includes("mcp_servers.")) === true; } +function configuredMcpToolAvailability( + appServerArgs: ReadonlyArray | undefined, + mcpCapabilities: ReadonlySet | undefined, +): T3CodeToolAvailability { + if (!hasConfiguredMcpServer(appServerArgs)) return { browser: false, device: false }; + // Callers predating the capability set attached the browser toolkit only. + if (mcpCapabilities === undefined) return { browser: true, device: false }; + return { browser: mcpCapabilities.has("preview"), device: mcpCapabilities.has("device") }; +} + export const CodexResumeCursorSchema = Schema.Struct({ threadId: Schema.String, }); @@ -166,6 +179,8 @@ export interface CodexSessionRuntimeOptions { readonly serviceTier?: CodexServiceTier | undefined; readonly resumeCursor?: CodexResumeCursor; readonly appServerArgs?: ReadonlyArray; + /** Capabilities the session's `t3-code` MCP credential grants; drives the prompt blocks. */ + readonly mcpCapabilities?: ReadonlySet; } export interface CodexSessionRuntimeSendTurnInput { @@ -569,7 +584,7 @@ function buildCodexCollaborationMode(input: { readonly interactionMode?: ProviderInteractionMode; readonly model?: string; readonly effort?: EffectCodexSchema.V2TurnStartParams__ReasoningEffort; - readonly browserToolsAvailable?: boolean; + readonly browserToolsAvailable?: boolean | T3CodeToolAvailability; }): EffectCodexSchema.V2TurnStartParams__CollaborationMode | undefined { if (input.interactionMode === undefined) { return undefined; @@ -603,7 +618,7 @@ export function buildTurnStartParams(input: { readonly effort?: EffectCodexSchema.V2TurnStartParams__ReasoningEffort; readonly interactionMode?: ProviderInteractionMode; /** Defaults to true so callers that predate the agent-access gate are unchanged. */ - readonly browserToolsAvailable?: boolean; + readonly browserToolsAvailable?: boolean | T3CodeToolAvailability; }): Effect.Effect< CodexTurnStartParamsWithCollaborationMode, CodexErrors.CodexAppServerProtocolParseError @@ -2352,7 +2367,10 @@ export const makeCodexSessionRuntime = ( // Derived from the session's own MCP configuration rather than the // setting, so the prompt describes the tools this turn actually // has even if the setting changed after the session started. - browserToolsAvailable: hasConfiguredMcpServer(options.appServerArgs), + browserToolsAvailable: configuredMcpToolAvailability( + options.appServerArgs, + options.mcpCapabilities, + ), }); const rawResponse = yield* client.raw.request("turn/start", params); const response = yield* decodeV2TurnStartResponse(rawResponse).pipe( diff --git a/apps/server/src/provider/Layers/CursorAdapter.ts b/apps/server/src/provider/Layers/CursorAdapter.ts index 1a77f964aac5..b19d46112fde 100644 --- a/apps/server/src/provider/Layers/CursorAdapter.ts +++ b/apps/server/src/provider/Layers/CursorAdapter.ts @@ -543,7 +543,14 @@ export function makeCursorAdapter( const mcpSession = McpProviderSession.readMcpProviderSession(input.threadId); const acp = yield* makeCursorAcpRuntime({ cursorSettings: effectiveCursorSettings, - ...(options?.environment ? { environment: options.environment } : {}), + ...(options?.environment || mcpSession?.agentDeviceEnvironment + ? { + environment: McpProviderSession.withAgentDeviceEnvironment( + options?.environment ?? process.env, + mcpSession, + ), + } + : {}), childProcessSpawner, cwd, runtimeMode: input.runtimeMode, diff --git a/apps/server/src/provider/Layers/GrokAdapter.ts b/apps/server/src/provider/Layers/GrokAdapter.ts index 25188adcffcc..a2f78a0d72d1 100644 --- a/apps/server/src/provider/Layers/GrokAdapter.ts +++ b/apps/server/src/provider/Layers/GrokAdapter.ts @@ -989,7 +989,14 @@ export function makeGrokAdapter(grokSettings: GrokSettings, options?: GrokAdapte const mcpSession = McpProviderSession.readMcpProviderSession(input.threadId); const acp = yield* makeGrokAcpRuntime({ grokSettings, - ...(options?.environment ? { environment: options.environment } : {}), + ...(options?.environment || mcpSession?.agentDeviceEnvironment + ? { + environment: McpProviderSession.withAgentDeviceEnvironment( + options?.environment ?? process.env, + mcpSession, + ), + } + : {}), childProcessSpawner, cwd, runtimeMode: input.runtimeMode, diff --git a/apps/server/src/provider/Layers/OpenCodeAdapter.ts b/apps/server/src/provider/Layers/OpenCodeAdapter.ts index 742ee9b86d6a..312a9494bebc 100644 --- a/apps/server/src/provider/Layers/OpenCodeAdapter.ts +++ b/apps/server/src/provider/Layers/OpenCodeAdapter.ts @@ -2827,19 +2827,22 @@ export function makeOpenCodeAdapter( // The runtime binds the server's lifetime to the Scope.Scope // we provide below — closing `sessionScope` kills the child // process automatically. No manual `server.close()` needed. + const mcpSession = McpProviderSession.readMcpProviderSession(input.threadId); const server = yield* openCodeRuntime.connectToOpenCodeServer({ binaryPath, directory, serverUrl, ...(serverPassword ? { serverPassword } : {}), - ...(options?.environment ? { environment: options.environment } : {}), + environment: McpProviderSession.withAgentDeviceEnvironment( + options?.environment ?? process.env, + mcpSession, + ), }); const client = openCodeRuntime.createOpenCodeSdkClient({ baseUrl: server.url, directory, ...(server.serverPassword ? { serverPassword: server.serverPassword } : {}), }); - const mcpSession = McpProviderSession.readMcpProviderSession(input.threadId); if (mcpSession && !server.external) { yield* runOpenCodeSdk("mcp.add", () => client.mcp.add({ diff --git a/apps/server/src/provider/Layers/ProviderService.test.ts b/apps/server/src/provider/Layers/ProviderService.test.ts index e7647866d604..0b2d8562ebc7 100644 --- a/apps/server/src/provider/Layers/ProviderService.test.ts +++ b/apps/server/src/provider/Layers/ProviderService.test.ts @@ -4801,12 +4801,16 @@ describe("agent browser access", () => { const projectId = ProjectId.make("project-browser-access"); const startSessionWith = ( - enableAgentBrowserAccess: boolean, + access: boolean | { readonly browser: boolean; readonly device: boolean }, threadId: ThreadId, projectOverride?: boolean, ) => Effect.gen(function* () { - const issued: Array = []; + const enableAgentBrowserAccess = typeof access === "boolean" ? access : access.browser; + // Device access shares the credential; a browser-only test turns it off + // too so "no credential at all" stays observable. + const enableAgentDeviceAccess = typeof access === "boolean" ? access : access.device; + const issued: Array<{ threadId: ThreadId; capabilities: ReadonlyArray }> = []; const codex = makeFakeCodexAdapter(); const providerAdapterLayer = Layer.succeed( ProviderAdapterRegistry.ProviderAdapterRegistry, @@ -4865,7 +4869,10 @@ describe("agent browser access", () => { const providerLayer = makeProviderServiceLive({ issueMcpCredential: (request) => Effect.sync(() => { - issued.push(request.threadId); + issued.push({ + threadId: request.threadId, + capabilities: [...request.capabilities].toSorted(), + }); return undefined; }), revokeMcpCredential: (revoked) => Effect.sync(() => void revokedThreads.push(revoked)), @@ -4876,6 +4883,7 @@ describe("agent browser access", () => { Layer.provide( ServerSettings.ServerSettingsService.layerTest({ enableAgentBrowserAccess, + enableAgentDeviceAccess, projectAgentBrowserAccessOverrides: projectOverride === undefined ? {} : { [projectId]: projectOverride }, }), @@ -4934,7 +4942,17 @@ describe("agent browser access", () => { const issued = yield* startSessionWith(true, threadId); - assert.deepEqual(issued, [threadId]); + assert.deepEqual(issued, [{ threadId, capabilities: ["device", "preview"] }]); + }).pipe(Effect.provide(NodeServices.layer)), + ); + + it.effect("drops only the preview capability when browser access alone is off", () => + Effect.gen(function* () { + const threadId = asThreadId("thread-browser-off-device-on"); + + const issued = yield* startSessionWith({ browser: false, device: true }, threadId); + + assert.deepEqual(issued, [{ threadId, capabilities: ["device"] }]); }).pipe(Effect.provide(NodeServices.layer)), ); @@ -4942,17 +4960,25 @@ describe("agent browser access", () => { Effect.gen(function* () { const threadId = asThreadId("thread-project-browser-off"); revokedThreads.length = 0; - const issued = yield* startSessionWith(true, threadId, false); + const issued = yield* startSessionWith({ browser: true, device: false }, threadId, false); assert.deepEqual(issued, []); assert.deepEqual(revokedThreads, [threadId]); }).pipe(Effect.provide(NodeServices.layer)), ); + it.effect("a project browser override leaves device access alone", () => + Effect.gen(function* () { + const threadId = asThreadId("thread-project-browser-off-device-on"); + const issued = yield* startSessionWith(true, threadId, false); + assert.deepEqual(issued, [{ threadId, capabilities: ["device"] }]); + }).pipe(Effect.provide(NodeServices.layer)), + ); + it.effect("requests an MCP credential when the project overrides browser access to on", () => Effect.gen(function* () { const threadId = asThreadId("thread-project-browser-on"); - const issued = yield* startSessionWith(false, threadId, true); - assert.deepEqual(issued, [threadId]); + const issued = yield* startSessionWith({ browser: false, device: false }, threadId, true); + assert.deepEqual(issued, [{ threadId, capabilities: ["preview"] }]); }).pipe(Effect.provide(NodeServices.layer)), ); }); diff --git a/apps/server/src/provider/Layers/ProviderService.ts b/apps/server/src/provider/Layers/ProviderService.ts index 016bcd5c4a28..2fb58568f08f 100644 --- a/apps/server/src/provider/Layers/ProviderService.ts +++ b/apps/server/src/provider/Layers/ProviderService.ts @@ -34,6 +34,7 @@ import { type ProviderSession, } from "@t3tools/contracts"; import { expandAssistantCitationsForProvider } from "@t3tools/shared/assistantCitations"; +import { HostProcessPlatform } from "@t3tools/shared/hostProcess"; import { causeErrorTag } from "@t3tools/shared/observability"; import { getModelSelectionStringOptionValue } from "@t3tools/shared/model"; import { resolveProjectAgentBrowserAccess } from "@t3tools/shared/serverSettings"; @@ -43,6 +44,7 @@ import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; +import * as Path from "effect/Path"; import * as PubSub from "effect/PubSub"; import * as Ref from "effect/Ref"; import * as Schema from "effect/Schema"; @@ -52,6 +54,9 @@ import * as Stream from "effect/Stream"; import { appendUserInputAttachmentPaths } from "../userInputAttachments.ts"; import { resolveAttachmentPath } from "../../attachmentStore.ts"; import * as ServerConfig from "../../config.ts"; +import { ensureAgentDeviceShim } from "../../device/AgentDeviceShim.ts"; +import * as DeviceService from "../../device/DeviceService.ts"; +import type * as McpInvocationContext from "../../mcp/McpInvocationContext.ts"; import { increment, providerMetricAttributes, @@ -251,6 +256,8 @@ export interface ProviderServiceLiveOptions { readonly issueMcpCredential?: typeof McpSessionRegistry.issueActiveMcpCredential; /** Same seam as `issueMcpCredential`, for observing the deny path's revoke. */ readonly revokeMcpCredential?: typeof McpSessionRegistry.revokeActiveMcpThread; + /** Overrides the device host lookup used to build the agent-device environment. */ + readonly deviceReadiness?: DeviceService.DeviceService["Service"]["agentReadinessIfSupported"]; } interface TurnAnalyticsMetadata { @@ -481,7 +488,16 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( options?.issueMcpCredential ?? McpSessionRegistry.issueActiveMcpCredential; const revokeMcpCredential = options?.revokeMcpCredential ?? McpSessionRegistry.revokeActiveMcpThread; + const deviceReadiness = + options?.deviceReadiness ?? + (() => + Effect.serviceOption(DeviceService.DeviceService).pipe( + Effect.flatMap((service) => + Option.isSome(service) ? service.value.agentReadinessIfSupported() : Effect.succeed(null), + ), + )); const fileSystem = yield* FileSystem.FileSystem; + const pathService = yield* Path.Path; const runtimeEventPubSub = yield* PubSub.unbounded(); const pendingCompactions = new Map(); const timedOutNativeCompactions = new Set(); @@ -889,9 +905,65 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( ), ); + const agentDeviceAccessEnabled = serverSettings.getSettings.pipe( + Effect.map((settings) => settings.enableAgentDeviceAccess), + Effect.catch((cause) => + Effect.logWarning( + "Could not read server settings; withholding agent device access for this session.", + { cause }, + ).pipe(Effect.as(false)), + ), + ); + + const agentAccessCapabilities = Effect.fn("ProviderService.agentAccessCapabilities")(function* ( + threadId: ThreadId, + ) { + const capabilities = new Set(); + if (yield* agentBrowserAccessEnabled(threadId)) capabilities.add("preview"); + if (yield* agentDeviceAccessEnabled) capabilities.add("device"); + return capabilities; + }); + + /** + * Starting a session with device access also brings the device host up, so + * the `agent-device` CLI is on the provider's PATH from its first turn. The + * environment is fixed at spawn time, so a host started later by + * `device_open` could not reach an already-running agent. Tools install once + * and the host is idempotent, so this is cheap after the first session; + * a host that fails to start withholds only the CLI, not the MCP tools. + */ + const hostPlatform = yield* HostProcessPlatform; + const agentDeviceEnvironment = Effect.gen(function* () { + const readiness = yield* deviceReadiness().pipe( + Effect.catch((cause) => + Effect.logWarning("Device host unavailable; starting session without agent-device", { + cause, + }).pipe(Effect.as(null)), + ), + ); + if (!readiness) return undefined; + const shimDir = yield* ensureAgentDeviceShim({ + entryPath: readiness.agentDevice.entryPath, + stateDir: serverConfig.stateDir, + }).pipe( + Effect.provideService(FileSystem.FileSystem, fileSystem), + Effect.provideService(Path.Path, pathService), + Effect.orElseSucceed(() => undefined), + ); + if (!shimDir) return undefined; + return { + PATH: shimDir, + PATH_SEPARATOR: hostPlatform === "win32" ? ";" : ":", + AGENT_DEVICE_DAEMON_BASE_URL: readiness.agentDevice.baseUrl, + AGENT_DEVICE_DAEMON_AUTH_TOKEN: readiness.agentDevice.token, + AGENT_DEVICE_NO_UPDATE_NOTIFIER: "1", + } satisfies Record; + }); + const prepareMcpSession = (threadId: ThreadId, providerInstanceId: ProviderInstanceId) => Effect.gen(function* () { - if (!(yield* agentBrowserAccessEnabled(threadId))) { + const capabilities = yield* agentAccessCapabilities(threadId); + if (capabilities.size === 0) { // Revoke as well as clear. Every other prepare path reaches // `issueActiveMcpCredential`, which revokes the thread first, so // skipping it here would leave a previously issued bearer token valid @@ -902,9 +974,17 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( yield* Effect.sync(() => McpProviderSession.clearMcpProviderSession(threadId)); return undefined; } - const credential = yield* issueMcpCredential({ threadId, providerInstanceId }); + const credential = yield* issueMcpCredential({ threadId, providerInstanceId, capabilities }); if (credential) { - yield* Effect.sync(() => McpProviderSession.setMcpProviderSession(credential.config)); + const deviceEnvironment = capabilities.has("device") + ? yield* agentDeviceEnvironment + : undefined; + yield* Effect.sync(() => + McpProviderSession.setMcpProviderSession({ + ...credential.config, + ...(deviceEnvironment ? { agentDeviceEnvironment: deviceEnvironment } : {}), + }), + ); } return credential; }); diff --git a/apps/server/src/provider/acp/AntigravityAcpSupport.ts b/apps/server/src/provider/acp/AntigravityAcpSupport.ts index 6e625127cb18..17b73e552006 100644 --- a/apps/server/src/provider/acp/AntigravityAcpSupport.ts +++ b/apps/server/src/provider/acp/AntigravityAcpSupport.ts @@ -35,6 +35,8 @@ export interface AntigravityAcpRuntimeInput extends Omit< | "transformSessionUpdate" | "transformStdout" > { + /** Device CLI environment supplied for this provider session. */ + readonly agentDeviceEnvironment?: Readonly>; readonly childProcessSpawner: ChildProcessSpawner.ChildProcessSpawner["Service"]; readonly onAuthorizationUrl?: (url: string) => Effect.Effect; /** diff --git a/apps/server/src/server.test.ts b/apps/server/src/server.test.ts index 2f32b6524d7b..41d380410516 100644 --- a/apps/server/src/server.test.ts +++ b/apps/server/src/server.test.ts @@ -5,6 +5,7 @@ import * as NodeCrypto from "node:crypto"; import { HostProcessEnvironment, HostProcessPlatform } from "@t3tools/shared/hostProcess"; import { + type DeviceServiceState, AuthAccessTokenType, AuthStandardClientScopes, AuthEnvironmentBootstrapTokenType, @@ -97,6 +98,7 @@ const encodeTestJson = Schema.encodeUnknownSync(Schema.fromJsonString(Schema.Unk import * as BackgroundPolicy from "./background/BackgroundPolicy.ts"; import * as ServerConfig from "./config.ts"; +import * as DeviceService from "./device/DeviceService.ts"; import { HTTP_ROUTER_CONFIG, makeRoutesLayer } from "./server.ts"; import { isThreadDetailEvent, @@ -802,6 +804,11 @@ const buildAppUnderTest = (options?: { listBindings: () => Effect.succeed([]), ...options?.layers?.providerSessionDirectory, }), + Layer.mock(DeviceService.DeviceService)({ + state: Effect.succeed(EMPTY_DEVICE_STATE), + currentReadiness: () => Effect.succeed(null), + sessionsForThread: () => Effect.succeed([]), + }), ), ), Layer.provide( @@ -1652,6 +1659,17 @@ const NodeHttpServerTestWithWsDeflate = HttpServer.layerTestClient.pipe( ), ); +const EMPTY_DEVICE_STATE: DeviceServiceState = { + hosts: [], + hostStatus: "idle", + devices: [], + sessions: [], + onboardingCompleted: false, + agentAccessEnabled: false, + hubBasePath: DeviceService.DEVICE_HUB_ROUTE_PREFIX, + revision: 0, +}; + it.layer(NodeServices.layer)("server router seam", (it) => { it.effect("parks HTTP ingress until command readiness", () => Effect.gen(function* () { diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index fd8ee4a4f699..8a083b56d4d6 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -57,6 +57,8 @@ import * as TerminalManager from "./terminal/Manager.ts"; import * as McpHttpServer from "./mcp/McpHttpServer.ts"; import * as McpSessionRegistry from "./mcp/McpSessionRegistry.ts"; import * as PreviewAutomationBroker from "./mcp/PreviewAutomationBroker.ts"; +import * as DeviceService from "./device/DeviceService.ts"; +import { deviceHubProxyRouteLayer } from "./device/DeviceHubProxy.ts"; import * as PreviewManager from "./preview/Manager.ts"; import * as PortScanner from "./preview/PortScanner.ts"; import * as ProcessRunner from "./processRunner.ts"; @@ -382,6 +384,12 @@ const PreviewLayerLive = Layer.empty.pipe( Layer.provideMerge(PortScannerLayerLive), ); +const DeviceLayerLive = DeviceService.layer.pipe( + Layer.provide(ServerSettingsLayerLive), + Layer.provide(ProcessRunner.layer), + Layer.provide(NetService.layer), +); + const WorkspaceEntriesLayerLive = WorkspaceEntries.layer.pipe(Layer.provide(WorkspacePaths.layer)); const WorkspaceFileSystemLayerLive = WorkspaceFileSystem.layer.pipe( @@ -465,7 +473,7 @@ const RuntimeCoreDependenciesLive = ReactorLayerLive.pipe( Layer.provideMerge(GitLayerLive), Layer.provideMerge(VcsLayerLive), Layer.provideMerge(ProviderRuntimeLayerLive), - Layer.provideMerge(Layer.mergeAll(TerminalLayerLive, PreviewLayerLive)), + Layer.provideMerge(Layer.mergeAll(TerminalLayerLive, PreviewLayerLive, DeviceLayerLive)), Layer.provideMerge(PersistenceLayerLive), // Both read a user-owned file out of the state directory and stream changes // to clients; neither depends on the other. @@ -549,6 +557,7 @@ export const makeRoutesLayer = Layer.mergeAll( otlpTracesProxyRouteLayer, assetRouteLayer, attachmentUploadRouteLayer, + deviceHubProxyRouteLayer, staticAndDevRouteLayer, websocketRpcRouteLayer, ), diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index 740be1330817..a7730e832ddc 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -113,6 +113,7 @@ import * as ServerRuntimeStartup from "./serverRuntimeStartup.ts"; import * as ServerSettings from "./serverSettings.ts"; import * as TerminalManager from "./terminal/Manager.ts"; import * as PreviewAutomationBroker from "./mcp/PreviewAutomationBroker.ts"; +import * as DeviceService from "./device/DeviceService.ts"; import * as PreviewManager from "./preview/Manager.ts"; import { issueAssetUrl } from "./assets/AssetAccess.ts"; import { deletePendingAttachment, issueAttachmentUploadUrl } from "./assets/AttachmentUpload.ts"; @@ -519,6 +520,7 @@ const makeWsRpcLayer = ( const vcsStatusBroadcaster = yield* VcsStatusBroadcaster.VcsStatusBroadcaster; const terminalManager = yield* TerminalManager.TerminalManager; const previewManager = yield* PreviewManager.PreviewManager; + const deviceService = yield* DeviceService.DeviceService; const portDiscovery = yield* PortScanner.PortDiscovery; const providerRegistry = yield* ProviderRegistry.ProviderRegistry; const providerService = yield* ProviderService.ProviderService; @@ -2685,6 +2687,40 @@ const makeWsRpcLayer = ( observeRpcStream(WS_METHODS.subscribePreviewEvents, previewManager.events, { "rpc.aggregate": "preview", }), + [WS_METHODS.deviceConfigure]: (input) => + observeRpcEffect(WS_METHODS.deviceConfigure, deviceService.configure(input), { + "rpc.aggregate": "device", + }), + [WS_METHODS.deviceList]: (_input) => + observeRpcEffect(WS_METHODS.deviceList, deviceService.list, { + "rpc.aggregate": "device", + }), + [WS_METHODS.deviceOpen]: (input) => + observeRpcEffect(WS_METHODS.deviceOpen, deviceService.open(input), { + "rpc.aggregate": "device", + }), + [WS_METHODS.deviceClose]: (input) => + observeRpcEffect(WS_METHODS.deviceClose, deviceService.close(input), { + "rpc.aggregate": "device", + }), + [WS_METHODS.deviceShutdown]: (input) => + observeRpcEffect(WS_METHODS.deviceShutdown, deviceService.shutdown(input), { + "rpc.aggregate": "device", + }), + [WS_METHODS.deviceDetail]: (input) => + observeRpcEffect(WS_METHODS.deviceDetail, deviceService.detail(input), { + "rpc.aggregate": "device", + }), + [WS_METHODS.deviceAction]: (input) => + observeRpcEffect(WS_METHODS.deviceAction, deviceService.action(input), { + "rpc.aggregate": "device", + }), + [WS_METHODS.subscribeDeviceState]: (_input) => + observeRpcStream( + WS_METHODS.subscribeDeviceState, + DeviceService.stateStream(deviceService), + { "rpc.aggregate": "device" }, + ), [WS_METHODS.subscribeDiscoveredLocalServers]: (input) => observeRpcStream( WS_METHODS.subscribeDiscoveredLocalServers, diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index 0fbef88c81e1..913bb01bd831 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -203,6 +203,10 @@ import { PullRequestDetailGhost } from "./pullRequest/PullRequestGhosts"; import { PullRequestsUnavailableState } from "./pullRequest/PullRequestsUnavailableState"; import { RightPanelTabs } from "./RightPanelTabs"; import { AgentsPanel } from "./AgentsPanel"; +import { useDeviceState } from "~/state/device"; +import { DeviceSetup } from "./device/DeviceSetup"; +import { Dialog } from "./ui/dialog"; +import { WizardPopup } from "./ui/wizard"; import { deriveAgentPanelModel, foldSubagentActivities, @@ -554,6 +558,9 @@ const PreviewPanel = lazy(() => import("./preview/PreviewPanel").then((module) => ({ default: module.PreviewPanel })), ); const DiffPanel = lazy(() => import("./DiffPanel")); +const DevicePanel = lazy(() => + import("./device/DevicePanel").then((module) => ({ default: module.DevicePanel })), +); const FilePreviewPanel = lazy(() => import("./files/FilePreviewPanel")); const EMPTY_PENDING_FILE_SURFACE_IDS: ReadonlySet = new Set(); const TYPE_TO_FOCUS_EDITABLE_SELECTOR = [ @@ -4127,6 +4134,34 @@ export default function ChatView(props: ChatViewProps) { if (!activeThreadRef) return; useRightPanelStore.getState().open(activeThreadRef, "agents"); }, [activeThreadRef]); + const { state: deviceState } = useDeviceState(activeThreadRef?.environmentId ?? null); + const [deviceSetupThread, setDeviceSetupThread] = useState(null); + const addDeviceSurface = useCallback(() => { + if (!activeThreadRef) return; + if (!deviceState.onboardingCompleted || deviceState.hostStatus === "disabled") { + setDeviceSetupThread(activeThreadRef); + return; + } + useRightPanelStore.getState().open(activeThreadRef, "device"); + }, [activeThreadRef, deviceState.onboardingCompleted, deviceState.hostStatus]); + // An agent's `device_open` surfaces in every client the same way a + // `preview_open` does: the thread starts a device or gains a session and the panel + // opens on it. Closing the last session leaves the tab in place so the + // user keeps their picker; only new sessions raise the panel. + const threadDeviceSessionCount = activeThreadRef + ? deviceState.sessions.filter((session) => session.threadId === activeThreadRef.threadId) + .length + + (deviceState.bootingDevices?.filter((device) => device.threadId === activeThreadRef.threadId) + .length ?? 0) + : 0; + const previousDeviceSessionCount = useRef(threadDeviceSessionCount); + useEffect(() => { + const previous = previousDeviceSessionCount.current; + previousDeviceSessionCount.current = threadDeviceSessionCount; + if (!activeThreadRef || threadDeviceSessionCount <= previous) return; + if (shouldUseRightPanelSheet) return; + useRightPanelStore.getState().open(activeThreadRef, "device"); + }, [activeThreadRef, shouldUseRightPanelSheet, threadDeviceSessionCount]); const openFileSurface = useCallback( (relativePath: string) => { if (!activeThreadRef || !activeProject) return; @@ -8069,6 +8104,19 @@ export default function ChatView(props: ChatViewProps) { environmentId={activeThreadRef?.environmentId ?? null} threadId={activeThreadRef?.threadId ?? null} /> + ) : renderedRightPanelSurface?.kind === "device" ? ( + + { + closeRightPanelSurface(renderedRightPanelSurface); + useRightPanelStore.getState().show(activeThreadRef); + }} + /> + ) : (renderedRightPanelSurface?.kind === "files" || renderedRightPanelSurface?.kind === "file") && ((activeProject && activeWorkspaceRoot) || @@ -8124,6 +8172,29 @@ export default function ChatView(props: ChatViewProps) { return (
+ { + if (!open) setDeviceSetupThread(null); + }} + > + + {activeThreadRef ? ( + { + useRightPanelStore.getState().open(activeThreadRef, "device"); + setDeviceSetupThread(null); + }} + /> + ) : null} + + {rightPanelControlsAtRoot ? panelLayoutControls : null}
{rightPanelContent} @@ -8671,12 +8744,14 @@ export default function ChatView(props: ChatViewProps) { onAddFiles={addFilesSurface} onAddPullRequest={addPullRequestSurface} onAddAgents={addAgentsSurface} + onAddDevice={addDeviceSurface} browserAvailable={isPreviewSupportedInRuntime()} terminalAvailable={activeProject !== null} diffAvailable={isServerThread && isGitRepo} filesAvailable={activeProject !== null} pullRequestAvailable={pullRequestSurfaceAvailable} agentsAvailable + deviceAvailable={activeThreadRef !== null} liveAgentCount={agentPanelModel.liveCount} > {rightPanelContent} diff --git a/apps/web/src/components/RightPanelTabs.test.tsx b/apps/web/src/components/RightPanelTabs.test.tsx index 81367d75580d..82ead6644d42 100644 --- a/apps/web/src/components/RightPanelTabs.test.tsx +++ b/apps/web/src/components/RightPanelTabs.test.tsx @@ -120,6 +120,7 @@ function renderTabs( onAddDiff={() => undefined} onAddFiles={() => undefined} onAddAgents={() => undefined} + onAddDevice={() => undefined} liveAgentCount={0} browserAvailable terminalAvailable={false} @@ -127,6 +128,7 @@ function renderTabs( filesAvailable={false} pullRequestAvailable={false} agentsAvailable={false} + deviceAvailable={false} >
content
, diff --git a/apps/web/src/components/RightPanelTabs.tsx b/apps/web/src/components/RightPanelTabs.tsx index e9dab0c9d2b4..2d6db2ec8435 100644 --- a/apps/web/src/components/RightPanelTabs.tsx +++ b/apps/web/src/components/RightPanelTabs.tsx @@ -8,6 +8,7 @@ import type { import { getTerminalLabel } from "@t3tools/shared/terminalLabels"; import { Bot, + Smartphone, ChevronDown, ChevronLeft, ChevronRight, @@ -105,12 +106,14 @@ interface RightPanelTabsProps { onAddFiles: () => void; onAddPullRequest: () => void; onAddAgents: () => void; + onAddDevice: () => void; browserAvailable: boolean; terminalAvailable: boolean; diffAvailable: boolean; filesAvailable: boolean; pullRequestAvailable: boolean; agentsAvailable: boolean; + deviceAvailable: boolean; pullRequestStatusSeeds?: Readonly>; /** Running + waiting subagents; badges the Agents card in the empty state. */ liveAgentCount: number; @@ -140,6 +143,7 @@ const SURFACE_DISABLED_REASONS = { diff: "Diff is only available for server threads in Git repositories.", pullRequest: "This thread's branch has no pull request yet.", agents: "Agents are only available from a thread.", + device: "Devices are only available from a thread.", } as const; /** Overlays that must win over the launcher's letter shortcuts. */ @@ -162,6 +166,7 @@ const SURFACE_UNAVAILABLE_HINTS = { diff: "Available for Git repositories.", pullRequest: "No pull request on this branch yet.", agents: "Available from a thread.", + device: "Available from a thread.", } as const; type TabContextMenuAction = @@ -299,12 +304,14 @@ function RightPanelEmptyState(props: { onAddFiles: () => void; onAddPullRequest: () => void; onAddAgents: () => void; + onAddDevice: () => void; browserAvailable: boolean; terminalAvailable: boolean; diffAvailable: boolean; filesAvailable: boolean; pullRequestAvailable: boolean; agentsAvailable: boolean; + deviceAvailable: boolean; liveAgentCount: number; }) { // -1 means no highlight: it only appears on hover or arrow use. @@ -371,6 +378,16 @@ function RightPanelEmptyState(props: { onClick: props.onAddAgents, badgeCount: props.liveAgentCount, }, + { + label: "Device", + description: "Watch an iOS Simulator or Android Emulator.", + icon: Smartphone, + shortcut: "M", + available: props.deviceAvailable, + disabledReason: SURFACE_UNAVAILABLE_HINTS.device, + onClick: props.onAddDevice, + badgeCount: 0, + }, ] as const; type SurfaceAction = (typeof actions)[number]; @@ -604,6 +621,8 @@ function surfaceTitle( return `#${surface.number}`; case "agents": return "Agents"; + case "device": + return "Device"; case "preview": { const snapshot = surface.resourceId ? sessions[surface.resourceId] : null; if (!snapshot || snapshot.navStatus._tag === "Idle") return "Browser"; @@ -685,6 +704,8 @@ function SurfaceIcon({ ); case "agents": return ; + case "device": + return ; } } @@ -814,6 +835,14 @@ export function RightPanelTabs(props: RightPanelTabsProps) { disabledReason: SURFACE_DISABLED_REASONS.agents, onClick: props.onAddAgents, }, + { + label: "Device", + icon: Smartphone, + shortcut: "M", + available: props.deviceAvailable, + disabledReason: SURFACE_DISABLED_REASONS.device, + onClick: props.onAddDevice, + }, ] as const; const handleAddSurfaceMenuKeyDown = (event: ReactKeyboardEvent) => { @@ -1251,12 +1280,14 @@ export function RightPanelTabs(props: RightPanelTabsProps) { onAddFiles={props.onAddFiles} onAddPullRequest={props.onAddPullRequest} onAddAgents={props.onAddAgents} + onAddDevice={props.onAddDevice} browserAvailable={props.browserAvailable} terminalAvailable={props.terminalAvailable} diffAvailable={props.diffAvailable} filesAvailable={props.filesAvailable} pullRequestAvailable={props.pullRequestAvailable} agentsAvailable={props.agentsAvailable} + deviceAvailable={props.deviceAvailable} liveAgentCount={props.liveAgentCount} /> ) : ( diff --git a/apps/web/src/components/chat/MessagesTimeline.logic.ts b/apps/web/src/components/chat/MessagesTimeline.logic.ts index a87531673506..badafd9c739d 100644 --- a/apps/web/src/components/chat/MessagesTimeline.logic.ts +++ b/apps/web/src/components/chat/MessagesTimeline.logic.ts @@ -337,7 +337,7 @@ export type MessagesTimelineRow = summaryKind: ToolGroupSummaryKind; toolSurface?: WorkLogEntry["toolSurface"]; toolIcon?: WorkLogEntry["toolIcon"]; - summaryToolIcon?: "browser" | "t3-code"; + summaryToolIcon?: "browser" | "device" | "t3-code"; hasFailure: boolean; } | { diff --git a/apps/web/src/components/chat/MessagesTimeline.tsx b/apps/web/src/components/chat/MessagesTimeline.tsx index 1d4abe39bf39..c05d7d0b8d0c 100644 --- a/apps/web/src/components/chat/MessagesTimeline.tsx +++ b/apps/web/src/components/chat/MessagesTimeline.tsx @@ -96,6 +96,7 @@ import { MousePointerClickIcon, PaintbrushIcon, SearchIcon, + SmartphoneIcon, SquarePenIcon, TerminalIcon, Undo2Icon, @@ -2794,6 +2795,7 @@ type WorkEntryIconName = | "check" | "circle-alert" | "computer" + | "device" | "eye" | "globe" | "hammer" @@ -3013,6 +3015,8 @@ function WorkEntryIcon({ name, className }: { name: WorkEntryIconName; className return ; case "computer": return ; + case "device": + return ; case "t3-code": return ; case "check": diff --git a/apps/web/src/components/device/DevicePanel.tsx b/apps/web/src/components/device/DevicePanel.tsx new file mode 100644 index 000000000000..d94913b03dff --- /dev/null +++ b/apps/web/src/components/device/DevicePanel.tsx @@ -0,0 +1,473 @@ +import type { + DevicePlatform, + DeviceServiceState, + DeviceSummary, + ScopedThreadRef, +} from "@t3tools/contracts"; +import { + ChevronLeft, + Circle, + Home, + Power, + RotateCcw, + SlidersHorizontal, + Smartphone, + Square, + X, +} from "lucide-react"; +import { useCallback, useEffect, useMemo, useState } from "react"; + +import { Button } from "~/components/ui/button"; +import { DiscoveryList, DiscoveryListRow } from "~/components/ui/discovery-list"; +import { Dialog } from "~/components/ui/dialog"; +import { WizardPopup } from "~/components/ui/wizard"; +import { + Select, + SelectGroup, + SelectGroupLabel, + SelectItem, + SelectPopup, + SelectTrigger, + SelectValue, +} from "~/components/ui/select"; +import { Spinner } from "~/components/ui/spinner"; +import { Toggle } from "~/components/ui/toggle"; +import { Tooltip, TooltipPopup, TooltipTrigger } from "~/components/ui/tooltip"; +import { cn } from "~/lib/utils"; +import { deviceEnvironment, useDeviceHubAccess, useDeviceState } from "~/state/device"; +import { formatEnvironmentQueryError } from "~/state/query"; +import { useAtomCommand } from "~/state/use-atom-command"; +import { DeviceStreamView, type DeviceStreamHandle } from "./DeviceStreamView"; +import { DeviceSetup } from "./DeviceSetup"; +import { DeviceToolsPanel } from "./DeviceToolsPanel"; +import { PreviewPanelShell, type PreviewPanelMode } from "../preview/PreviewPanelShell"; + +const NEW_DEVICE_VALUE = "__new__"; + +const platformLabel = (platform: DevicePlatform) => + platform === "ios" ? "iOS Simulators" : "Android Emulators"; + +const deviceKey = (device: Pick) => + `${device.hostId}\u0000${device.id}`; + +/** + * The Device right-panel surface: one open device (from the thread's device + * sessions) with a picker to switch or boot another. Booting and streaming are + * server-owned; this panel only asks and renders. + */ +export function DevicePanel(props: { + readonly mode: PreviewPanelMode; + readonly threadRef: ScopedThreadRef; + /** `null` renders the picker with nothing open. */ + readonly deviceId: string | null; + readonly visible: boolean; + readonly onDismissSetup: () => void; +}) { + const { environmentId, threadId } = props.threadRef; + const { state, loaded } = useDeviceState(environmentId); + const list = useAtomCommand(deviceEnvironment.list, { reportFailure: false }); + const open = useAtomCommand(deviceEnvironment.open); + const close = useAtomCommand(deviceEnvironment.close); + const [operationError, setOperationError] = useState(null); + const [pendingDeviceKey, setPendingDeviceKey] = useState(null); + const [handle, setHandle] = useState(null); + const [toolsOpen, setToolsOpen] = useState(false); + const [axOverlay, setAxOverlay] = useState(false); + const access = useDeviceHubAccess(environmentId); + + const hostDisabled = state.hostStatus === "disabled"; + + // Opening setup never grants permission to install or start helpers. + useEffect(() => { + if (!props.visible || !loaded || hostDisabled) return; + void list({ environmentId, input: {} }); + }, [environmentId, list, loaded, props.visible, hostDisabled]); + + const sessions = useMemo( + () => state.sessions.filter((session) => session.threadId === threadId), + [state.sessions, threadId], + ); + const activeSession = + (props.deviceId + ? sessions.find((session) => session.deviceId === props.deviceId) + : undefined) ?? sessions.at(-1); + const activeDevice = activeSession + ? state.devices.find( + (device) => device.hostId === activeSession.hostId && device.id === activeSession.deviceId, + ) + : undefined; + + const grouped = useMemo(() => groupDevices(state), [state]); + + const selectDevice = useCallback( + async (value: string) => { + if (value === NEW_DEVICE_VALUE) return; + const device = state.devices.find((candidate) => deviceKey(candidate) === value); + if (!device) return; + setOperationError(null); + setPendingDeviceKey(value); + try { + const result = await open({ + environmentId, + input: { + threadId, + hostId: device.hostId, + deviceId: device.id, + platform: device.platform, + }, + }); + if (result._tag === "Failure") setOperationError(formatEnvironmentQueryError(result.cause)); + } finally { + setPendingDeviceKey(null); + } + }, + [environmentId, open, state.devices, threadId], + ); + + const closeActive = useCallback( + (powerOff: boolean) => { + if (!activeSession) return; + setOperationError(null); + void close({ + environmentId, + input: { threadId, deviceId: activeSession.deviceId, shutdown: powerOff }, + }).then((result) => { + if (result._tag === "Failure") setOperationError(formatEnvironmentQueryError(result.cause)); + }); + }, + [activeSession, close, environmentId, threadId], + ); + + const bootingDevices = + state.bootingDevices?.filter((device) => device.threadId === threadId) ?? []; + const hostReady = state.hostStatus === "ready"; + const hostBusy = state.hostStatus === "installing" || state.hostStatus === "starting"; + const unavailablePlatforms = state.hosts.flatMap((host) => + host.platforms.filter((platform) => !platform.available), + ); + + if (loaded && (!state.onboardingCompleted || hostDisabled)) { + return ( + { + if (!isOpen) props.onDismissSetup(); + }} + > + + + + + ); + } + + return ( + +
+ + {activeDevice ? ( + <> + handle?.pressButton("home")} + disabled={!handle?.inputConnected} + > + + + {activeDevice.platform === "android" ? ( + <> + handle?.pressButton("back")} + disabled={!handle?.inputConnected} + > + + + handle?.pressButton("recents")} + disabled={!handle?.inputConnected} + > + + + + ) : ( + handle?.rotate()} + disabled={!handle?.inputConnected} + > + + + )} + setToolsOpen(Boolean(pressed))} + > + + + closeActive(true)}> + + + closeActive(false)}> + + + + ) : null} +
+ {hostReady && state.hostStatusDetail ? ( +
+ {state.hostStatusDetail} +
+ ) : null} + {bootingDevices.length > 0 ? ( +
+ Starting {bootingDevices.map((device) => device.name).join(", ")}… This can take a minute. +
+ ) : null} + {operationError ? ( +
+

{operationError}

+ +
+ ) : null} +
+ {activeDevice && activeSession ? ( + <> +
+ +
+ {toolsOpen ? ( + setToolsOpen(false)} + className="absolute inset-y-0 right-0 z-10 w-full max-w-72 border-l shadow-lg @[560px]:static @[560px]:w-72 @[560px]:shrink-0 @[560px]:shadow-none" + /> + ) : null} + + ) : ( +
+
+ {grouped.length === 0 || hostBusy || pendingDeviceKey ? ( + <> + {hostBusy || pendingDeviceKey ? ( + + ) : ( + + )} +

+ {state.hostStatus === "failed" + ? (state.hostStatusDetail ?? "The device hub failed to start.") + : pendingDeviceKey + ? "Booting device… this can take a minute." + : hostBusy + ? state.hostStatus === "installing" + ? "Installing device tools…" + : "Starting the device hub…" + : !loaded + ? "Connecting…" + : grouped.length === 0 + ? "No simulators or emulators were found on this environment." + : "Choose a device to open."} +

+ + ) : null} + {hostReady && grouped.length > 0 ? ( +
+ {grouped.map((group) => ( +
+
+ +

{platformLabel(group.platform)}

+
+ + {group.devices.map((device) => ( + + + + } + title={device.name} + description={`${device.version} · ${device.booted ? "Running" : "Stopped"}`} + disabled={pendingDeviceKey !== null} + aria-label={`${device.booted ? "Open" : "Start"} ${device.name}`} + onClick={() => void selectDevice(deviceKey(device))} + action={ + pendingDeviceKey === deviceKey(device) ? ( + + ) : ( + + {device.booted ? "Open" : "Start"} + + ) + } + /> + ))} + +
+ ))} +
+ ) : null} + {hostReady && + !state.devices.some((device) => device.platform === "android") && + !unavailablePlatforms.some((platform) => platform.platform === "android") ? ( +

+ No Android virtual devices found. Create one in Android Studio's Device Manager, + then refresh. +

+ ) : null} + {loaded && !hostBusy ? ( + + ) : null} + {unavailablePlatforms.length > 0 && hostReady ? ( +
    + {unavailablePlatforms.map((platform) => ( +
  • + {platform.platform === "ios" ? "iOS" : "Android"}: {platform.reason} +
    • + ))} +
+ ) : null} +
+
+ )} +
+ + ); +} + +function DeviceButton(props: { + readonly label: string; + readonly onClick: () => void; + readonly disabled?: boolean; + readonly children: React.ReactNode; +}) { + return ( + + + } + > + {props.children} + + {props.label} + + ); +} + +function groupDevices(state: DeviceServiceState) { + const groups: Array<{ platform: DevicePlatform; devices: DeviceSummary[] }> = []; + for (const platform of ["ios", "android"] as const) { + const devices = state.devices + .filter((device) => device.platform === platform) + .toSorted((a, b) => Number(b.booted) - Number(a.booted) || a.name.localeCompare(b.name)); + if (devices.length > 0) groups.push({ platform, devices }); + } + return groups; +} diff --git a/apps/web/src/components/device/DeviceSetup.tsx b/apps/web/src/components/device/DeviceSetup.tsx new file mode 100644 index 000000000000..d203fe0ed965 --- /dev/null +++ b/apps/web/src/components/device/DeviceSetup.tsx @@ -0,0 +1,307 @@ +import type { DevicePlatform, DeviceServiceState, EnvironmentId } from "@t3tools/contracts"; +import { Check, CircleAlert } from "lucide-react"; +import { useState } from "react"; + +import { Button } from "~/components/ui/button"; +import { DialogClose } from "~/components/ui/dialog"; +import { WizardHeader, WizardPanel, WizardSteps, WizardFooter } from "~/components/ui/wizard"; +import { Spinner } from "~/components/ui/spinner"; +import { Switch } from "~/components/ui/switch"; +import { deviceEnvironment } from "~/state/device"; +import { useAtomCommand } from "~/state/use-atom-command"; +import { cn } from "~/lib/utils"; + +const platformName = (platform: DevicePlatform) => (platform === "ios" ? "iOS" : "Android"); + +export const deviceHubDescription = + "Open simulators and emulators. Required device support is set up automatically."; +export const agentDeviceDescription = + "Allow agents to start and control devices. Required tools are set up automatically and available to new agent sessions."; + +export function platformSetupStatus(state: DeviceServiceState, platform: DevicePlatform) { + const availability = state.hosts + .flatMap((host) => host.platforms) + .find((candidate) => candidate.platform === platform); + if (!availability?.available) { + return { + ready: false, + message: availability?.reason ?? `${platformName(platform)} support was not detected.`, + }; + } + if ( + state.hostStatus === "ready" && + !state.devices.some((device) => device.platform === platform) + ) { + return { + ready: false, + message: + platform === "ios" + ? "Xcode is installed, but no iOS Simulator is available. Install a runtime in Xcode Settings → Components." + : "The Android SDK is installed, but no virtual device exists. Create one in Android Studio → Device Manager.", + }; + } + return { + ready: true, + message: + platform === "ios" + ? "Xcode and iOS Simulator are available." + : "The Android SDK and Emulator are available.", + }; +} + +export function DeviceSetup(props: { + readonly environmentId: EnvironmentId; + readonly state: DeviceServiceState; + readonly onComplete?: () => void; +}) { + const configure = useAtomCommand(deviceEnvironment.configure); + const list = useAtomCommand(deviceEnvironment.list, { reportFailure: false }); + const [pending, setPending] = useState<"hub" | "check" | "agent" | "complete" | null>(null); + const [step, setStep] = useState(0); + const enabled = props.state.hostStatus !== "disabled"; + const busy = props.state.hostStatus === "installing" || props.state.hostStatus === "starting"; + + const update = async ( + kind: NonNullable, + input: { enabled?: boolean; agentAccessEnabled?: boolean; onboardingCompleted?: boolean }, + ) => { + setPending(kind); + try { + const result = await configure({ environmentId: props.environmentId, input }); + if (kind === "complete" && result._tag === "Success") props.onComplete?.(); + } finally { + setPending(null); + } + }; + + return ( + <> + + busy || pending !== null || requested > step} + /> + + + + {step === 0 ? ( +
+

Enable the device hub

+
+

{deviceHubDescription}

+ + void update("hub", { + enabled: Boolean(checked), + ...(checked ? {} : { agentAccessEnabled: false }), + }) + } + /> +
+ +
+ ) : null} + + {step === 1 ? ( +
+

Check simulator support

+ { + setPending("check"); + void list({ environmentId: props.environmentId, input: {} }).finally(() => + setPending(null), + ); + }} + /> +
+ ) : null} + + {step === 2 ? ( +
+

Allow agent control

+
+

{agentDeviceDescription}

+ + void update("agent", { agentAccessEnabled: Boolean(checked) }) + } + /> +
+ +

+ Leave this off to keep manual device controls without giving agents access. +

+
+ ) : null} + {props.state.hostStatus === "failed" && props.state.hostStatusDetail ? ( +

+ {props.state.hostStatusDetail} +

+ ) : null} + + + + {step === 0 ? ( + }>Cancel + ) : ( + + )} + {step < 2 ? ( + + ) : ( + + )} + + + ); +} + +export function DeviceHubSetupStatus({ + state, + pending, + compact = false, +}: { + readonly state: DeviceServiceState; + readonly pending: boolean; + readonly compact?: boolean; +}) { + if (!pending && state.hostStatus !== "ready") return null; + return ( +

+ {pending ? : } + {pending + ? state.hostStatus === "installing" + ? compact + ? "Installing…" + : "Installing device hub…" + : state.hostStatus === "starting" + ? compact + ? "Starting…" + : "Starting device hub…" + : compact + ? "Updating…" + : "Updating device hub…" + : "Device hub is ready."} +

+ ); +} + +function DevicePlatformSetup(props: { + readonly state: DeviceServiceState; + readonly checking: boolean; + readonly disabled: boolean; + readonly onCheck: () => void; +}) { + return ( +
+ + +

+ You can use either platform. Fixing a missing platform does not block the other one. +

+ +
+ ); +} + +export function AgentDeviceSetupStatus(props: { + readonly state: DeviceServiceState; + readonly pending: boolean; + readonly compact?: boolean; +}) { + if (props.pending) { + const label = + props.state.hostStatus === "installing" + ? props.compact + ? "Installing…" + : "Installing agent tools…" + : props.state.hostStatus === "starting" + ? props.compact + ? "Starting…" + : "Starting agent tools…" + : props.compact + ? "Updating…" + : "Updating agent access…"; + return ( +

+ + {label} +

+ ); + } + if ( + props.state.agentAccessEnabled && + props.state.hostStatus === "ready" && + props.state.hosts.some((host) => host.agentDeviceInstalled) + ) { + return ( +

+ + Agent tools are ready. +

+ ); + } + return null; +} + +export function PlatformStatus(props: { + readonly platform: string; + readonly status: { readonly ready: boolean; readonly message: string }; + readonly compact?: boolean; +}) { + const Icon = props.status.ready ? Check : CircleAlert; + return ( +
+ +
+

{props.platform}

+

+ {props.compact && props.status.ready ? "Ready" : props.status.message} +

+
+
+ ); +} diff --git a/apps/web/src/components/device/DeviceStreamView.test.tsx b/apps/web/src/components/device/DeviceStreamView.test.tsx new file mode 100644 index 000000000000..a46e5ed7b7b8 --- /dev/null +++ b/apps/web/src/components/device/DeviceStreamView.test.tsx @@ -0,0 +1,58 @@ +import { act } from "react"; +import { create, type ReactTestRenderer } from "react-test-renderer"; +import { EnvironmentId } from "@t3tools/contracts"; +import { afterEach, expect, it, vi } from "vite-plus/test"; + +vi.mock("~/state/device", () => ({ + useDeviceHubAccess: () => access, + refreshDeviceHubAccess: vi.fn(), +})); +const access = { httpBase: "http://test", wsBase: "ws://test", query: {}, credentials: true }; +vi.mock("./deviceStream", () => ({ + createDeviceStreamClient: ( + _target: unknown, + _canvas: unknown, + events: { onMjpegFallback: (url: string) => void }, + ) => ({ + start: () => events.onMjpegFallback("http://test/stream.mjpeg"), + stop: vi.fn(), + }), +})); +import { DeviceStreamView } from "./DeviceStreamView"; +let renderer: ReactTestRenderer | undefined; +afterEach(async () => { + await act(async () => renderer?.unmount()); + vi.unstubAllGlobals(); +}); + +it("removes MJPEG requests while hidden and reconnects when shown", async () => { + vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true); + vi.stubGlobal( + "ResizeObserver", + class { + observe() {} + disconnect() {} + }, + ); + const view = (visible: boolean) => ( + + ); + await act(async () => { + renderer = create(view(true), { + createNodeMock: () => ({ + style: { setProperty() {} }, + getBoundingClientRect: () => ({ width: 400, height: 800 }), + }), + }); + }); + expect(renderer!.root.findAllByType("img")).toHaveLength(1); + await act(async () => renderer!.update(view(false))); + expect(renderer!.root.findAllByType("img")).toHaveLength(0); + await act(async () => renderer!.update(view(true))); + expect(renderer!.root.findAllByType("img")).toHaveLength(1); +}); diff --git a/apps/web/src/components/device/DeviceStreamView.tsx b/apps/web/src/components/device/DeviceStreamView.tsx new file mode 100644 index 000000000000..15fc96cbb2e7 --- /dev/null +++ b/apps/web/src/components/device/DeviceStreamView.tsx @@ -0,0 +1,327 @@ +import type { DevicePlatform, EnvironmentId } from "@t3tools/contracts"; +import { useEffect, useMemo, useRef, useState } from "react"; + +import { cn } from "~/lib/utils"; +import { refreshDeviceHubAccess, useDeviceHubAccess } from "~/state/device"; +import { Spinner } from "~/components/ui/spinner"; +import { type DeviceAxElement, fetchDeviceAxTree } from "./deviceHubApi"; +import { + createDeviceStreamClient, + type DeviceHardwareButton, + type DeviceScreenSize, + type DeviceStreamClient, + type DeviceStreamStatus, +} from "./deviceStream"; + +const AX_POLL_INTERVAL_MS = 2_000; + +export interface DeviceStreamHandle { + readonly pressButton: (button: DeviceHardwareButton) => void; + readonly rotate: () => void; + /** False while the input socket is down; controls should disable. */ + readonly inputConnected: boolean; +} + +/** + * The live device screen. Pointer events map onto normalized coordinates in + * the displayed frame and go to the device; keyboard input is forwarded while + * the surface is focused. `visible=false` tears the stream down so a hidden + * panel decodes nothing. + */ +export function DeviceStreamView(props: { + readonly environmentId: EnvironmentId; + readonly platform: DevicePlatform; + readonly deviceId: string; + readonly visible: boolean; + /** Draw accessibility element frames over the screen. */ + readonly axOverlay?: boolean; + readonly onHandle?: (handle: DeviceStreamHandle | null) => void; + readonly onScreen?: (screen: DeviceScreenSize | null) => void; +}) { + const access = useDeviceHubAccess(props.environmentId); + const canvasRef = useRef(null); + const clientRef = useRef(null); + const [status, setStatus] = useState("connecting"); + const [detail, setDetail] = useState(undefined); + const [screen, setScreen] = useState(null); + const [mjpegUrl, setMjpegUrl] = useState(null); + const [mjpegGeneration, setMjpegGeneration] = useState(0); + const [inputState, setInputState] = useState<{ connected: boolean; detail?: string }>({ + connected: false, + }); + const { onHandle, onScreen } = props; + + useEffect(() => { + const canvas = canvasRef.current; + if (!access || !canvas || !props.visible) { + setStatus("connecting"); + onHandle?.(null); + return; + } + const client = createDeviceStreamClient( + { platform: props.platform, deviceId: props.deviceId, access }, + canvas, + { + onStatus: (next, nextDetail) => { + setStatus(next); + setDetail(nextDetail); + }, + onScreen: (next) => { + setScreen(next); + onScreen?.(next); + }, + onUnauthorized: () => { + // A fresh ticket re-runs this effect through the access dependency. + refreshDeviceHubAccess(props.environmentId); + }, + onMjpegFallback: (url) => { + setMjpegUrl(url); + setMjpegGeneration((generation) => generation + 1); + }, + onInputConnected: (connected, detail) => { + setInputState({ connected, ...(detail ? { detail } : {}) }); + onHandle?.({ + pressButton: client.pressButton, + rotate: client.rotate, + inputConnected: connected, + }); + }, + }, + ); + clientRef.current = client; + setMjpegUrl(null); + setInputState({ connected: false }); + client.start(); + onHandle?.({ pressButton: client.pressButton, rotate: client.rotate, inputConnected: false }); + return () => { + client.stop(); + clientRef.current = null; + onHandle?.(null); + onScreen?.(null); + setScreen(null); + }; + }, [ + access, + onHandle, + onScreen, + props.deviceId, + props.environmentId, + props.platform, + props.visible, + ]); + + // Displayed aspect ratio (width / height) of the device as the user sees it. + const aspect = useMemo(() => { + if (!screen) return props.platform === "ios" ? 9 / 19.5 : 9 / 20; + const landscape = + screen.orientation === "landscape_left" || screen.orientation === "landscape_right"; + const w = landscape + ? Math.max(screen.width, screen.height) + : Math.min(screen.width, screen.height); + const h = landscape + ? Math.min(screen.width, screen.height) + : Math.max(screen.width, screen.height); + return w / h; + }, [props.platform, screen]); + + // The frame is the largest box at `aspect` that fits the container, so a + // narrow panel shows a shorter phone rather than a squeezed one. CSS + // `aspect-ratio` alone cannot do this: with the height pinned to 100% the + // width clamp wins and distorts the drawn frame. + const hostRef = useRef(null); + const [host, setHost] = useState({ width: 0, height: 0 }); + useEffect(() => { + const element = hostRef.current; + if (!element) return; + const update = () => { + const rect = element.getBoundingClientRect(); + setHost((current) => + current.width === rect.width && current.height === rect.height + ? current + : { width: rect.width, height: rect.height }, + ); + }; + update(); + const observer = new ResizeObserver(update); + observer.observe(element); + return () => observer.disconnect(); + }, []); + const frame = useMemo(() => { + if (host.width === 0 || host.height === 0) return { width: 0, height: 0 }; + const byHeight = { width: host.height * aspect, height: host.height }; + return byHeight.width <= host.width + ? byHeight + : { width: host.width, height: host.width / aspect }; + }, [aspect, host]); + + // serve-sim streams the raw framebuffer; rotate the display for a device + // that reports landscape while its frames stay portrait. + const rotation = useMemo(() => { + if (props.platform !== "ios" || !screen || screen.width > screen.height) return 0; + switch (screen.orientation) { + case "landscape_left": + return -90; + case "landscape_right": + return 90; + case "portrait_upside_down": + return 180; + default: + return 0; + } + }, [props.platform, screen]); + + // A sideways rotation draws the raw portrait frame into a landscape box: + // the media element takes the transposed size and is rotated about the + // box's center. + const sideways = rotation === 90 || rotation === -90; + const mediaStyle: React.CSSProperties = sideways + ? { + width: frame.height, + height: frame.width, + left: (frame.width - frame.height) / 2, + top: (frame.height - frame.width) / 2, + transform: `rotate(${rotation}deg)`, + } + : { + width: frame.width, + height: frame.height, + ...(rotation ? { transform: `rotate(${rotation}deg)` } : {}), + }; + + // The accessibility tree is polled while the overlay is on; each poll is + // one JSON fetch, so there is nothing to repaint between polls. + const [axElements, setAxElements] = useState>([]); + useEffect(() => { + if (!props.axOverlay || !access || !props.visible) return; + const target = { access, platform: props.platform, deviceId: props.deviceId }; + let controller: AbortController | null = null; + let timer: ReturnType | null = null; + let stopped = false; + const poll = async () => { + controller = new AbortController(); + try { + const tree = await fetchDeviceAxTree(target, controller.signal); + if (!stopped) setAxElements(tree.elements); + } catch { + // Keep the last good tree; the next poll retries. + } + if (!stopped) timer = setTimeout(() => void poll(), AX_POLL_INTERVAL_MS); + }; + void poll(); + return () => { + stopped = true; + controller?.abort(); + if (timer) clearTimeout(timer); + setAxElements([]); + }; + }, [access, props.axOverlay, props.deviceId, props.platform, props.visible]); + + const pointerActive = useRef(false); + const normalizedPoint = (event: React.PointerEvent) => { + const rect = event.currentTarget.getBoundingClientRect(); + const x = (event.clientX - rect.left) / rect.width; + const y = (event.clientY - rect.top) / rect.height; + return { x: Math.min(1, Math.max(0, x)), y: Math.min(1, Math.max(0, y)) }; + }; + + return ( +
{ + if (event.metaKey && !["r", "R"].includes(event.key)) return; + event.preventDefault(); + clientRef.current?.sendKey(event.nativeEvent, "down"); + }} + onKeyUp={(event) => { + clientRef.current?.sendKey(event.nativeEvent, "up"); + }} + > +
{ + event.currentTarget.setPointerCapture(event.pointerId); + (event.currentTarget.parentElement as HTMLElement | null)?.focus(); + pointerActive.current = true; + const { x, y } = normalizedPoint(event); + clientRef.current?.sendTouch("begin", x, y); + }} + onPointerMove={(event) => { + if (!pointerActive.current) return; + const { x, y } = normalizedPoint(event); + clientRef.current?.sendTouch("move", x, y); + }} + onPointerUp={(event) => { + if (!pointerActive.current) return; + pointerActive.current = false; + const { x, y } = normalizedPoint(event); + clientRef.current?.sendTouch("end", x, y); + }} + onPointerCancel={(event) => { + if (!pointerActive.current) return; + pointerActive.current = false; + const { x, y } = normalizedPoint(event); + clientRef.current?.sendTouch("end", x, y); + }} + > + + {props.visible && access && mjpegUrl ? ( + + ) : null} + {axElements.length > 0 ? ( +
+ {axElements.map((element) => ( +
+ {element.label ? ( + + {element.label} + + ) : null} +
+ ))} +
+ ) : null} +
+ {status === "streaming" && !inputState.connected ? ( +
+ + Input disconnected{inputState.detail ? ` (${inputState.detail})` : ""}, reconnecting… + +
+ ) : null} + {status !== "streaming" ? ( +
+ {status === "connecting" ? : null} + {status === "error" ? (detail ?? "Stream failed.") : "Connecting to device…"} + {status === "connecting" && detail ? ( + {detail} + ) : null} +
+ ) : null} +
+ ); +} diff --git a/apps/web/src/components/device/DeviceToolsPanel.tsx b/apps/web/src/components/device/DeviceToolsPanel.tsx new file mode 100644 index 000000000000..7f3984257ecd --- /dev/null +++ b/apps/web/src/components/device/DeviceToolsPanel.tsx @@ -0,0 +1,722 @@ +import type { DeviceHubAccess } from "@t3tools/client-runtime/state/deviceHubAccess"; +import type { + DeviceActionInput, + DeviceDetail, + DevicePermission, + DeviceSummary, + DeviceTextSize, + EnvironmentId, +} from "@t3tools/contracts"; +import { ChevronDown, X } from "lucide-react"; +import { useCallback, useEffect, useMemo, useState } from "react"; + +import { Button } from "~/components/ui/button"; +import { Collapsible, CollapsiblePanel, CollapsibleTrigger } from "~/components/ui/collapsible"; +import { Input } from "~/components/ui/input"; +import { + Select, + SelectItem, + SelectPopup, + SelectTrigger, + SelectValue, +} from "~/components/ui/select"; +import { Spinner } from "~/components/ui/spinner"; +import { Switch } from "~/components/ui/switch"; +import { Toggle, ToggleGroup } from "~/components/ui/toggle-group"; +import { cn } from "~/lib/utils"; +import { deviceEnvironment } from "~/state/device"; +import { formatEnvironmentQueryError } from "~/state/query"; +import { useAtomCommand } from "~/state/use-atom-command"; +import { + type DeviceEventLogEntry, + type DeviceForegroundInfo, + subscribeDeviceEventLog, + subscribeDeviceForeground, +} from "./deviceHubApi"; + +type ActionBody = DeviceActionInput extends infer A + ? A extends { readonly type: string } + ? Omit + : never + : never; + +const TEXT_SIZES: ReadonlyArray<{ value: DeviceTextSize; label: string }> = [ + { value: "small", label: "Small" }, + { value: "default", label: "Default" }, + { value: "large", label: "Large" }, + { value: "extra-large", label: "Extra large" }, +]; + +const COLOR_FILTERS = [ + { value: "none", label: "None" }, + { value: "grayscale", label: "Grayscale" }, + { value: "red-green", label: "Red / green (protanopia)" }, + { value: "green-red", label: "Green / red (deuteranopia)" }, + { value: "blue-yellow", label: "Blue / yellow (tritanopia)" }, +] as const; + +const ORIENTATIONS = [ + { value: "portrait", label: "Portrait" }, + { value: "landscape_left", label: "Landscape left" }, + { value: "portrait_upside_down", label: "Upside down" }, + { value: "landscape_right", label: "Landscape right" }, +] as const; + +const IOS_PERMISSIONS: ReadonlyArray<{ value: DevicePermission; label: string }> = [ + { value: "camera", label: "Camera" }, + { value: "microphone", label: "Microphone" }, + { value: "photos", label: "Photos" }, + { value: "contacts", label: "Contacts" }, + { value: "calendar", label: "Calendar" }, + { value: "reminders", label: "Reminders" }, + { value: "location", label: "Location" }, + { value: "notifications", label: "Notifications" }, + { value: "motion", label: "Motion" }, + { value: "media-library", label: "Media library" }, + { value: "faceid", label: "Face ID" }, +]; + +const ANDROID_PERMISSIONS: ReadonlyArray<{ value: DevicePermission; label: string }> = [ + { value: "camera", label: "Camera" }, + { value: "microphone", label: "Microphone" }, + { value: "photos", label: "Photos" }, + { value: "contacts", label: "Contacts" }, + { value: "calendar", label: "Calendar" }, + { value: "location", label: "Location" }, + { value: "notifications", label: "Notifications" }, + { value: "motion", label: "Physical activity" }, +]; + +const LOCATION_PRESETS = [ + { label: "San Francisco", latitude: 37.7749, longitude: -122.4194 }, + { label: "New York", latitude: 40.7128, longitude: -74.006 }, + { label: "London", latitude: 51.5074, longitude: -0.1278 }, + { label: "Stockholm", latitude: 59.3293, longitude: 18.0686 }, + { label: "Tokyo", latitude: 35.6762, longitude: 139.6503 }, +] as const; + +/** + * The Tools drawer for one open device: current settings read from the device, + * one control per supported action, and the read-only feeds the hub exposes. + * Every change is a `device.action` round trip; the returned detail replaces + * local state so the controls never show a value the device did not confirm. + */ +export function DeviceToolsPanel(props: { + readonly environmentId: EnvironmentId; + readonly device: DeviceSummary; + readonly access: DeviceHubAccess | null; + readonly axOverlay: boolean; + readonly onAxOverlayChange: (enabled: boolean) => void; + readonly onClose: () => void; + readonly className?: string; +}) { + const { environmentId, device } = props; + const readDetail = useAtomCommand(deviceEnvironment.detail, { reportFailure: false }); + const runAction = useAtomCommand(deviceEnvironment.action, { reportFailure: false }); + const [detail, setDetail] = useState(null); + const [pending, setPending] = useState(false); + const [error, setError] = useState(null); + const [foreground, setForeground] = useState(undefined); + const isIos = device.platform === "ios"; + + const target = useMemo( + () => ({ hostId: device.hostId, deviceId: device.id }), + [device.hostId, device.id], + ); + + // The panel is keyed by device, so a mount is always a fresh device. + useEffect(() => { + let cancelled = false; + void readDetail({ environmentId, input: target }).then((result) => { + if (cancelled) return; + if (result._tag === "Success") setDetail(result.value); + else setError(formatEnvironmentQueryError(result.cause)); + }); + return () => { + cancelled = true; + }; + }, [environmentId, readDetail, target]); + + useEffect(() => { + if (!props.access) return; + return subscribeDeviceForeground( + { access: props.access, platform: device.platform, deviceId: device.id }, + setForeground, + ); + }, [device.id, device.platform, props.access]); + + const act = useCallback( + async (body: ActionBody) => { + setPending(true); + setError(null); + try { + const result = await runAction({ + environmentId, + input: { ...target, ...body } as DeviceActionInput, + }); + if (result._tag === "Success") setDetail(result.value); + else setError(formatEnvironmentQueryError(result.cause)); + } finally { + setPending(false); + } + }, + [environmentId, runAction, target], + ); + + const settings = detail?.settings; + const foregroundApp = foreground === undefined ? (detail?.foregroundApp ?? null) : foreground; + const disabled = pending || detail === null; + + return ( +
+
+ Tools + {pending ? : null} + +
+
+ {error ? ( +

{error}

+ ) : null} + {detail === null && !error ? ( +
+ Reading device settings… +
+ ) : null} + +
+ + {foregroundApp?.id ?? "—"} + + {foregroundApp ? ( +
+ + +
+ ) : null} + act({ type: "openUrl", url })} + /> + act({ type: "launchApp", appId })} + /> +
+ +
+ + { + const next = value[0]; + if (next === "light" || next === "dark") + void act({ type: "setAppearance", value: next }); + }} + > + Light + Dark + + + + act({ type: "setTextSize", value })} + /> + + {isIos ? ( + <> + + { + const next = value[0]; + if (next === "clear" || next === "tinted") { + void act({ type: "setLiquidGlass", value: next }); + } + }} + > + Clear + Tinted + + + + act({ type: "setColorFilter", value })} + /> + + + ) : ( + + act({ type: "setOrientation", value })} + /> + + )} + act({ type: "setToggle", setting: "reduceMotion", value })} + /> + {isIos ? ( + <> + act({ type: "setToggle", setting: "increaseContrast", value })} + /> + + act({ type: "setToggle", setting: "reduceTransparency", value }) + } + /> + act({ type: "setToggle", setting: "showBorders", value })} + /> + act({ type: "setToggle", setting: "voiceOver", value })} + /> + + ) : ( + act({ type: "setToggle", setting: "networkEnabled", value })} + /> + )} +
+ +
+ { + props.onAxOverlayChange(value); + return Promise.resolve(); + }} + /> +
+ + act({ type: "setLocation", latitude, longitude })} + onClear={() => act({ type: "clearLocation" })} + /> + + + act({ type: "setPermission", appId, permission, decision }) + } + /> + + {isIos ? ( +
+ + foregroundApp + ? act({ type: "sendPush", appId: foregroundApp.id, payload }) + : Promise.resolve() + } + /> + {!foregroundApp ? ( +

Open an app first.

+ ) : null} +
+ ) : null} + + {isIos && props.access ? : null} +
+
+ ); +} + +function Section(props: { readonly title: string; readonly children: React.ReactNode }) { + return ( +
+

+ {props.title} +

+ {props.children} +
+ ); +} + +function Row(props: { readonly label: string; readonly children: React.ReactNode }) { + return ( +
+ {props.label} +
{props.children}
+
+ ); +} + +function SwitchRow(props: { + readonly label: string; + readonly checked: boolean | undefined; + readonly disabled: boolean; + readonly onChange: (value: boolean) => Promise; +}) { + return ( + + void props.onChange(checked)} + /> + + ); +} + +function ChoiceSelect(props: { + readonly ariaLabel: string; + readonly value: V | null; + readonly options: ReadonlyArray<{ readonly value: V; readonly label: string }>; + readonly disabled: boolean; + readonly placeholder?: string; + readonly onChange: (value: V) => Promise; +}) { + const current = props.options.find((option) => option.value === props.value); + return ( + + ); +} + +function SubmitRow(props: { + readonly placeholder: string; + readonly action: string; + readonly disabled: boolean; + readonly onSubmit: (value: string) => Promise; +}) { + const [value, setValue] = useState(""); + const submit = () => { + const trimmed = value.trim(); + if (!trimmed) return; + void props.onSubmit(trimmed).then(() => setValue("")); + }; + return ( +
{ + event.preventDefault(); + submit(); + }} + > + setValue(event.target.value)} + /> + +
+ ); +} + +function LocationSection(props: { + readonly disabled: boolean; + readonly canClear: boolean; + readonly onSet: (latitude: number, longitude: number) => Promise; + readonly onClear: () => Promise; +}) { + const [latitude, setLatitude] = useState(""); + const [longitude, setLongitude] = useState(""); + const parsed = { latitude: Number(latitude), longitude: Number(longitude) }; + const valid = + latitude.trim() !== "" && + longitude.trim() !== "" && + Math.abs(parsed.latitude) <= 90 && + Math.abs(parsed.longitude) <= 180; + return ( +
+
+ setLatitude(event.target.value)} + /> + setLongitude(event.target.value)} + /> +
+
+ + value={null} + disabled={props.disabled} + onValueChange={(value) => { + const preset = LOCATION_PRESETS.find((candidate) => candidate.label === value); + if (!preset) return; + setLatitude(String(preset.latitude)); + setLongitude(String(preset.longitude)); + void props.onSet(preset.latitude, preset.longitude); + }} + > + + + Preset… + + + + {LOCATION_PRESETS.map((preset) => ( + + {preset.label} + + ))} + + + + {props.canClear ? ( + + ) : null} +
+
+ ); +} + +function PermissionsSection(props: { + readonly permissions: ReadonlyArray<{ value: DevicePermission; label: string }>; + readonly canReset: boolean; + readonly defaultAppId: string; + readonly disabled: boolean; + readonly onDecide: ( + appId: string, + permission: DevicePermission, + decision: "grant" | "revoke" | "reset", + ) => Promise; +}) { + const [appId, setAppId] = useState(""); + const [permission, setPermission] = useState("camera"); + const resolvedAppId = appId.trim() || props.defaultAppId; + const decide = (decision: "grant" | "revoke" | "reset") => + void props.onDecide(resolvedAppId, permission, decision); + return ( +
+ setAppId(event.target.value)} + /> +
+ { + setPermission(value); + return Promise.resolve(); + }} + /> + + + {props.canReset ? ( + + ) : null} +
+
+ ); +} + +const EVENT_LOG_LIMIT = 100; + +function EventLogSection(props: { + readonly access: DeviceHubAccess; + readonly device: DeviceSummary; +}) { + const [open, setOpen] = useState(false); + const [entries, setEntries] = useState>([]); + + useEffect(() => { + if (!open) return; + const unsubscribe = subscribeDeviceEventLog( + { access: props.access, platform: props.device.platform, deviceId: props.device.id }, + (incoming, reset) => { + setEntries((current) => { + const merged = reset ? [...incoming] : [...current, ...incoming]; + return merged.length > EVENT_LOG_LIMIT ? merged.slice(-EVENT_LOG_LIMIT) : merged; + }); + }, + ); + return () => { + unsubscribe(); + setEntries([]); + }; + }, [open, props.access, props.device.id, props.device.platform]); + + return ( + + + Event log + + + +
    + {entries.length === 0 ? ( +
  1. No events yet.
    2. + ) : ( + entries.map((entry) => ( +
  2. + + {entry.timestamp.slice(11, 19)} + + {entry.summary} +
    3. + )) + )} +
+ + + ); +} diff --git a/apps/web/src/components/device/deviceHubApi.test.ts b/apps/web/src/components/device/deviceHubApi.test.ts new file mode 100644 index 000000000000..52575f4a4224 --- /dev/null +++ b/apps/web/src/components/device/deviceHubApi.test.ts @@ -0,0 +1,44 @@ +import { afterEach, describe, expect, it, vi } from "vite-plus/test"; +import { subscribeDeviceForeground } from "./deviceHubApi"; + +describe("foreground app events", () => { + afterEach(() => vi.unstubAllGlobals()); + + it("clears the last app when it exits and ignores malformed events", () => { + let source: FakeEventSource; + class FakeEventSource { + onmessage: ((event: { data: string }) => void) | null = null; + addEventListener(_type: string, listener: (event: { data: string }) => void) { + this.onmessage = listener; + } + close = vi.fn(); + constructor() { + source = this; + } + } + vi.stubGlobal("EventSource", FakeEventSource); + const onChange = vi.fn(); + const stop = subscribeDeviceForeground( + { + platform: "ios", + deviceId: "test", + access: { httpBase: "http://test", wsBase: "ws://test", query: {}, credentials: true }, + }, + onChange, + ); + const emit = (data: unknown) => source.onmessage?.({ data: JSON.stringify(data) }); + emit({ bundleId: "com.example.app", pid: 123 }); + emit({ bundleId: null }); + emit({ bundleId: "com.example.other" }); + emit({ bundleId: "" }); + emit({ other: "not app state" }); + expect(onChange.mock.calls.map(([app]) => app)).toEqual([ + { id: "com.example.app", pid: 123 }, + null, + { id: "com.example.other" }, + null, + ]); + stop(); + expect(source!.close).toHaveBeenCalledOnce(); + }); +}); diff --git a/apps/web/src/components/device/deviceHubApi.ts b/apps/web/src/components/device/deviceHubApi.ts new file mode 100644 index 000000000000..dbcf4fa0015c --- /dev/null +++ b/apps/web/src/components/device/deviceHubApi.ts @@ -0,0 +1,248 @@ +import { withDeviceHubQuery } from "@t3tools/client-runtime/state/deviceHubAccess"; +import type { DeviceHubAccess } from "@t3tools/client-runtime/state/deviceHubAccess"; +import type { DevicePlatform } from "@t3tools/contracts"; + +/** + * Read-only hub endpoints the Tools drawer consumes directly: the accessibility + * tree, the foreground app, and the event log. Everything that changes device + * state goes through the `device.action` RPC instead, so this file never POSTs. + */ + +export interface DeviceAxElement { + readonly id: string; + readonly label: string; + readonly role: string; + /** Normalized to the displayed screen: 0..1 on both axes. */ + readonly x: number; + readonly y: number; + readonly width: number; + readonly height: number; +} + +export interface DeviceAxTree { + readonly elements: ReadonlyArray; + readonly errors: ReadonlyArray; +} + +export interface DeviceEventLogEntry { + readonly id: number; + readonly timestamp: string; + readonly kind: string; + readonly summary: string; +} + +export interface DeviceForegroundInfo { + readonly id: string; + readonly label?: string; + readonly pid?: number; + readonly isReactNative?: boolean; +} + +interface Target { + readonly access: DeviceHubAccess; + readonly platform: DevicePlatform; + readonly deviceId: string; +} + +const vendorBase = (target: Target) => + `${target.access.httpBase}${target.platform === "ios" ? "/vendor/serve-sim" : "/vendor/serve-emu"}`; + +const hubUrl = (target: Target, path: string, params?: Record) => { + const search = params ? `?${new URLSearchParams(params).toString()}` : ""; + return withDeviceHubQuery(`${vendorBase(target)}${path}${search}`, target.access); +}; + +const fetchJson = async (target: Target, url: string, signal?: AbortSignal): Promise => { + const response = await fetch(url, { + cache: "no-store", + credentials: target.access.credentials ? "include" : "same-origin", + ...(signal ? { signal } : {}), + }); + if (!response.ok) throw new Error(`${response.status} ${response.statusText}`); + return response.json(); +}; + +const isRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null; + +const numberOr = (value: unknown, fallback: number) => + typeof value === "number" && Number.isFinite(value) ? value : fallback; + +const AX_ELEMENT_LIMIT = 500; + +/** + * serve-sim's helper returns the native nested tree; the root node is the + * application covering the whole screen. Flatten it the way serve-sim's own + * overlay does: skip nodes with the root's frame, cap the count. + */ +const flattenIosAxTree = (roots: ReadonlyArray): ReadonlyArray => { + const first = roots[0]; + const rootFrame = isRecord(first) && isRecord(first.frame) ? first.frame : null; + const screenWidth = Math.max(1, numberOr(rootFrame?.width, 1)); + const screenHeight = Math.max(1, numberOr(rootFrame?.height, 1)); + const elements: DeviceAxElement[] = []; + const visit = (node: unknown, path: string) => { + if (elements.length >= AX_ELEMENT_LIMIT || !isRecord(node) || !isRecord(node.frame)) return; + const frame = node.frame; + const width = numberOr(frame.width, 0); + const height = numberOr(frame.height, 0); + const coversScreen = + Math.abs(width - screenWidth) < 0.5 && Math.abs(height - screenHeight) < 0.5; + if (!coversScreen && width > 0 && height > 0) { + elements.push({ + id: typeof node.AXUniqueId === "string" ? node.AXUniqueId : path, + label: typeof node.AXLabel === "string" ? node.AXLabel : "", + role: typeof node.type === "string" ? node.type : "", + x: numberOr(frame.x, 0) / screenWidth, + y: numberOr(frame.y, 0) / screenHeight, + width: width / screenWidth, + height: height / screenHeight, + }); + } + const children = Array.isArray(node.children) ? node.children : []; + children.forEach((child, index) => visit(child, `${path}.${index}`)); + }; + roots.forEach((root, index) => visit(root, String(index))); + return elements; +}; + +export async function fetchDeviceAxTree( + target: Target, + signal?: AbortSignal, +): Promise { + if (target.platform === "ios") { + const payload = await fetchJson( + target, + hubUrl(target, `/helper/${encodeURIComponent(target.deviceId)}/ax`), + signal, + ); + if (!Array.isArray(payload)) { + const error = isRecord(payload) && typeof payload.error === "string" ? payload.error : null; + return { elements: [], errors: [error ?? "Unexpected accessibility payload."] }; + } + return { elements: flattenIosAxTree(payload), errors: [] }; + } + const payload = await fetchJson( + target, + hubUrl(target, "/api/accessibility", { device: target.deviceId }), + signal, + ); + if (!isRecord(payload) || !Array.isArray(payload.nodes)) { + const error = isRecord(payload) && typeof payload.error === "string" ? payload.error : null; + return { elements: [], errors: [error ?? "Unexpected accessibility payload."] }; + } + // uiautomator reports pixel bounds; the first node is the full window. + const nodes = payload.nodes.filter( + (node): node is Record => isRecord(node) && isRecord(node.bounds), + ); + const root = nodes[0]?.bounds as Record | undefined; + const screenWidth = Math.max(1, numberOr(root?.right, 1)); + const screenHeight = Math.max(1, numberOr(root?.bottom, 1)); + // Layout containers span the whole window and would tint the entire + // screen; only nodes a user could point at are worth drawing. + const elements = nodes.slice(1).flatMap((node): DeviceAxElement[] => { + const bounds = node.bounds as Record; + const left = numberOr(bounds.left, 0); + const top = numberOr(bounds.top, 0); + const width = (numberOr(bounds.right, left) - left) / screenWidth; + const height = (numberOr(bounds.bottom, top) - top) / screenHeight; + const text = typeof node.text === "string" ? node.text : ""; + const description = typeof node.contentDescription === "string" ? node.contentDescription : ""; + const label = text || description; + if (width >= 0.95 && height >= 0.9) return []; + if (!label && node.clickable !== true) return []; + const className = typeof node.className === "string" ? node.className : ""; + return [ + { + id: String(node.id ?? ""), + label, + role: className.split(".").at(-1) ?? "", + x: left / screenWidth, + y: top / screenHeight, + width, + height, + }, + ]; + }); + return { elements, errors: [] }; +} + +const openEventSource = ( + target: Target, + url: string, + onMessage: (data: unknown) => void, +): (() => void) => { + const source = new EventSource(url, { withCredentials: target.access.credentials }); + source.addEventListener("message", (event) => { + try { + onMessage(JSON.parse(String(event.data))); + } catch { + // Keep-alive comments and malformed frames carry nothing to render. + } + }); + return () => source.close(); +}; + +/** iOS only: the frontmost app, pushed by serve-sim whenever it changes. */ +export function subscribeDeviceForeground( + target: Target, + onChange: (app: DeviceForegroundInfo | null) => void, +): () => void { + if (target.platform !== "ios") return () => {}; + return openEventSource( + target, + hubUrl(target, "/appstate", { device: target.deviceId }), + (data) => { + if (!isRecord(data)) return; + if (data.bundleId === null || data.bundleId === "") { + onChange(null); + return; + } + if (typeof data.bundleId !== "string") return; + onChange({ + id: data.bundleId, + ...(typeof data.pid === "number" ? { pid: data.pid } : {}), + ...(typeof data.isReactNative === "boolean" ? { isReactNative: data.isReactNative } : {}), + }); + }, + ); +} + +const toEventLogEntry = (raw: unknown): DeviceEventLogEntry | null => { + if (!isRecord(raw) || typeof raw.id !== "number") return null; + return { + id: raw.id, + timestamp: typeof raw.timestamp === "string" ? raw.timestamp : "", + kind: typeof raw.kind === "string" ? raw.kind : "", + summary: + typeof raw.summary === "string" ? raw.summary : typeof raw.msg === "string" ? raw.msg : "", + }; +}; + +/** + * iOS only: serve-sim's event log, seeded with recent history and then pushed + * live. Android's session recorder only tracks replayable gestures, which the + * user already sees themselves, so it is not surfaced. + */ +export function subscribeDeviceEventLog( + target: Target, + onEvents: (entries: ReadonlyArray, reset: boolean) => void, +): () => void { + if (target.platform !== "ios") return () => {}; + return openEventSource( + target, + hubUrl(target, "/api/event-log/events", { device: target.deviceId, limit: "100" }), + (data) => { + if (!isRecord(data)) return; + if (Array.isArray(data.events)) { + onEvents( + data.events.flatMap((raw) => toEventLogEntry(raw) ?? []), + true, + ); + return; + } + const entry = toEventLogEntry(data.event); + if (entry) onEvents([entry], false); + }, + ); +} diff --git a/apps/web/src/components/device/deviceStream.test.ts b/apps/web/src/components/device/deviceStream.test.ts new file mode 100644 index 000000000000..0ce7ea041bbb --- /dev/null +++ b/apps/web/src/components/device/deviceStream.test.ts @@ -0,0 +1,179 @@ +import { afterEach, describe, expect, it, vi } from "vite-plus/test"; + +import { + createDeviceStreamClient, + AvccDemuxer, + avcCodecString, + parseSemuPacket, + scanAccessUnit, +} from "./deviceStream"; + +const envelope = (tag: number, payload: number[]) => { + const length = 1 + payload.length; + return [ + (length >>> 24) & 0xff, + (length >>> 16) & 0xff, + (length >>> 8) & 0xff, + length & 0xff, + tag, + ...payload, + ]; +}; + +describe("AvccDemuxer", () => { + it("reassembles envelopes split across reads", () => { + const demuxer = new AvccDemuxer(); + const bytes = new Uint8Array([ + ...envelope(1, [1, 0x64, 0x00, 0x1f]), + ...envelope(2, [9, 9, 9]), + ...envelope(0x7f, [0]), + ...envelope(3, [4]), + ]); + const first = demuxer.push(bytes.subarray(0, 7)); + const rest = demuxer.push(bytes.subarray(7)); + const chunks = [...first, ...rest]; + expect(chunks.map((chunk) => chunk.type)).toEqual(["description", "keyframe", "delta"]); + expect(Array.from(chunks[0]!.payload)).toEqual([1, 0x64, 0x00, 0x1f]); + expect(Array.from(chunks[1]!.payload)).toEqual([9, 9, 9]); + }); + + it("derives the WebCodecs codec string from the avcC record", () => { + expect(avcCodecString(new Uint8Array([1, 0x64, 0x00, 0x1f]))).toBe("avc1.64001f"); + expect(avcCodecString(new Uint8Array([1]))).toBe("avc1.42E01E"); + }); +}); + +describe("serve-emu frames", () => { + it("strips the SEMU header and reads the keyframe flag and timestamp", () => { + const buffer = new ArrayBuffer(16 + 3); + const view = new DataView(buffer); + view.setUint32(0, 0x53454d55); + view.setUint8(4, 1); + view.setUint8(5, 1); + view.setBigUint64(8, 123456n); + new Uint8Array(buffer).set([7, 8, 9], 16); + const packet = parseSemuPacket(buffer); + expect(packet.isKey).toBe(true); + expect(packet.timestamp).toBe(123456); + expect(Array.from(packet.data)).toEqual([7, 8, 9]); + }); + + it("treats a frame without the header as raw data", () => { + const packet = parseSemuPacket(new Uint8Array([0, 0, 1, 0x65]).buffer); + expect(packet.isKey).toBeNull(); + expect(packet.data.length).toBe(4); + }); + + it("finds the SPS and IDR NAL units in an Annex-B access unit", () => { + const unit = new Uint8Array([0, 0, 0, 1, 0x67, 0x64, 0x00, 0x1f, 0, 0, 1, 0x65, 0xaa]); + const scanned = scanAccessUnit(unit); + expect(scanned.isKey).toBe(true); + expect(scanned.sps && avcCodecString(scanned.sps)).toBe("avc1.64001f"); + expect(scanAccessUnit(new Uint8Array([0, 0, 1, 0x41, 0x00])).isKey).toBe(false); + }); +}); + +describe("iOS input startup", () => { + afterEach(() => { + vi.useRealTimers(); + vi.unstubAllGlobals(); + }); + + const setup = () => { + vi.useFakeTimers(); + const sockets: FakeSocket[] = []; + class FakeSocket { + static OPEN = 1; + readyState = 1; + binaryType = ""; + onopen: (() => void) | null = null; + onmessage: ((event: { data: ArrayBuffer }) => void) | null = null; + onclose: ((event: { code: number; reason: string }) => void) | null = null; + send = vi.fn(); + close = vi.fn(); + constructor() { + sockets.push(this); + } + } + vi.stubGlobal("WebSocket", FakeSocket); + const signals: AbortSignal[] = []; + vi.stubGlobal( + "fetch", + vi.fn((_url, init: RequestInit) => { + const signal = init.signal!; + signals.push(signal); + return Promise.resolve( + new Response( + new ReadableStream({ + start(controller) { + signal.addEventListener("abort", () => controller.error(new Error("aborted"))); + }, + }), + ), + ); + }), + ); + const client = createDeviceStreamClient( + { + platform: "ios", + deviceId: "test-device", + access: { + httpBase: "http://test/api/device-hub", + wsBase: "ws://test/api/device-hub", + credentials: true, + query: {}, + }, + }, + { getContext: () => null } as unknown as HTMLCanvasElement, + { + onStatus: vi.fn(), + onScreen: vi.fn(), + onUnauthorized: vi.fn(), + onMjpegFallback: vi.fn(), + onInputConnected: vi.fn(), + }, + ); + return { client, sockets, signals }; + }; + + it("connects input when the MJPEG prime never produces a frame", async () => { + const { client, sockets, signals } = setup(); + client.start(); + await vi.advanceTimersByTimeAsync(2_000); + expect(signals[0]?.aborted).toBe(true); + expect(sockets).toHaveLength(1); + client.stop(); + }); + + it("aborts priming immediately when hidden without opening a socket later", async () => { + const { client, sockets, signals } = setup(); + client.start(); + await vi.advanceTimersByTimeAsync(0); + client.stop(); + expect(signals[0]?.aborted).toBe(true); + await vi.advanceTimersByTimeAsync(2_000); + expect(sockets).toHaveLength(0); + }); + + it("maps displayed landscape touches to raw iOS coordinates once", async () => { + const { client, sockets } = setup(); + client.start(); + await vi.advanceTimersByTimeAsync(2_000); + const socket = sockets[0]!; + const json = new TextEncoder().encode( + JSON.stringify({ width: 400, height: 800, orientation: "landscape_left" }), + ); + const packet = new Uint8Array(1 + json.length); + packet[0] = 0x82; + packet.set(json, 1); + socket.onmessage?.({ data: packet.buffer }); + client.sendTouch("begin", 0.2, 0.7); + const sent = socket.send.mock.calls[0]![0] as Uint8Array; + expect(JSON.parse(new TextDecoder().decode(sent.subarray(1)))).toEqual({ + type: "begin", + x: 1 - 0.7, + y: 0.2, + }); + client.stop(); + }); +}); diff --git a/apps/web/src/components/device/deviceStream.ts b/apps/web/src/components/device/deviceStream.ts new file mode 100644 index 000000000000..8a79eb09e137 --- /dev/null +++ b/apps/web/src/components/device/deviceStream.ts @@ -0,0 +1,697 @@ +/** + * Framework-free client for expo-device-hub's per-device streams, reached + * through the T3 proxy. One class handles both platforms because the hub + * vendors two servers with different wire formats: + * + * - iOS (serve-sim): video is an HTTP `stream.avcc` body of length-prefixed + * envelopes (`u32be length, u8 tag, payload`; tag 1 avcC description, + * 2 keyframe, 3 delta, 4 JPEG seed) decoded with WebCodecs; input goes over + * `helper/ws?device=` as `[tag][json]` packets. When WebCodecs is + * unavailable (plain-http remote origins) the MJPEG endpoint is used as an + * `` source instead. + * - Android (serve-emu): one WebSocket at `ws?device=&frame-meta=1` + * carries H.264 access units prefixed with a 16-byte "SEMU" header + * (magic, version, key flag, pts) and accepts JSON gestures upstream. + * + * The decoder only runs while frames arrive and the viewer is attached; a + * hidden panel calls `stop()` so an idle device costs nothing on the GPU. + */ +import type { DeviceHubAccess } from "@t3tools/client-runtime/state/deviceHubAccess"; +import { withDeviceHubQuery } from "@t3tools/client-runtime/state/deviceHubAccess"; +import type { DevicePlatform } from "@t3tools/contracts"; + +export type DeviceStreamStatus = "connecting" | "streaming" | "error"; + +export interface DeviceScreenSize { + readonly width: number; + readonly height: number; + readonly orientation: "portrait" | "portrait_upside_down" | "landscape_left" | "landscape_right"; +} + +export interface DeviceStreamEvents { + readonly onStatus: (status: DeviceStreamStatus, detail?: string) => void; + readonly onScreen: (screen: DeviceScreenSize) => void; + /** The proxy rejected the credential; the owner should refresh access and reconnect. */ + readonly onUnauthorized: () => void; + /** + * H.264 cannot be decoded here (no WebCodecs, or the simulator's profile is + * unsupported); the owner should show this MJPEG URL in an `` instead of + * the canvas. + */ + readonly onMjpegFallback: (url: string) => void; + /** Whether touches and keys can currently reach the device. */ + readonly onInputConnected: (connected: boolean, detail?: string) => void; +} + +export interface DeviceStreamTarget { + readonly platform: DevicePlatform; + readonly deviceId: string; + readonly access: DeviceHubAccess; +} + +export type DeviceHardwareButton = "home" | "back" | "recents" | "power" | "appSwitcher"; + +const RETRY_DELAY_MS = 1_000; +const FRAME_DURATION_US = 16_667; +const SEMU_MAGIC = 0x53454d55; +const SEMU_HEADER_BYTES = 16; +const SEMU_FLAG_KEY = 1; +const SOFT_DECODE_QUEUE = 8; + +// serve-sim binary WS message tags (browser -> helper). +const IOS_MSG_TOUCH = 0x03; +const IOS_MSG_BUTTON = 0x04; +const IOS_MSG_KEY = 0x06; +const IOS_MSG_ORIENTATION = 0x07; +const IOS_MSG_HARDWARE_KEYBOARD = 0x0d; +// helper -> browser. +const IOS_TAG_SCREEN_CONFIG = 0x82; + +const encoder = new TextEncoder(); +const decoder = new TextDecoder(); + +const isWebCodecsSupported = (): boolean => + typeof globalThis !== "undefined" && + "VideoDecoder" in globalThis && + "EncodedVideoChunk" in globalThis; + +function taggedJson(tag: number, payload: unknown): Uint8Array { + const json = encoder.encode(JSON.stringify(payload)); + const out = new Uint8Array(1 + json.length); + out[0] = tag; + out.set(json, 1); + return out; +} + +/** Build the WebCodecs `avc1.PPCCLL` string from an avcC record or an SPS NAL. */ +export function avcCodecString(bytes: Uint8Array): string { + if (bytes.length < 4) return "avc1.42E01E"; + const hex = (byte: number) => byte.toString(16).padStart(2, "0"); + return `avc1.${hex(bytes[1]!)}${hex(bytes[2]!)}${hex(bytes[3]!)}`; +} + +/** Split serve-emu's SEMU-framed message into metadata and the Annex-B payload. */ +export function parseSemuPacket(raw: ArrayBuffer): { + readonly data: Uint8Array; + readonly isKey: boolean | null; + readonly timestamp: number | null; +} { + const bytes = new Uint8Array(raw); + if (bytes.byteLength > SEMU_HEADER_BYTES) { + const view = new DataView(raw, 0, SEMU_HEADER_BYTES); + if (view.getUint32(0, false) === SEMU_MAGIC && view.getUint8(4) === 1) { + const pts = view.getBigUint64(8, false); + return { + data: bytes.subarray(SEMU_HEADER_BYTES), + isKey: (view.getUint8(5) & SEMU_FLAG_KEY) !== 0, + timestamp: pts <= BigInt(Number.MAX_SAFE_INTEGER) ? Number(pts) : null, + }; + } + } + return { data: bytes, isKey: null, timestamp: null }; +} + +const isVideoSessionMessage = (text: string) => { + try { + const message = JSON.parse(text) as { type?: unknown }; + return message.type === "video-session"; + } catch { + return false; + } +}; + +/** Walk an Annex-B access unit for its keyframe flag and SPS bytes. */ +export function scanAccessUnit(buf: Uint8Array): { isKey: boolean; sps: Uint8Array | null } { + let isKey = false; + let sps: Uint8Array | null = null; + const len = buf.length; + let i = 0; + while (i + 2 < len) { + if (buf[i] === 0 && buf[i + 1] === 0) { + let codeLen = 0; + if (buf[i + 2] === 1) codeLen = 3; + else if (i + 3 < len && buf[i + 2] === 0 && buf[i + 3] === 1) codeLen = 4; + if (codeLen) { + const nalType = buf[i + codeLen]! & 0x1f; + if (nalType === 7 && !sps) sps = buf.subarray(i + codeLen); + if (nalType === 5) isKey = true; + i += codeLen + 1; + continue; + } + } + i++; + } + return { isKey, sps }; +} + +export type AvccChunk = { + readonly type: "description" | "keyframe" | "delta" | "seed"; + readonly payload: Uint8Array; +}; + +const AVCC_TAGS: Record = { + 1: "description", + 2: "keyframe", + 3: "delta", + 4: "seed", +}; + +/** Turns a fragmented AVCC byte stream into complete envelopes. */ +export class AvccDemuxer { + private buffer = new Uint8Array(64 * 1024); + private length = 0; + + push(bytes: Uint8Array): AvccChunk[] { + if (this.length + bytes.length > this.buffer.length) { + let capacity = this.buffer.length; + while (capacity < this.length + bytes.length) capacity *= 2; + const grown = new Uint8Array(capacity); + grown.set(this.buffer.subarray(0, this.length)); + this.buffer = grown; + } + this.buffer.set(bytes, this.length); + this.length += bytes.length; + + const chunks: AvccChunk[] = []; + let offset = 0; + while (this.length - offset >= 4) { + const view = new DataView(this.buffer.buffer, this.buffer.byteOffset + offset, 4); + const frameLength = view.getUint32(0, false); + if (this.length - offset - 4 < frameLength) break; + if (frameLength >= 1) { + const type = AVCC_TAGS[this.buffer[offset + 4]!]; + if (type) { + chunks.push({ type, payload: this.buffer.slice(offset + 5, offset + 4 + frameLength) }); + } + } + offset += 4 + frameLength; + } + if (offset > 0) { + this.buffer.copyWithin(0, offset, this.length); + this.length -= offset; + } + return chunks; + } + + reset(): void { + this.length = 0; + } +} + +export interface DeviceStreamClient { + readonly start: () => void; + readonly stop: () => void; + /** Normalized 0..1 coordinates in the displayed frame. */ + readonly sendTouch: (phase: "begin" | "move" | "end", x: number, y: number) => void; + readonly sendKey: (event: KeyboardEvent, phase: "down" | "up") => void; + readonly pressButton: (button: DeviceHardwareButton) => void; + readonly rotate: () => void; +} + +const HID_USAGE_BY_CODE: Readonly> = { + Enter: 0x28, + Escape: 0x29, + Backspace: 0x2a, + Tab: 0x2b, + Space: 0x2c, + Minus: 0x2d, + Equal: 0x2e, + BracketLeft: 0x2f, + BracketRight: 0x30, + Backslash: 0x31, + Semicolon: 0x33, + Quote: 0x34, + Backquote: 0x35, + Comma: 0x36, + Period: 0x37, + Slash: 0x38, + Delete: 0x4c, + ArrowRight: 0x4f, + ArrowLeft: 0x50, + ArrowDown: 0x51, + ArrowUp: 0x52, + ControlLeft: 0xe0, + ShiftLeft: 0xe1, + AltLeft: 0xe2, + MetaLeft: 0xe3, + ControlRight: 0xe4, + ShiftRight: 0xe5, + AltRight: 0xe6, + MetaRight: 0xe7, +}; + +function hidUsageForCode(code: string): number | null { + if (/^Key[A-Z]$/.test(code)) return 0x04 + (code.charCodeAt(3) - 65); + if (/^Digit[1-9]$/.test(code)) return 0x1e + (code.charCodeAt(5) - 49); + if (code === "Digit0") return 0x27; + return HID_USAGE_BY_CODE[code] ?? null; +} + +const ANDROID_KEYCODE_BY_KEY: Readonly> = { + ArrowUp: 19, + ArrowDown: 20, + ArrowLeft: 21, + ArrowRight: 22, + Tab: 61, + Enter: 66, + Backspace: 67, + Delete: 112, + Home: 122, + End: 123, + PageUp: 92, + PageDown: 93, +}; + +const IOS_ORIENTATIONS: ReadonlyArray = [ + "portrait", + "landscape_left", + "portrait_upside_down", + "landscape_right", +]; + +export function createDeviceStreamClient( + target: DeviceStreamTarget, + canvas: HTMLCanvasElement, + events: DeviceStreamEvents, +): DeviceStreamClient { + const { access, platform, deviceId } = target; + const vendor = platform === "ios" ? "/vendor/serve-sim" : "/vendor/serve-emu"; + const device = encodeURIComponent(deviceId); + const httpUrl = (path: string) => + withDeviceHubQuery(`${access.httpBase}${vendor}${path}`, access); + const wsUrl = (path: string) => withDeviceHubQuery(`${access.wsBase}${vendor}${path}`, access); + const useWebCodecs = isWebCodecsSupported(); + + let stopped = true; + let socket: WebSocket | null = null; + let controller: AbortController | null = null; + const retryTimers = new Map<"video" | "input", ReturnType>(); + let primeController: AbortController | null = null; + let videoDecoder: VideoDecoder | null = null; + let timestamp = 0; + let awaitingKeyframe = true; + let screen: DeviceScreenSize | null = null; + let firstFrame = false; + let configuring = false; + let mjpeg = false; + + const mjpegUrl = () => httpUrl(`/helper/${device}/stream.mjpeg`); + + const fallBackToMjpeg = () => { + if (stopped || mjpeg) return; + mjpeg = true; + closeDecoder(); + events.onMjpegFallback(mjpegUrl()); + setStatus("streaming"); + }; + + const setStatus = (status: DeviceStreamStatus, detail?: string) => { + if (!stopped) events.onStatus(status, detail); + }; + + const paint = (source: CanvasImageSource, width: number, height: number) => { + if (stopped) return; + if (canvas.width !== width || canvas.height !== height) { + canvas.width = width; + canvas.height = height; + if (platform === "android") { + screen = { width, height, orientation: width > height ? "landscape_left" : "portrait" }; + events.onScreen(screen); + } + } + canvas.getContext("2d")?.drawImage(source, 0, 0, width, height); + if (!firstFrame) { + firstFrame = true; + setStatus("streaming"); + } + }; + + const closeDecoder = () => { + try { + videoDecoder?.close(); + } catch { + // Already closed. + } + videoDecoder = null; + awaitingKeyframe = true; + }; + + const makeDecoder = () => + new VideoDecoder({ + output: (frame) => { + try { + paint(frame, frame.displayWidth, frame.displayHeight); + } finally { + frame.close(); + } + }, + error: () => { + closeDecoder(); + requestKeyframe(); + }, + }); + + /** + * Resolves false when this browser cannot decode the stream's profile + * (simulators encode High 5.1, which headless and some hardware decoders + * reject). iOS then falls back to MJPEG; Android has no MJPEG. + */ + const configureDecoder = async (config: VideoDecoderConfig): Promise => { + const full: VideoDecoderConfig = { ...config, optimizeForLatency: true }; + const support = await VideoDecoder.isConfigSupported(full).catch(() => ({ supported: false })); + if (stopped) return false; + if (!support.supported) { + setStatus("error", `This browser cannot decode ${config.codec}.`); + return false; + } + if (!videoDecoder || videoDecoder.state === "closed") videoDecoder = makeDecoder(); + try { + videoDecoder.configure(full); + return true; + } catch (cause) { + setStatus("error", `Video decoder: ${(cause as Error).message}`); + return false; + } + }; + + const decode = (isKey: boolean, data: Uint8Array, pts?: number | null) => { + if (!videoDecoder || videoDecoder.state !== "configured") return; + if (awaitingKeyframe) { + if (!isKey) return; + awaitingKeyframe = false; + } + if (videoDecoder.decodeQueueSize > SOFT_DECODE_QUEUE) { + closeDecoder(); + requestKeyframe(); + return; + } + try { + videoDecoder.decode( + new EncodedVideoChunk({ + type: isKey ? "key" : "delta", + timestamp: pts ?? timestamp, + data, + }), + ); + timestamp += FRAME_DURATION_US; + } catch { + closeDecoder(); + requestKeyframe(); + } + }; + + const requestKeyframe = () => { + if (platform === "android" && socket?.readyState === WebSocket.OPEN) { + socket.send(JSON.stringify({ type: "reset-video", ack: false })); + } + }; + + const scheduleRetry = (channel: "video" | "input", run: () => void) => { + if (stopped || retryTimers.has(channel)) return; + retryTimers.set( + channel, + setTimeout(() => { + retryTimers.delete(channel); + run(); + }, RETRY_DELAY_MS), + ); + }; + + const handleUnauthorized = () => { + stop(); + events.onUnauthorized(); + }; + + // iOS video: fetch the AVCC body and demux into the decoder. + const readIosVideo = async () => { + const demuxer = new AvccDemuxer(); + controller = new AbortController(); + try { + const response = await fetch(httpUrl(`/helper/${device}/stream.avcc`), { + signal: controller.signal, + credentials: access.credentials ? "include" : "same-origin", + }); + if (response.status === 401 || response.status === 403) return handleUnauthorized(); + if (!response.ok || !response.body) throw new Error(`stream ${response.status}`); + const reader = response.body.getReader(); + for (;;) { + const { done, value } = await reader.read(); + if (done || stopped) break; + for (const chunk of demuxer.push(value)) { + switch (chunk.type) { + case "seed": + void createImageBitmap(new Blob([chunk.payload as BlobPart], { type: "image/jpeg" })) + .then((bitmap) => { + paint(bitmap, bitmap.width, bitmap.height); + bitmap.close(); + }) + .catch(() => {}); + break; + case "description": { + awaitingKeyframe = true; + const configured = await configureDecoder({ + codec: avcCodecString(chunk.payload), + description: chunk.payload, + }); + if (!configured) { + await reader.cancel().catch(() => {}); + fallBackToMjpeg(); + return; + } + break; + } + case "keyframe": + case "delta": + decode(chunk.type === "keyframe", chunk.payload); + break; + } + } + } + } catch (cause) { + if (stopped) return; + setStatus("connecting", (cause as Error).message); + } + if (!stopped) scheduleRetry("video", () => void readIosVideo()); + }; + + /** + * serve-sim's helper only accepts HID and pushes its screen config once + * screen capture is running, and the AVCC stream does not reliably start + * it. Touching the MJPEG endpoint does; one aborted request is enough. + */ + const primeIosHelper = async () => { + const controller = new AbortController(); + primeController = controller; + const timeout = setTimeout(() => controller.abort(), 2_000); + try { + const response = await fetch(httpUrl(`/helper/${device}/stream.mjpeg`), { + signal: controller.signal, + credentials: access.credentials ? "include" : "same-origin", + }); + if (response.status === 401 || response.status === 403) return handleUnauthorized(); + await response.body?.getReader().read(); + } catch { + // A failed prime just means the socket may take a retry to come up. + } finally { + clearTimeout(timeout); + controller.abort(); + if (primeController === controller) primeController = null; + } + }; + + // iOS input socket; also carries the screen config the helper pushes. + const connectIosInput = async () => { + if (stopped) return; + await primeIosHelper(); + if (stopped) return; + const ws = new WebSocket(wsUrl(`/helper/ws?device=${device}`)); + ws.binaryType = "arraybuffer"; + socket = ws; + ws.onopen = () => { + ws.send(taggedJson(IOS_MSG_HARDWARE_KEYBOARD, { enabled: false })); + events.onInputConnected(true); + }; + ws.onmessage = (event) => { + if (!(event.data instanceof ArrayBuffer)) return; + const bytes = new Uint8Array(event.data); + if (bytes.length < 1 || bytes[0] !== IOS_TAG_SCREEN_CONFIG) return; + try { + const config = JSON.parse(decoder.decode(bytes.subarray(1))) as DeviceScreenSize; + if (config.width > 0 && config.height > 0) { + screen = config; + events.onScreen(config); + } + } catch { + // Ignore malformed config frames. + } + }; + ws.onclose = (event) => { + if (socket === ws) socket = null; + if (!stopped) { + events.onInputConnected( + false, + event.reason || (event.code === 1006 ? "input socket refused" : `closed ${event.code}`), + ); + } + if (event.code === 1008 || event.code === 4401) return handleUnauthorized(); + scheduleRetry("input", () => void connectIosInput()); + }; + ws.onerror = () => ws.close(); + }; + + // Android: one socket for video and input. + const connectAndroid = () => { + if (stopped) return; + const ws = new WebSocket(wsUrl(`/ws?device=${device}&frame-meta=1`)); + ws.binaryType = "arraybuffer"; + socket = ws; + ws.onopen = () => { + setStatus("connecting"); + events.onInputConnected(true); + }; + ws.onmessage = (event) => { + if (typeof event.data === "string") { + // The encoder restarts at a new size when the device rotates; the + // next keyframe carries a fresh SPS, so the decoder is rebuilt from it. + if (isVideoSessionMessage(event.data)) closeDecoder(); + return; + } + if (!(event.data instanceof ArrayBuffer)) return; + const packet = parseSemuPacket(event.data); + const needsScan = + packet.isKey === null || + (packet.isKey && (!videoDecoder || videoDecoder.state !== "configured")); + const scanned = needsScan ? scanAccessUnit(packet.data) : null; + const isKey = packet.isKey ?? scanned?.isKey ?? false; + if (scanned?.sps && (!videoDecoder || videoDecoder.state !== "configured")) { + if (configuring) return; + configuring = true; + void configureDecoder({ codec: avcCodecString(scanned.sps) }).then((configured) => { + configuring = false; + awaitingKeyframe = true; + if (configured) requestKeyframe(); + }); + return; + } + if (!videoDecoder || videoDecoder.state !== "configured") { + if (!isKey) requestKeyframe(); + return; + } + decode(isKey, packet.data, packet.timestamp); + }; + ws.onclose = (event) => { + if (socket === ws) socket = null; + closeDecoder(); + if (!stopped) events.onInputConnected(false, event.reason || `closed ${event.code}`); + if (event.code === 1008 || event.code === 4401) return handleUnauthorized(); + if (!stopped) { + setStatus("connecting", event.reason || undefined); + scheduleRetry("input", connectAndroid); + } + }; + ws.onerror = () => ws.close(); + }; + + const start = () => { + if (!stopped) return; + stopped = false; + firstFrame = false; + events.onStatus("connecting"); + if (platform === "ios") { + void connectIosInput(); + if (useWebCodecs) void readIosVideo(); + else fallBackToMjpeg(); + } else if (useWebCodecs) { + connectAndroid(); + } else { + setStatus("error", "This browser cannot decode the Android stream (WebCodecs unavailable)."); + } + }; + + const stop = () => { + if (stopped) return; + stopped = true; + mjpeg = false; + for (const timer of retryTimers.values()) clearTimeout(timer); + retryTimers.clear(); + primeController?.abort(); + primeController = null; + controller?.abort(); + controller = null; + socket?.close(); + socket = null; + closeDecoder(); + }; + + const send = (payload: Uint8Array | string) => { + if (socket?.readyState === WebSocket.OPEN) socket.send(payload); + }; + + const rawPoint = (x: number, y: number) => { + // serve-sim streams the raw framebuffer; rotated devices need input + // remapped into that raw space. + if (platform !== "ios" || !screen || screen.width > screen.height) return { x, y }; + switch (screen.orientation) { + case "landscape_left": + return { x: 1 - y, y: x }; + case "landscape_right": + return { x: y, y: 1 - x }; + case "portrait_upside_down": + return { x: 1 - x, y: 1 - y }; + default: + return { x, y }; + } + }; + + return { + start, + stop, + sendTouch: (phase, x, y) => { + if (platform === "ios") { + send(taggedJson(IOS_MSG_TOUCH, { type: phase, ...rawPoint(x, y) })); + return; + } + const action = phase === "begin" ? "down" : phase === "move" ? "move" : "up"; + send(JSON.stringify({ type: "touch", action, x, y })); + }, + sendKey: (event, phase) => { + if (platform === "ios") { + const usage = hidUsageForCode(event.code); + if (usage !== null) send(taggedJson(IOS_MSG_KEY, { type: phase, usage })); + return; + } + if (phase !== "down") return; + if (event.key === "Escape") return send(JSON.stringify({ type: "back" })); + const keycode = ANDROID_KEYCODE_BY_KEY[event.key]; + if (keycode !== undefined) return send(JSON.stringify({ type: "key", keycode })); + if (event.key.length === 1 && !event.metaKey && !event.ctrlKey) { + send(JSON.stringify({ type: "text", text: event.key })); + } + }, + pressButton: (button) => { + if (platform === "ios") { + const name = + button === "home" + ? "home" + : button === "appSwitcher" + ? "app_switcher" + : button === "power" + ? "lock" + : null; + if (name) send(taggedJson(IOS_MSG_BUTTON, { button: name })); + return; + } + const type = button === "appSwitcher" ? "recents" : button; + if (type === "home" || type === "back" || type === "recents" || type === "power") { + send(JSON.stringify({ type })); + } + }, + rotate: () => { + if (platform !== "ios") return; + const current = screen?.orientation ?? "portrait"; + const next = + IOS_ORIENTATIONS[(IOS_ORIENTATIONS.indexOf(current) + 1) % IOS_ORIENTATIONS.length]!; + send(taggedJson(IOS_MSG_ORIENTATION, { orientation: next })); + }, + }; +} diff --git a/apps/web/src/components/preview/PreviewEmptyState.tsx b/apps/web/src/components/preview/PreviewEmptyState.tsx index 163849154000..35b4b2f13d64 100644 --- a/apps/web/src/components/preview/PreviewEmptyState.tsx +++ b/apps/web/src/components/preview/PreviewEmptyState.tsx @@ -3,6 +3,7 @@ import { Globe, History, RadioTower } from "lucide-react"; import type { BrowserHistoryEntry } from "~/browserHistoryStore"; import { Empty, EmptyDescription, EmptyMedia, EmptyTitle } from "~/components/ui/empty"; +import { DiscoveryList } from "../ui/discovery-list"; import { PreviewLocalServerCard } from "./PreviewLocalServerCard"; import { PreviewRecentUrlCard } from "./PreviewRecentUrlCard"; @@ -55,7 +56,7 @@ export function PreviewEmptyState({

Recently used

-
+ {recents.map((entry) => ( onRemoveRecent(entry.url)} /> ))} -
+
) : null} {servers.length > 0 ? ( @@ -74,7 +75,7 @@ export function PreviewEmptyState({

Local servers

-
+ {servers.map((server) => ( onOpenUrl(server.requestedUrl)} /> ))} -
+

Select a live local server to open it in this browser tab.

diff --git a/apps/web/src/components/preview/PreviewLocalServerCard.tsx b/apps/web/src/components/preview/PreviewLocalServerCard.tsx index 263cdb294f48..198d0495473c 100644 --- a/apps/web/src/components/preview/PreviewLocalServerCard.tsx +++ b/apps/web/src/components/preview/PreviewLocalServerCard.tsx @@ -1,4 +1,5 @@ import type { ScopedThreadRef } from "@t3tools/contracts"; +import { DiscoveryListRow } from "../ui/discovery-list"; import { PreviewFaviconIcon } from "./PreviewFaviconIcon"; import type { PreviewableServer } from "./useDiscoveredLocalServers"; @@ -12,19 +13,12 @@ interface Props { export function PreviewLocalServerCard({ threadRef, server, onOpen }: Props) { const subtitle = describeServer(server); return ( - + icon={} + title={subtitle} + description={`${server.host}:${server.port}`} + /> ); } diff --git a/apps/web/src/components/settings/IntegrationsSettings.test.tsx b/apps/web/src/components/settings/IntegrationsSettings.test.tsx index 5d185fee5824..9ce3b2126c0e 100644 --- a/apps/web/src/components/settings/IntegrationsSettings.test.tsx +++ b/apps/web/src/components/settings/IntegrationsSettings.test.tsx @@ -1,4 +1,8 @@ -import { DEFAULT_CLIENT_SETTINGS, DEFAULT_UNIFIED_SETTINGS } from "@t3tools/contracts"; +import { + DEFAULT_CLIENT_SETTINGS, + DEFAULT_UNIFIED_SETTINGS, + type DeviceServiceState, +} from "@t3tools/contracts"; import { createMemoryHistory, createRootRoute, @@ -36,6 +40,7 @@ vi.mock("./settingsLayout", async (importOriginal) => ({ })); import { IntegrationsSettingsPanel } from "./IntegrationsSettings"; +import { platformSetupStatus } from "../device/DeviceSetup"; let renderer: ReactTestRenderer | undefined; @@ -74,4 +79,59 @@ describe("Integrations browser discovery", () => { await openSettings(); expect(listBrowserImportSources).not.toHaveBeenCalled(); }); + + it("places device settings directly after browser settings", async () => { + await openSettings(); + const sections = renderer!.root + .findAll((node) => node.type === "section") + .map((node) => node.props.id) + .filter(Boolean); + expect(sections.indexOf("devices")).toBeGreaterThan(sections.indexOf("browser")); + }); +}); + +const deviceState = (overrides: Partial = {}): DeviceServiceState => ({ + hosts: [ + { + id: "local", + kind: "local", + label: "This machine", + hubInstalled: false, + agentDeviceInstalled: false, + platforms: [ + { platform: "ios", available: true }, + { platform: "android", available: true }, + ], + }, + ], + hostStatus: "ready", + devices: [], + sessions: [], + onboardingCompleted: false, + agentAccessEnabled: false, + hubBasePath: "/api/device-hub", + revision: 0, + ...overrides, +}); + +describe("device setup guidance", () => { + it("directs users to install an iOS runtime and create an Android virtual device", () => { + expect(platformSetupStatus(deviceState(), "ios").message).toContain("Xcode Settings"); + expect(platformSetupStatus(deviceState(), "android").message).toContain("Device Manager"); + }); + + it("preserves a specific missing-tool explanation from the server", () => { + const state = deviceState({ + hosts: [ + { + ...deviceState().hosts[0]!, + platforms: [ + { platform: "ios", available: true }, + { platform: "android", available: false, reason: "Android Emulator is missing." }, + ], + }, + ], + }); + expect(platformSetupStatus(state, "android").message).toBe("Android Emulator is missing."); + }); }); diff --git a/apps/web/src/components/settings/IntegrationsSettings.tsx b/apps/web/src/components/settings/IntegrationsSettings.tsx index a7a3c7553540..a3f76f49cd5f 100644 --- a/apps/web/src/components/settings/IntegrationsSettings.tsx +++ b/apps/web/src/components/settings/IntegrationsSettings.tsx @@ -39,10 +39,21 @@ import { MoreVertical, Plus as PlusIcon } from "lucide-react"; import { useCallback, useRef, useState } from "react"; import { ScreenRotationIcon } from "~/browser/ScreenRotationIcon"; +import { AnimatedHeight } from "~/components/AnimatedHeight"; import { resolveEnvironmentOptionLabel } from "~/components/BranchToolbar.logic"; import { previewBridge } from "~/components/preview/previewBridge"; import { cn, randomUUID } from "~/lib/utils"; import { useEnvironments, usePrimaryEnvironment } from "~/state/environments"; +import { deviceEnvironment, useDeviceState } from "~/state/device"; +import { useAtomCommand } from "~/state/use-atom-command"; +import { + AgentDeviceSetupStatus, + DeviceHubSetupStatus, + PlatformStatus, + platformSetupStatus, + deviceHubDescription, + agentDeviceDescription, +} from "~/components/device/DeviceSetup"; import { isElectron } from "../../env"; import { Badge } from "../ui/badge"; @@ -571,6 +582,117 @@ function AgentBrowserAccessSetting() { ); } +function DeviceIntegrationSettings() { + const primaryEnvironment = usePrimaryEnvironment(); + const environmentId = primaryEnvironment?.environmentId ?? null; + const { state, loaded } = useDeviceState(environmentId); + const configure = useAtomCommand(deviceEnvironment.configure); + const list = useAtomCommand(deviceEnvironment.list, { reportFailure: false }); + const [pending, setPending] = useState<"hub" | "check" | "agent" | null>(null); + const enabled = state.hostStatus !== "disabled"; + const busy = state.hostStatus === "installing" || state.hostStatus === "starting"; + const [platformsRevealed, setPlatformsRevealed] = useState(false); + // Keep diagnostics visible through subsequent agent setup and refresh phases. + if (platformsRevealed && !enabled) setPlatformsRevealed(false); + if (!platformsRevealed && state.hostStatus === "ready" && pending !== "hub") { + setPlatformsRevealed(true); + } + + const update = async ( + kind: NonNullable, + input: { enabled?: boolean; agentAccessEnabled?: boolean }, + ) => { + if (!environmentId) return; + setPending(kind); + try { + const result = await configure({ environmentId, input }); + if (result._tag === "Success" && input.enabled === true && !state.onboardingCompleted) { + await configure({ environmentId, input: { onboardingCompleted: true } }); + } + } finally { + setPending(null); + } + }; + + return ( + + + {pending === "hub" ? : null} + + void update("hub", { + enabled: Boolean(checked), + ...(checked ? {} : { agentAccessEnabled: false }), + }) + } + /> + + } + /> + + {platformsRevealed ? ( + + + + + } + control={ + + } + /> + ) : null} + + + {pending === "agent" ? : null} + + void update("agent", { agentAccessEnabled: Boolean(checked) }) + } + /> + + } + /> + {state.hostStatus === "failed" && state.hostStatusDetail ? ( +

+ {state.hostStatusDetail} +

+ ) : null} + + ); +} + function BrowserAutoShowFloatingPreviewSetting({ disabled }: { readonly disabled: boolean }) { const autoShow = useClientSettings((settings) => settings.browserAutoShowFloatingPreview); const updateSettings = useUpdatePrimarySettings(); @@ -1170,6 +1292,7 @@ export function IntegrationsSettingsPanel() { previewDefaults )} + ); } diff --git a/apps/web/src/components/settings/settingsSearch.ts b/apps/web/src/components/settings/settingsSearch.ts index e32baf6f3987..655f1758c113 100644 --- a/apps/web/src/components/settings/settingsSearch.ts +++ b/apps/web/src/components/settings/settingsSearch.ts @@ -396,6 +396,27 @@ export const SETTINGS_SEARCH_ITEMS = [ to: "/settings/projects", searchTerms: ["allow open drive preview tools sessions"], }, + { + id: "agent-device-access", + title: "Agent device access", + to: "/settings/integrations", + targetId: "devices", + searchTerms: ["allow simulator emulator ios android drive tools sessions"], + }, + { + id: "device-hub", + title: "Device hub", + to: "/settings/integrations", + targetId: "devices", + searchTerms: ["simulator emulator ios android install start"], + }, + { + id: "device-platform-support", + title: "Simulator support", + to: "/settings/integrations", + targetId: "devices", + searchTerms: ["xcode android studio sdk avd runtime"], + }, { id: "browser-profiles", title: "Browser profiles", diff --git a/apps/web/src/components/ui/discovery-list.tsx b/apps/web/src/components/ui/discovery-list.tsx new file mode 100644 index 000000000000..d03ca0edd5e0 --- /dev/null +++ b/apps/web/src/components/ui/discovery-list.tsx @@ -0,0 +1,37 @@ +import type { ComponentProps, ReactNode } from "react"; + +export function DiscoveryList({ children }: { readonly children: ReactNode }) { + return ( +
+ {children} +
+ ); +} + +export function DiscoveryListRow({ + icon, + title, + description, + action, + ...props +}: Omit, "title" | "children" | "className"> & { + readonly icon: ReactNode; + readonly title: ReactNode; + readonly description: ReactNode; + readonly action?: ReactNode; +}) { + return ( + + ); +} diff --git a/apps/web/src/rightPanelStore.ts b/apps/web/src/rightPanelStore.ts index d167afeddb7f..bb50d200f377 100644 --- a/apps/web/src/rightPanelStore.ts +++ b/apps/web/src/rightPanelStore.ts @@ -24,6 +24,7 @@ const RIGHT_PANEL_KINDS = [ "files", "file", "preview", + "device", "terminal", "pull-request", "agents", @@ -33,6 +34,12 @@ export type RightPanelKind = (typeof RIGHT_PANEL_KINDS)[number]; export type RightPanelSurface = | { id: `browser:${string}`; kind: "preview"; resourceId: string } | { id: "browser:new"; kind: "preview"; resourceId: null } + /** + * One Device tab per thread. The tab is the surface; which device it shows + * comes from the thread's server-side device sessions, so an agent opening a + * device from another client lands in the same tab. + */ + | { id: "device"; kind: "device" } | { id: `terminal:${string}`; kind: "terminal"; @@ -78,7 +85,8 @@ const RIGHT_PANEL_STORAGE_KEY = "t3code:right-panel-state:v2"; // v9 removed the "plan" surface kind (plans render inline in the transcript). // v10 keys pull-request surfaces by reference instead of a singleton tab. // v11 stops persisting the pull-request list's shared panel, so a restart opens the page fresh. -const RIGHT_PANEL_STORAGE_VERSION = 11; +// v12 adds the device surface. +const RIGHT_PANEL_STORAGE_VERSION = 12; /** A fixed workspace-level ref: each PR surface carries its own real environment. */ export const PULL_REQUESTS_PANEL_REF = scopeThreadRef( @@ -171,6 +179,8 @@ const singletonSurface = ( return { id: "files", kind }; case "agents": return { id: "agents", kind }; + case "device": + return { id: "device", kind }; } }; diff --git a/apps/web/src/routes/_chat.pull-requests.tsx b/apps/web/src/routes/_chat.pull-requests.tsx index 99986567cbfa..428a2f6dcfc6 100644 --- a/apps/web/src/routes/_chat.pull-requests.tsx +++ b/apps/web/src/routes/_chat.pull-requests.tsx @@ -1924,12 +1924,14 @@ function PullRequestsRouteView() { onAddFiles={() => undefined} onAddPullRequest={() => undefined} onAddAgents={() => undefined} + onAddDevice={() => undefined} browserAvailable={false} terminalAvailable={false} diffAvailable={false} filesAvailable={false} pullRequestAvailable={false} agentsAvailable={false} + deviceAvailable={false} liveAgentCount={0} pullRequestStatusSeeds={listedPullRequestTabStatuses} > diff --git a/apps/web/src/state/device.ts b/apps/web/src/state/device.ts new file mode 100644 index 000000000000..b9c1adf86cdd --- /dev/null +++ b/apps/web/src/state/device.ts @@ -0,0 +1,70 @@ +import { useAtomValue } from "@effect/atom-react"; +import { createDeviceEnvironmentAtoms } from "@t3tools/client-runtime/state/device"; +import { + type DeviceHubAccess, + resolveDeviceHubAccess, +} from "@t3tools/client-runtime/state/deviceHubAccess"; +import type { DeviceServiceState, EnvironmentId } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; +import { AsyncResult, Atom } from "effect/unstable/reactivity"; + +import { connectionAtomRuntime } from "../connection/runtime"; +import { appAtomRegistry } from "../rpc/atomRegistry"; +import { environmentSession } from "./session"; +import { useEnvironmentQuery } from "./query"; + +export const deviceEnvironment = createDeviceEnvironmentAtoms(connectionAtomRuntime); + +const EMPTY_DEVICE_STATE: DeviceServiceState = { + hosts: [], + hostStatus: "idle", + devices: [], + sessions: [], + onboardingCompleted: false, + agentAccessEnabled: false, + hubBasePath: "/api/device-hub", + revision: 0, +}; + +export function useDeviceState(environmentId: EnvironmentId | null): { + readonly state: DeviceServiceState; + readonly loaded: boolean; +} { + const query = useEnvironmentQuery( + environmentId === null ? null : deviceEnvironment.state({ environmentId, input: {} }), + ); + return { state: query.data ?? EMPTY_DEVICE_STATE, loaded: query.data !== undefined }; +} + +/** + * Hub access for one environment. Bearer and DPoP connections mint a ticket + * here; a stream that gets a 401 back refreshes this atom and reconnects. + * Keyed on the prepared connection so a re-pair produces new credentials. + */ +const deviceHubAccessAtom = Atom.family((environmentId: EnvironmentId) => + connectionAtomRuntime + .atom((get) => { + const prepared = Option.getOrNull( + get(environmentSession.preparedConnectionValueAtom(environmentId)), + ); + if (prepared === null) return Effect.never; + return resolveDeviceHubAccess({ prepared, hubBasePath: EMPTY_DEVICE_STATE.hubBasePath }); + }) + .pipe(Atom.setIdleTTL(60_000), Atom.withLabel(`device-hub-access:${environmentId}`)), +); + +export function useDeviceHubAccess(environmentId: EnvironmentId | null): DeviceHubAccess | null { + const result = useAtomValue( + environmentId === null ? EMPTY_ACCESS_ATOM : deviceHubAccessAtom(environmentId), + ); + return AsyncResult.isSuccess(result) ? result.value : null; +} + +const EMPTY_ACCESS_ATOM = Atom.make(AsyncResult.initial()).pipe( + Atom.withLabel("device-hub-access:empty"), +); + +export function refreshDeviceHubAccess(environmentId: EnvironmentId): void { + appAtomRegistry.refresh(deviceHubAccessAtom(environmentId)); +} diff --git a/apps/web/vite.config.ts b/apps/web/vite.config.ts index 702c18840983..ec4f15fc7eeb 100644 --- a/apps/web/vite.config.ts +++ b/apps/web/vite.config.ts @@ -230,16 +230,17 @@ export default defineConfig(() => { ? { // One entry per shared prefix; the server's dev catch-all 404s the // same list, so the two sides cannot drift. `/ws` is the app's own - // socket — Vite's HMR socket is matched separately and exactly - // (path "/" plus a vite-hmr subprotocol), so the two upgrade - // handlers don't collide. + // socket and `/api` carries the device hub's stream sockets — + // Vite's HMR socket is matched separately and exactly (path "/" + // plus a vite-hmr subprotocol), so the upgrade handlers don't + // collide. proxy: Object.fromEntries( DEV_PROXIED_PATH_PREFIXES.map((prefix) => [ prefix, { target: devProxyTarget, changeOrigin: true, - ...(prefix === "/ws" ? { ws: true } : {}), + ...(prefix === "/ws" || prefix === "/api" ? { ws: true } : {}), }, ]), ), diff --git a/docs/README.md b/docs/README.md index 2d4809b7003a..4691e6f83c8e 100644 --- a/docs/README.md +++ b/docs/README.md @@ -13,6 +13,7 @@ - [Keyboard shortcuts](./user/keybindings.md) - [SnapShots](./user/snap-shot.md) - [Import browser sessions](./user/browser-import.md) +- [Devices](./user/devices.md) - [Usage and limits](./user/usage.md) - [Product usage data](./user/telemetry.md) - [Remote access](./user/remote-access.md) @@ -46,6 +47,7 @@ source alone does not explain. Most code changes do not need an internal documen - [Mobile navigation](./internals/mobile-navigation.md) - [Mobile development lifecycle](./internals/mobile-development.md) - [Terminal runtime](./internals/terminal-runtime.md) +- [Devices](./internals/devices.md) - [Voice input](./internals/voice-input.md) ### Runbooks diff --git a/docs/internals/devices.md b/docs/internals/devices.md new file mode 100644 index 000000000000..49027022fdbc --- /dev/null +++ b/docs/internals/devices.md @@ -0,0 +1,87 @@ +# Devices + +The environment server owns simulators and emulators the way it owns +terminals: discovery, streaming, and agent access all run there, and every +client reaches them through the environment connection. This is what makes the +Device panel work over Tailscale and T3 Connect, and what will let a device +host on another machine slot in later. + +## Two external tools, one seam + +[expo-device-hub](../../apps/server/src/device/LocalDeviceHost.ts) streams and +[agent-device](../../apps/server/src/device/AgentDeviceShim.ts) drives. Each is +npm-installed at a pinned version into the T3 home after its matching Device +panel consent step. Manual setup installs and starts only expo-device-hub; +agent-device remains absent and stopped until agent access is granted. Both run +with the server's Node; `npx` would make the first `device_open` after a reboot +depend on the registry. The hub is a supervised child rather than an imported +middleware because serve-sim loads private CoreSimulator frameworks through a +native addon, and a crash there must not take the server down. + +Everything platform-specific sits behind +[`DeviceHost`](../../apps/server/src/device/DeviceHost.ts). The service, the +proxy, and the MCP tools only see a hub origin and an agent-device endpoint. +An SSH or cloud host would forward those two things to the server and change +nothing above it. + +## The hub is never exposed + +serve-sim has a shell-exec route whose token is readable from its own +unauthenticated `/api`, and serve-emu's action routes have no auth at all. The +hub binds loopback and the only way in is the +[proxy](../../apps/server/src/device/DeviceHubProxy.ts), which allowlists the +stream, config, and screenshot routes and authenticates every request as an +environment session. `` and `WebSocket` cannot carry headers, so the proxy +authenticates like the `/ws` upgrade: cookie, or a short-lived `wsTicket` that +bearer and DPoP clients mint over authenticated HTTP. The ticket is stripped +before the request reaches the hub. + +Stream responses carry `Cache-Control: no-transform`; the compression +middleware would otherwise buffer an MJPEG body that never ends. In browser dev, +the Vite proxy must forward WebSocket upgrades for `/api`, not only `/ws`. + +## Device settings never go through the hub + +serve-sim's preview drives its Tools panel by sending shell commands over that +same exec channel. Proxying it, even allowlisted, would hand any environment +session arbitrary command execution on the host, so T3 does not. The +[`device.action`](../../apps/server/src/device/DeviceActions.ts) RPC runs the +underlying `simctl`, `adb`, and serve-sim helper binaries itself through +`DeviceHostReady.run`, one typed action per control, and returns the settings +it reads back. The proxy allowlist grows only with read routes (accessibility +tree, foreground app, event log) and refuses non-GET methods everywhere except +screenshot capture and stream tuning. + +## Agents drive through the CLI + +The `device_*` toolkit is deliberately four tools: list, open, screenshot, and +close. Driving happens through the `agent-device` CLI, which has the semantic +snapshot model agents need and stays current with its own releases. T3 prepends +a shim directory to the provider's PATH and sets +`AGENT_DEVICE_DAEMON_BASE_URL` and `AGENT_DEVICE_DAEMON_AUTH_TOKEN` so the +agent never handles the endpoint or token. + +That environment is fixed when the provider subprocess spawns, so +[`prepareMcpSession`](../../apps/server/src/provider/Layers/ProviderService.ts) +starts agent-device only when device support and agent access have both been +enabled, the session has the `device` capability, and the machine can run at +least one platform. Starting it later from `device_open` would leave the +already-running agent without the CLI. + +How to drive a device is returned from `device_open`, not kept in an +always-loaded prompt or skill: it costs nothing in threads that never open a +device and cannot drift from the pinned CLI version. The always-on prompt block +is a few lines that point at the tools and forbid raw `simctl` and `adb`. + +## The viewer decodes both vendored protocols + +The hub vendors two streaming servers with different wire formats. iOS video is +an HTTP body of AVCC envelopes decoded with WebCodecs, with input on a separate +binary WebSocket; Android multiplexes SEMU-framed H.264 and JSON gestures over +one WebSocket. [`deviceStream.ts`](../../apps/web/src/components/device/deviceStream.ts) +speaks both so one panel covers both platforms. + +Simulators encode H.264 High 5.1. Hardware decoders on some machines and all +headless browsers reject that profile, and WebCodecs is secure-context only, so +the viewer probes `isConfigSupported` and falls back to the MJPEG endpoint on +iOS. Android has no MJPEG; there the panel reports that it cannot decode. diff --git a/docs/user/devices.md b/docs/user/devices.md new file mode 100644 index 000000000000..51a0a578b607 --- /dev/null +++ b/docs/user/devices.md @@ -0,0 +1,63 @@ +# Devices + +The Device panel shows a live iOS Simulator or Android Emulator next to a +thread, so you can watch an agent verify mobile work and tap the device +yourself. Agents get the same device through `device_*` tools and the +`agent-device` command line, which T3 Code sets up for them. + +## Open a device + +Open the right panel in a project thread and choose **Device**. On first use, +the panel walks through three steps: starting the device hub, checking iOS and +Android support, and choosing whether agents may control devices. Opening the +panel alone does not download or start anything. If the hub is already +installed, the setup screen says so and reuses it. + +Choose a running device to watch it, or choose **Start** next to a stopped +device to boot it. The panel shows when you or an agent starts a device. +Turn off the device hub in **Settings → Integrations → Devices** to stop the +helper processes; simulators and emulators keep running until you power them +off. + +Simulators run on the machine that hosts the environment server. iOS needs +macOS with Xcode. Android needs the SDK Platform-Tools, Android Emulator, +and Command-line Tools (latest), plus a virtual device created in Android +Studio's Device Manager. T3 Code detects standard SDK locations; set +`ANDROID_HOME` for a custom location. The panel explains missing dependencies. +After installing them, restart the environment server and refresh devices. + +The screen is interactive: click and drag to touch, type while the screen is +focused, and use the toolbar for Home, Back, and Recents on Android, rotate on +iOS, and power off. Close the tab to stop watching; the device keeps running +unless you power it off. + +## Tools + +The toolbar's **Tools** button opens a drawer for the open device. It shows the +foreground app, and lets you switch light and dark mode, change text size, +flip accessibility settings, overlay the accessibility element frames on the +screen, set a fake location, and grant or revoke app permissions. iOS also +exposes Liquid Glass, color filters, VoiceOver, and sending a test push +notification; Android adds orientation and toggling the network. The drawer +only shows what the platform can do, and every control reflects the value read +back from the device after a change. + +## Agents and devices + +When an agent opens a device, the panel opens in web and desktop clients connected +to the thread. Mobile clients show device activity in the thread timeline. Agents drive the device through the `agent-device` command line. T3 +Code installs and starts it only after **Agent device access** is enabled. iOS +taps build a small test runner on first use, which takes a couple of minutes +once per server. Restart an existing agent session after granting access so it +receives the device CLI environment. + +To keep agents away from simulators, turn off **Agent device access** in +**Settings → Integrations → Devices**. This hides the device tools from agents +started from then on; your own Device panel is unaffected. + +## Remote connections + +The device stream goes through the environment server, so it works over the +local network, Tailscale, and T3 Connect. Live video needs a secure page +(HTTPS or localhost); on a plain-HTTP remote origin iOS falls back to a slower +still-image stream and Android cannot show video. diff --git a/packages/client-runtime/package.json b/packages/client-runtime/package.json index 368585556cfe..6eb62eaa44e2 100644 --- a/packages/client-runtime/package.json +++ b/packages/client-runtime/package.json @@ -131,6 +131,14 @@ "types": "./src/state/presentation.ts", "default": "./src/state/presentation.ts" }, + "./state/device": { + "types": "./src/state/device.ts", + "default": "./src/state/device.ts" + }, + "./state/deviceHubAccess": { + "types": "./src/state/deviceHubAccess.ts", + "default": "./src/state/deviceHubAccess.ts" + }, "./state/preview": { "types": "./src/state/preview.ts", "default": "./src/state/preview.ts" diff --git a/packages/client-runtime/src/rpc/client.ts b/packages/client-runtime/src/rpc/client.ts index 4ddfb9c4160e..cfabaa00c0b7 100644 --- a/packages/client-runtime/src/rpc/client.ts +++ b/packages/client-runtime/src/rpc/client.ts @@ -51,6 +51,7 @@ export type EnvironmentSubscriptionRpcTag = | typeof WS_METHODS.subscribeTerminalMetadata | typeof WS_METHODS.subscribePreviewEvents | typeof WS_METHODS.subscribeDiscoveredLocalServers + | typeof WS_METHODS.subscribeDeviceState | typeof WS_METHODS.subscribeResourceTelemetry | typeof WS_METHODS.pullRequestsSubscribeRefreshes | typeof WS_METHODS.previewAutomationConnect diff --git a/packages/client-runtime/src/state/device.ts b/packages/client-runtime/src/state/device.ts new file mode 100644 index 000000000000..df27f793720c --- /dev/null +++ b/packages/client-runtime/src/state/device.ts @@ -0,0 +1,68 @@ +import { WS_METHODS } from "@t3tools/contracts"; +import { Atom } from "effect/unstable/reactivity"; + +import type { EnvironmentRegistry } from "../connection/registry.ts"; +import { + createAtomCommandScheduler, + createEnvironmentRpcCommand, + createEnvironmentRpcSubscriptionAtomFamily, +} from "./runtime.ts"; + +export function createDeviceEnvironmentAtoms( + runtime: Atom.AtomRuntime, +) { + const scheduler = createAtomCommandScheduler(); + const concurrency = { + mode: "serial" as const, + key: ({ environmentId }: { environmentId: string }) => environmentId, + }; + return { + /** Server-pushed device hosts, devices, and open sessions for one environment. */ + state: createEnvironmentRpcSubscriptionAtomFamily(runtime, { + label: "environment-data:device:state", + tag: WS_METHODS.subscribeDeviceState, + }), + configure: createEnvironmentRpcCommand(runtime, { + label: "environment-data:device:configure", + tag: WS_METHODS.deviceConfigure, + scheduler, + concurrency, + }), + list: createEnvironmentRpcCommand(runtime, { + label: "environment-data:device:list", + tag: WS_METHODS.deviceList, + scheduler, + concurrency, + }), + open: createEnvironmentRpcCommand(runtime, { + label: "environment-data:device:open", + tag: WS_METHODS.deviceOpen, + scheduler, + concurrency, + }), + close: createEnvironmentRpcCommand(runtime, { + label: "environment-data:device:close", + tag: WS_METHODS.deviceClose, + scheduler, + concurrency, + }), + shutdown: createEnvironmentRpcCommand(runtime, { + label: "environment-data:device:shutdown", + tag: WS_METHODS.deviceShutdown, + scheduler, + concurrency, + }), + detail: createEnvironmentRpcCommand(runtime, { + label: "environment-data:device:detail", + tag: WS_METHODS.deviceDetail, + scheduler, + concurrency, + }), + action: createEnvironmentRpcCommand(runtime, { + label: "environment-data:device:action", + tag: WS_METHODS.deviceAction, + scheduler, + concurrency, + }), + }; +} diff --git a/packages/client-runtime/src/state/deviceHubAccess.ts b/packages/client-runtime/src/state/deviceHubAccess.ts new file mode 100644 index 000000000000..328dea83ce7a --- /dev/null +++ b/packages/client-runtime/src/state/deviceHubAccess.ts @@ -0,0 +1,72 @@ +/** + * Credentials for the Device panel's media requests. + * + * The panel reaches simulator streams through `/api/device-hub/*` on the + * environment origin. ``, `EventSource`, and `WebSocket` cannot set + * bearer or DPoP headers, so bearer and DPoP connections mint a + * short-lived WebSocket ticket and pass it as `wsTicket`, the same way the + * app's own `/ws` upgrade authenticates. Cookie sessions send the cookie. + * + * A ticket lives five minutes server-side and is bound to the session, not + * to one request, so one ticket covers everything a panel opens at once. + * Callers fetch a fresh one each time they (re)connect a stream. + */ +import * as Effect from "effect/Effect"; +import type { HttpClient } from "effect/unstable/http"; + +import { RemoteEnvironmentAuthorization } from "../authorization/service.ts"; +import type { PreparedConnection } from "../connection/model.ts"; +import { environmentEndpointUrl } from "../environment/endpoint.ts"; +import { ManagedRelayDpopSigner } from "../relay/managedRelay.ts"; +import type { RemoteEnvironmentRequestError } from "../rpc/http.ts"; +import { executeAuthenticatedEnvironmentHttpRequest } from "./environmentHttpAuth.ts"; + +const TICKET_TIMEOUT_MS = 8_000; + +export interface DeviceHubAccess { + /** Absolute origin-relative base, e.g. `https://env.example/api/device-hub`. */ + readonly httpBase: string; + /** Same base with the `ws(s)` scheme. */ + readonly wsBase: string; + /** Query parameters to append to every hub request; empty for cookie sessions. */ + readonly query: Readonly>; + /** Whether requests must include cookies (same-origin session). */ + readonly credentials: boolean; +} + +export const resolveDeviceHubAccess = Effect.fn("clientRuntime.state.resolveDeviceHubAccess")( + function* (input: { + readonly prepared: PreparedConnection; + readonly hubBasePath: string; + }): Effect.fn.Return { + const httpBase = environmentEndpointUrl(input.prepared.httpBaseUrl, input.hubBasePath); + const wsBase = httpBase.replace(/^http/, "ws"); + if (input.prepared.httpAuthorization === null) { + return { httpBase, wsBase, query: {}, credentials: true }; + } + const signer = yield* Effect.serviceOption(ManagedRelayDpopSigner); + const remoteAuthorization = yield* Effect.serviceOption(RemoteEnvironmentAuthorization); + const ticket = yield* executeAuthenticatedEnvironmentHttpRequest({ + prepared: input.prepared, + signer, + remoteAuthorization, + method: "POST", + url: (httpBaseUrl) => environmentEndpointUrl(httpBaseUrl, "/api/auth/websocket-ticket"), + timeoutMs: TICKET_TIMEOUT_MS, + request: ({ client, headers }) => client.auth.webSocketTicket({ headers }), + }); + return { + httpBase, + wsBase, + query: { wsTicket: ticket.ticket }, + credentials: false, + }; + }, +); + +export const withDeviceHubQuery = (url: string, access: DeviceHubAccess): string => { + const entries = Object.entries(access.query); + if (entries.length === 0) return url; + const separator = url.includes("?") ? "&" : "?"; + return `${url}${separator}${new URLSearchParams(entries).toString()}`; +}; diff --git a/packages/client-runtime/src/work-log/presentation.test.ts b/packages/client-runtime/src/work-log/presentation.test.ts index e056e92afe5c..2a2334ace676 100644 --- a/packages/client-runtime/src/work-log/presentation.test.ts +++ b/packages/client-runtime/src/work-log/presentation.test.ts @@ -221,6 +221,19 @@ describe("resolveWorkEntryToolPresentation", () => { }); }); + it("labels device tools with the device icon", () => { + expect( + resolveWorkEntryToolPresentation({ + label: "mcp__t3-code__device_open", + toolLifecycleStatus: "completed", + }), + ).toEqual({ displayName: "Opened a device in the Device panel", icon: "device" }); + expect(resolveWorkEntryToolPresentation({ label: "t3-code · device_screenshot" })).toEqual({ + displayName: "Taking a screenshot of the device", + icon: "device", + }); + }); + it("uses structured MCP identity when the provider supplies a custom title", () => { expect( resolveWorkEntryToolPresentation({ diff --git a/packages/client-runtime/src/work-log/presentation.ts b/packages/client-runtime/src/work-log/presentation.ts index 2e1ef3bbf003..9b2e3988752c 100644 --- a/packages/client-runtime/src/work-log/presentation.ts +++ b/packages/client-runtime/src/work-log/presentation.ts @@ -96,6 +96,15 @@ const T3_MCP_TOOL_LABELS: Record< preview_set_appearance: ["Set", "Setting", "Set", "preview browser appearance"], preview_recording_start: ["Start", "Starting", "Started", "recording the preview browser"], preview_recording_stop: ["Stop", "Stopping", "Stopped", "recording the preview browser"], + device_list: ["List", "Listing", "Listed", "simulators and emulators"], + device_open: ["Open", "Opening", "Opened", "a device in the Device panel"], + device_screenshot: [ + "Take a screenshot of", + "Taking a screenshot of", + "Took a screenshot of", + "the device", + ], + device_close: ["Close", "Closing", "Closed", "a device"], }; function resolveT3McpToolPresentation(value: string | undefined, status: string | undefined) { @@ -122,7 +131,11 @@ function resolveT3McpToolPresentation(value: string | undefined, status: string return { displayName: `${verb} ${detail}`, - icon: name.startsWith("preview_") ? ("browser" as const) : ("t3-code" as const), + icon: name.startsWith("preview_") + ? ("browser" as const) + : name.startsWith("device_") + ? ("device" as const) + : ("t3-code" as const), }; } diff --git a/packages/contracts/src/device.ts b/packages/contracts/src/device.ts new file mode 100644 index 000000000000..309d156430fa --- /dev/null +++ b/packages/contracts/src/device.ts @@ -0,0 +1,516 @@ +/** + * Device - Schemas for first-class iOS Simulator and Android Emulator support. + * + * The server owns device discovery, the streaming helper (expo-device-hub), + * and the agent driver (agent-device). Clients render the live screen from the + * server-proxied stream, and agents reach devices through the `device_*` MCP + * tools plus the `agent-device` CLI the server preconfigures for them. + * + * Devices live on a *host*. Only the local host (the machine the server runs + * on) exists today; the host id is carried everywhere so SSH and cloud hosts + * can be added without changing the client contract. + * + * @module Device + */ +import { Schema } from "effect"; + +import { ThreadId, TrimmedNonEmptyString } from "./baseSchemas.ts"; + +export const DevicePlatform = Schema.Literals(["ios", "android"]); +export type DevicePlatform = typeof DevicePlatform.Type; + +export const DeviceHostId = TrimmedNonEmptyString.check(Schema.isMaxLength(128)); +export type DeviceHostId = typeof DeviceHostId.Type; + +/** The server machine. Always present; other host kinds are future work. */ +export const LOCAL_DEVICE_HOST_ID = "local" as DeviceHostId; + +/** Simulator udid or adb serial (an AVD name while it is not running). */ +export const DeviceId = TrimmedNonEmptyString.check(Schema.isMaxLength(256)); +export type DeviceId = typeof DeviceId.Type; + +export const DeviceSummary = Schema.Struct({ + hostId: DeviceHostId, + id: DeviceId, + platform: DevicePlatform, + name: TrimmedNonEmptyString, + /** OS label such as "iOS 18.0" or "Android 15.0". */ + version: Schema.String, + booted: Schema.Boolean, + physical: Schema.Boolean, +}); +export type DeviceSummary = typeof DeviceSummary.Type; + +/** + * What the host can do right now. Platforms missing their toolchain are + * reported rather than hidden so the picker and the agent can explain why a + * platform is absent instead of showing an empty list. + */ +export const DevicePlatformAvailability = Schema.Struct({ + platform: DevicePlatform, + available: Schema.Boolean, + reason: Schema.optional(Schema.String), +}); +export type DevicePlatformAvailability = typeof DevicePlatformAvailability.Type; + +export const DeviceHostSummary = Schema.Struct({ + id: DeviceHostId, + kind: Schema.Literals(["local"]), + label: TrimmedNonEmptyString, + platforms: Schema.Array(DevicePlatformAvailability), + hubInstalled: Schema.Boolean, + agentDeviceInstalled: Schema.Boolean, +}); +export type DeviceHostSummary = typeof DeviceHostSummary.Type; + +/** + * Lifecycle of the helper processes on a host. Tools are installed on first + * use, so a fresh install spends a while in `installing` before any device can + * stream; the UI shows that instead of an empty picker. + */ +export const DeviceHostStatus = Schema.Literals([ + "disabled", + "idle", + "installing", + "starting", + "ready", + "failed", +]); +export type DeviceHostStatus = typeof DeviceHostStatus.Type; + +/** + * A device a thread is looking at. One session per (thread, device); the same + * device may be open in several threads, since the stream is shared. + */ +export const DeviceSession = Schema.Struct({ + threadId: ThreadId, + hostId: DeviceHostId, + deviceId: DeviceId, + platform: DevicePlatform, + openedAt: Schema.String, +}); +export type DeviceSession = typeof DeviceSession.Type; + +export const DeviceServiceState = Schema.Struct({ + hosts: Schema.Array(DeviceHostSummary), + hostStatus: DeviceHostStatus, + hostStatusDetail: Schema.optional(Schema.String), + devices: Schema.Array(DeviceSummary), + sessions: Schema.Array(DeviceSession), + bootingDevices: Schema.optional( + Schema.Array(Schema.Struct({ ...DeviceSummary.fields, threadId: ThreadId })), + ), + onboardingCompleted: Schema.Boolean, + agentAccessEnabled: Schema.Boolean, + /** Origin-relative path the client prefixes to hub routes. */ + hubBasePath: Schema.String, + revision: Schema.Int, +}); +export type DeviceServiceState = typeof DeviceServiceState.Type; + +export const DeviceListInput = Schema.Struct({}); +export type DeviceListInput = typeof DeviceListInput.Type; + +export const DeviceConfigureInput = Schema.Struct({ + enabled: Schema.optional(Schema.Boolean), + agentAccessEnabled: Schema.optional(Schema.Boolean), + onboardingCompleted: Schema.optional(Schema.Boolean), +}); +export type DeviceConfigureInput = typeof DeviceConfigureInput.Type; + +export const DeviceOpenInput = Schema.Struct({ + threadId: ThreadId, + hostId: Schema.optional(DeviceHostId), + deviceId: DeviceId, + platform: DevicePlatform, + /** Boot the simulator or emulator when it is not running. Defaults to true. */ + boot: Schema.optional(Schema.Boolean), +}); +export type DeviceOpenInput = typeof DeviceOpenInput.Type; + +export const DeviceCloseInput = Schema.Struct({ + threadId: ThreadId, + /** Omit to close every device session for the thread. */ + deviceId: Schema.optional(DeviceId), + /** Also shut the simulator or emulator down. Defaults to false. */ + shutdown: Schema.optional(Schema.Boolean), +}); +export type DeviceCloseInput = typeof DeviceCloseInput.Type; + +export const DeviceShutdownInput = Schema.Struct({ + hostId: Schema.optional(DeviceHostId), + deviceId: DeviceId, + platform: DevicePlatform, +}); +export type DeviceShutdownInput = typeof DeviceShutdownInput.Type; + +// Device settings and actions. Each setting names the platforms that support +// it; the panel hides the rest. Values are normalized across platforms where +// both have the concept (appearance, text size) and platform-specific where +// only one does. + +export const DeviceAppearance = Schema.Literals(["light", "dark"]); +export type DeviceAppearance = typeof DeviceAppearance.Type; + +/** + * iOS content-size categories map onto twelve steps; Android `font_scale` + * is continuous. Four shared steps cover what people actually reach for. + */ +export const DeviceTextSize = Schema.Literals(["small", "default", "large", "extra-large"]); +export type DeviceTextSize = typeof DeviceTextSize.Type; + +export const DeviceColorFilter = Schema.Literals([ + "none", + "grayscale", + "red-green", + "green-red", + "blue-yellow", +]); +export type DeviceColorFilter = typeof DeviceColorFilter.Type; + +export const DeviceOrientation = Schema.Literals([ + "portrait", + "landscape_left", + "portrait_upside_down", + "landscape_right", +]); +export type DeviceOrientation = typeof DeviceOrientation.Type; + +/** Current values as read from the device; `undefined` means unsupported or unread. */ +export const DeviceSettings = Schema.Struct({ + appearance: Schema.optional(DeviceAppearance), + textSize: Schema.optional(DeviceTextSize), + reduceMotion: Schema.optional(Schema.Boolean), + increaseContrast: Schema.optional(Schema.Boolean), + reduceTransparency: Schema.optional(Schema.Boolean), + showBorders: Schema.optional(Schema.Boolean), + voiceOver: Schema.optional(Schema.Boolean), + liquidGlass: Schema.optional(Schema.Literals(["clear", "tinted"])), + colorFilter: Schema.optional(DeviceColorFilter), + networkEnabled: Schema.optional(Schema.Boolean), + location: Schema.optional( + Schema.NullOr(Schema.Struct({ latitude: Schema.Number, longitude: Schema.Number })), + ), +}); +export type DeviceSettings = typeof DeviceSettings.Type; + +/** The app in the foreground, when the platform can tell us. */ +export const DeviceForegroundApp = Schema.Struct({ + id: Schema.String, + name: Schema.optional(Schema.String), + version: Schema.optional(Schema.String), +}); +export type DeviceForegroundApp = typeof DeviceForegroundApp.Type; + +export const DeviceDetail = Schema.Struct({ + hostId: DeviceHostId, + deviceId: DeviceId, + settings: DeviceSettings, + foregroundApp: Schema.NullOr(DeviceForegroundApp), + readAt: Schema.String, +}); +export type DeviceDetail = typeof DeviceDetail.Type; + +export const DevicePermission = Schema.Literals([ + "camera", + "microphone", + "photos", + "contacts", + "calendar", + "reminders", + "location", + "notifications", + "motion", + "media-library", + "faceid", +]); +export type DevicePermission = typeof DevicePermission.Type; + +const DeviceTarget = { + hostId: Schema.optional(DeviceHostId), + deviceId: DeviceId, +}; + +export const DeviceActionInput = Schema.Union([ + Schema.Struct({ + ...DeviceTarget, + type: Schema.Literal("setAppearance"), + value: DeviceAppearance, + }), + Schema.Struct({ ...DeviceTarget, type: Schema.Literal("setTextSize"), value: DeviceTextSize }), + Schema.Struct({ + ...DeviceTarget, + type: Schema.Literal("setToggle"), + setting: Schema.Literals([ + "reduceMotion", + "increaseContrast", + "reduceTransparency", + "showBorders", + "voiceOver", + "networkEnabled", + ]), + value: Schema.Boolean, + }), + Schema.Struct({ + ...DeviceTarget, + type: Schema.Literal("setLiquidGlass"), + value: Schema.Literals(["clear", "tinted"]), + }), + Schema.Struct({ + ...DeviceTarget, + type: Schema.Literal("setColorFilter"), + value: DeviceColorFilter, + }), + Schema.Struct({ + ...DeviceTarget, + type: Schema.Literal("setOrientation"), + value: DeviceOrientation, + }), + Schema.Struct({ + ...DeviceTarget, + type: Schema.Literal("setLocation"), + latitude: Schema.Number.check(Schema.isBetween({ minimum: -90, maximum: 90 })), + longitude: Schema.Number.check(Schema.isBetween({ minimum: -180, maximum: 180 })), + }), + Schema.Struct({ ...DeviceTarget, type: Schema.Literal("clearLocation") }), + Schema.Struct({ + ...DeviceTarget, + type: Schema.Literal("setPermission"), + appId: TrimmedNonEmptyString, + permission: DevicePermission, + decision: Schema.Literals(["grant", "revoke", "reset"]), + }), + Schema.Struct({ ...DeviceTarget, type: Schema.Literal("openUrl"), url: TrimmedNonEmptyString }), + Schema.Struct({ + ...DeviceTarget, + type: Schema.Literal("launchApp"), + appId: TrimmedNonEmptyString, + }), + Schema.Struct({ + ...DeviceTarget, + type: Schema.Literal("terminateApp"), + appId: TrimmedNonEmptyString, + }), + Schema.Struct({ ...DeviceTarget, type: Schema.Literal("shake") }), + Schema.Struct({ + ...DeviceTarget, + type: Schema.Literal("sendPush"), + appId: TrimmedNonEmptyString, + /** APNs-style payload; a bare string becomes the alert body. */ + payload: Schema.Union([Schema.String, Schema.Record(Schema.String, Schema.Unknown)]), + }), +]); +export type DeviceActionInput = typeof DeviceActionInput.Type; +export type DeviceActionType = DeviceActionInput["type"]; + +export const DeviceDetailInput = Schema.Struct(DeviceTarget); +export type DeviceDetailInput = typeof DeviceDetailInput.Type; + +export class DeviceHostUnavailableError extends Schema.TaggedError()( + "DeviceHostUnavailableError", + { + hostId: DeviceHostId, + reason: Schema.String, + }, +) { + override get message(): string { + return `Device host ${this.hostId} is unavailable: ${this.reason}`; + } +} + +export class DevicePlatformUnavailableError extends Schema.TaggedError()( + "DevicePlatformUnavailableError", + { + hostId: DeviceHostId, + platform: DevicePlatform, + reason: Schema.String, + }, +) { + override get message(): string { + return `${this.platform} devices are unavailable on host ${this.hostId}: ${this.reason}`; + } +} + +export class DeviceNotFoundError extends Schema.TaggedError()( + "DeviceNotFoundError", + { + hostId: DeviceHostId, + deviceId: DeviceId, + }, +) { + override get message(): string { + return `Device ${this.deviceId} was not found on host ${this.hostId}.`; + } +} + +export class DeviceBootError extends Schema.TaggedError()("DeviceBootError", { + hostId: DeviceHostId, + deviceId: DeviceId, + reason: Schema.Literals(["disk_space", "timeout", "launch_failed"]), + cause: Schema.Defect(), +}) { + override get message(): string { + const explanation = { + disk_space: "There is not enough free disk space on the environment server.", + timeout: "The device did not become ready in time.", + launch_failed: + "The simulator or emulator could not start. Check its configuration on the environment server.", + }[this.reason]; + return `Device ${this.deviceId} failed to boot: ${explanation}`; + } +} + +export class DeviceOperationError extends Schema.TaggedError()( + "DeviceOperationError", + { + operation: Schema.String, + reason: Schema.Literals([ + "command_failed", + "request_failed", + "invalid_payload", + "settings_failed", + "hub_rejected", + ]), + exitCode: Schema.optional(Schema.Number), + cause: Schema.Defect(), + }, +) { + override get message(): string { + const explanation = { + command_failed: `The device command failed${this.exitCode === undefined ? "" : ` (exit code ${this.exitCode})`}.`, + request_failed: "Could not communicate with device support. Try refreshing devices.", + invalid_payload: "The device request could not be encoded.", + settings_failed: "Could not read or save device settings.", + hub_rejected: "The device hub could not complete the request.", + }[this.reason]; + return `Device ${this.operation} failed: ${explanation}`; + } +} + +export class DeviceActionUnavailableError extends Schema.TaggedError()( + "DeviceActionUnavailableError", + { + operation: Schema.String, + platform: DevicePlatform, + reason: Schema.Literals(["unsupported", "helper_missing"]), + }, +) { + override get message(): string { + return this.reason === "helper_missing" + ? `Device ${this.operation} requires a helper missing from this install. Set up device support again.` + : `Device ${this.operation} is not supported on ${this.platform}.`; + } +} + +export const DeviceError = Schema.Union([ + DeviceHostUnavailableError, + DevicePlatformUnavailableError, + DeviceNotFoundError, + DeviceBootError, + DeviceOperationError, + DeviceActionUnavailableError, +]); +export type DeviceError = typeof DeviceError.Type; + +// MCP tool shapes. Kept next to the RPC shapes so the tool surface and the +// panel describe devices the same way. + +export const DeviceToolListResult = Schema.Struct({ + hosts: Schema.Array(DeviceHostSummary), + devices: Schema.Array(DeviceSummary), + /** Devices already open in this thread's Device panel. */ + open: Schema.Array(Schema.Struct({ hostId: DeviceHostId, deviceId: DeviceId })), +}); +export type DeviceToolListResult = typeof DeviceToolListResult.Type; + +export const DeviceToolOpenInput = Schema.Struct({ + deviceId: Schema.optional( + DeviceId.annotate({ + description: + "Simulator udid or emulator serial from device_list. Omit to use the booted device for the platform, or the most recently used one.", + }), + ), + platform: Schema.optional( + DevicePlatform.annotate({ + description: "Required when deviceId is omitted and both platforms are available.", + }), + ), + hostId: Schema.optional( + DeviceHostId.annotate({ description: "Device host from device_list. Defaults to local." }), + ), +}).annotate({ + description: + "Boots the device if needed, starts its live stream, and opens the Device panel so the user can watch. Returns how to drive it with the agent-device CLI.", +}); +export type DeviceToolOpenInput = typeof DeviceToolOpenInput.Type; + +export const DeviceToolOpenResult = Schema.Struct({ + device: DeviceSummary, + /** Ready-to-run agent-device invocation pinned to this device. */ + agentDevice: Schema.Struct({ + command: Schema.String, + /** Flags that pin every command to this device, e.g. `--udid `. */ + targetArgs: Schema.Array(Schema.String), + }), + quickStart: Schema.String, +}); +export type DeviceToolOpenResult = typeof DeviceToolOpenResult.Type; + +export const DeviceToolTargetInput = Schema.Struct({ + deviceId: Schema.optional( + DeviceId.annotate({ + description: + "Device from device_list. Omit to use the device most recently opened in this thread.", + }), + ), + hostId: Schema.optional(DeviceHostId), +}); +export type DeviceToolTargetInput = typeof DeviceToolTargetInput.Type; + +export const DeviceToolScreenshotResult = Schema.Struct({ + device: DeviceSummary, + screenshot: Schema.Struct({ + mimeType: Schema.Literal("image/png"), + data: Schema.String, + width: Schema.Int, + height: Schema.Int, + }), +}); +export type DeviceToolScreenshotResult = typeof DeviceToolScreenshotResult.Type; + +export const DeviceToolCloseInput = Schema.Struct({ + deviceId: Schema.optional( + DeviceId.annotate({ + description: "Device to close. Omit to close every device in this thread.", + }), + ), + hostId: Schema.optional(DeviceHostId), + shutdown: Schema.optional( + Schema.Boolean.annotate({ + description: "Also power the simulator or emulator off. Defaults to false.", + }), + ), +}); +export type DeviceToolCloseInput = typeof DeviceToolCloseInput.Type; + +export class DeviceToolUnavailableError extends Schema.TaggedError()( + "DeviceToolUnavailableError", + { + reason: Schema.String, + }, +) { + override get message(): string { + return this.reason; + } +} + +export const DeviceToolError = Schema.Union([ + DeviceToolUnavailableError, + DeviceHostUnavailableError, + DevicePlatformUnavailableError, + DeviceNotFoundError, + DeviceBootError, + DeviceOperationError, + DeviceActionUnavailableError, +]); +export type DeviceToolError = typeof DeviceToolError.Type; diff --git a/packages/contracts/src/index.ts b/packages/contracts/src/index.ts index 74a1b4939f1a..32ac53dae9ba 100644 --- a/packages/contracts/src/index.ts +++ b/packages/contracts/src/index.ts @@ -34,6 +34,7 @@ export * from "./assets.ts"; export * from "./review.ts"; export * from "./browserImport.ts"; export * from "./browserProfile.ts"; +export * from "./device.ts"; export * from "./preview.ts"; export * from "./previewAutomation.ts"; export * from "./resourceTelemetry.ts"; diff --git a/packages/contracts/src/rpc.ts b/packages/contracts/src/rpc.ts index 9dbcaa9f4164..d80d3b3949ef 100644 --- a/packages/contracts/src/rpc.ts +++ b/packages/contracts/src/rpc.ts @@ -179,6 +179,19 @@ import { PreviewResizeInput, PreviewSessionSnapshot, } from "./preview.ts"; +import { + DeviceActionInput, + DeviceCloseInput, + DeviceConfigureInput, + DeviceDetail, + DeviceDetailInput, + DeviceError, + DeviceListInput, + DeviceOpenInput, + DeviceServiceState, + DeviceSession, + DeviceShutdownInput, +} from "./device.ts"; import { PreviewAutomationError, PreviewAutomationHost, @@ -310,6 +323,15 @@ export const WS_METHODS = { previewAutomationRespond: "previewAutomation.respond", previewAutomationFocusHost: "previewAutomation.focusHost", + // Device methods + deviceConfigure: "device.configure", + deviceList: "device.list", + deviceOpen: "device.open", + deviceClose: "device.close", + deviceShutdown: "device.shutdown", + deviceDetail: "device.detail", + deviceAction: "device.action", + // Server meta serverProbe: "server.probe", serverGetConfig: "server.getConfig", @@ -374,6 +396,7 @@ export const WS_METHODS = { subscribeTerminalMetadata: "subscribeTerminalMetadata", subscribePreviewEvents: "subscribePreviewEvents", subscribeDiscoveredLocalServers: "subscribeDiscoveredLocalServers", + subscribeDeviceState: "subscribeDeviceState", subscribeServerConfig: "subscribeServerConfig", subscribeServerLifecycle: "subscribeServerLifecycle", subscribeAuthAccess: "subscribeAuthAccess", @@ -1062,6 +1085,53 @@ const WsSubscribeDiscoveredLocalServersRpc = Rpc.make(WS_METHODS.subscribeDiscov stream: true, }); +const WsDeviceListRpc = Rpc.make(WS_METHODS.deviceList, { + payload: DeviceListInput, + success: DeviceServiceState, + error: Schema.Union([DeviceError, EnvironmentAuthorizationError]), +}); + +const WsDeviceConfigureRpc = Rpc.make(WS_METHODS.deviceConfigure, { + payload: DeviceConfigureInput, + success: DeviceServiceState, + error: Schema.Union([DeviceError, EnvironmentAuthorizationError]), +}); + +const WsDeviceOpenRpc = Rpc.make(WS_METHODS.deviceOpen, { + payload: DeviceOpenInput, + success: DeviceSession, + error: Schema.Union([DeviceError, EnvironmentAuthorizationError]), +}); + +const WsDeviceCloseRpc = Rpc.make(WS_METHODS.deviceClose, { + payload: DeviceCloseInput, + error: Schema.Union([DeviceError, EnvironmentAuthorizationError]), +}); + +const WsDeviceShutdownRpc = Rpc.make(WS_METHODS.deviceShutdown, { + payload: DeviceShutdownInput, + error: Schema.Union([DeviceError, EnvironmentAuthorizationError]), +}); + +const WsDeviceDetailRpc = Rpc.make(WS_METHODS.deviceDetail, { + payload: DeviceDetailInput, + success: DeviceDetail, + error: Schema.Union([DeviceError, EnvironmentAuthorizationError]), +}); + +const WsDeviceActionRpc = Rpc.make(WS_METHODS.deviceAction, { + payload: DeviceActionInput, + success: DeviceDetail, + error: Schema.Union([DeviceError, EnvironmentAuthorizationError]), +}); + +const WsSubscribeDeviceStateRpc = Rpc.make(WS_METHODS.subscribeDeviceState, { + payload: Schema.Struct({}), + success: DeviceServiceState, + error: EnvironmentAuthorizationError, + stream: true, +}); + const WsOrchestrationDispatchCommandRpc = Rpc.make(ORCHESTRATION_WS_METHODS.dispatchCommand, { payload: ClientOrchestrationCommand, success: OrchestrationRpcSchemas.dispatchCommand.output, @@ -1290,6 +1360,14 @@ export const WsRpcGroup = RpcGroup.make( WsPreviewAutomationFocusHostRpc, WsSubscribePreviewEventsRpc, WsSubscribeDiscoveredLocalServersRpc, + WsDeviceConfigureRpc, + WsDeviceListRpc, + WsDeviceOpenRpc, + WsDeviceCloseRpc, + WsDeviceShutdownRpc, + WsDeviceDetailRpc, + WsDeviceActionRpc, + WsSubscribeDeviceStateRpc, WsSubscribeServerConfigRpc, WsSubscribeServerLifecycleRpc, WsSubscribeAuthAccessRpc, diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index 3491103da94f..1bfcaadc5d6e 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -962,6 +962,22 @@ export const ServerSettings = Schema.Struct({ defaultModelSelection: Schema.NullOr(ModelSelection).pipe( Schema.withDecodingDefault(Effect.succeed(null)), ), + /** + * Whether agents may drive simulators and emulators. Gates the `device_*` + * MCP tools and the preconfigured `agent-device` CLI the same way + * `enableAgentBrowserAccess` gates the browser: server-authoritative, applied + * when the provider session is prepared. The user's own Device panel is + * unaffected. + */ + enableAgentDeviceAccess: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(false))), + /** + * Whether this server may install and run T3's device helper processes. + * Kept separate from agent access so enabling the user's Device panel does + * not also grant providers control of simulators and emulators. + */ + enableDeviceSupport: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(false))), + /** Whether the server-local Device panel setup flow has been completed. */ + deviceOnboardingCompleted: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(false))), sidebarAutoSettleAfterDays: Schema.NullOr(SidebarAutoSettleAfterDays).pipe( Schema.withDecodingDefault(Effect.succeed(DEFAULT_SIDEBAR_AUTO_SETTLE_AFTER_DAYS)), ), @@ -1234,6 +1250,9 @@ export const ServerSettingsPatch = Schema.Struct({ Schema.Record(ProjectId, Schema.NullOr(Schema.Boolean)), ), defaultModelSelection: Schema.optionalKey(Schema.NullOr(ModelSelection)), + enableAgentDeviceAccess: Schema.optionalKey(Schema.Boolean), + enableDeviceSupport: Schema.optionalKey(Schema.Boolean), + deviceOnboardingCompleted: Schema.optionalKey(Schema.Boolean), sidebarAutoSettleAfterDays: Schema.optionalKey(Schema.NullOr(SidebarAutoSettleAfterDays)), sidebarAutoSettleOnMerge: Schema.optionalKey(Schema.Boolean), backgroundActivity: Schema.optionalKey(