diff --git a/apple/macos-helper/Sources/AgentDeviceMacOSHelper/AppWindowScreenshot.swift b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/AppWindowScreenshot.swift index da4c6247c7..50d38069b0 100644 --- a/apple/macos-helper/Sources/AgentDeviceMacOSHelper/AppWindowScreenshot.swift +++ b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/AppWindowScreenshot.swift @@ -10,20 +10,15 @@ func captureAppWindowScreenshot(app: NSRunningApplication, outPath: String) thro throw HelperError.commandFailed("app window screenshots require macOS 14 or newer") } let pid = app.processIdentifier - let frontWindowID = frontWindowNumber(pid: pid) + let front = try frontWindow(pid: pid) let content = try awaitShareableContent() - let candidates = content.windows.filter { - $0.owningApplication?.processID == pid && $0.windowLayer == 0 && $0.frame.width > 0 - && $0.frame.height > 0 - } - guard - let window = candidates.first(where: { Int($0.windowID) == frontWindowID }) - ?? candidates.max(by: { $0.frame.width * $0.frame.height < $1.frame.width * $1.frame.height }) - else { - throw HelperError.commandFailed( - "screenshot could not find a window for the app", - details: ["reason": "window-not-found", "bundleId": app.bundleIdentifier ?? ""] - ) + let candidates = content.windows.filter { $0.owningApplication?.processID == pid } + let window: SCWindow + switch frontShareableWindow(candidates, front: front, windowID: { Int($0.windowID) }) { + case .success(let shareable): + window = shareable + case .failure(let refusal): + throw refusal.error(bundleId: app.bundleIdentifier) } let filter = SCContentFilter(desktopIndependentWindow: window) @@ -52,6 +47,58 @@ func captureAppWindowScreenshot(app: NSRunningApplication, outPath: String) thro try writePNG(capturedImage, to: outPath) } +/// Why an app screenshot refuses, and the error the host reports for it: reason, remedy, and the +/// front window it refused. The capture is the app's front window or nothing; another window of +/// the app would show state the user is not looking at. +enum AppWindowScreenshotRefusal: Error { + case windowNotFound + case frontWindowNotShareable(frontWindowId: Int) + case frontWindowCaptureExcluded(frontWindowId: Int) + + func error(bundleId: String?) -> HelperError { + var details = ["bundleId": bundleId ?? ""] + let message: String + switch self { + case .windowNotFound: + details["reason"] = "window-not-found" + details["hint"] = + "Open or unminimize a window of the app on the current Space, then retry the screenshot." + message = "screenshot could not find an on-screen window for the app" + case .frontWindowCaptureExcluded(let frontWindowId): + details["reason"] = "front-window-capture-excluded" + details["frontWindowId"] = String(frontWindowId) + details["hint"] = + "The app excludes the front window from screen capture. Run `snapshot` to read the window's UI tree; retrying the excluded window will not help." + message = + "screenshot could not capture the app's front window: the app excludes it from capture" + case .frontWindowNotShareable(let frontWindowId): + details["reason"] = "front-window-not-shareable" + details["frontWindowId"] = String(frontWindowId) + details["hint"] = + "The front window is unavailable to screen capture. If it is opening or closing, retry after it settles." + message = + "screenshot could not capture the app's front window: ScreenCaptureKit does not offer it" + } + return .commandFailed(message, details: details) + } +} + +/// The shareable candidate that is the app's front window, or the reason the screenshot refuses. +func frontShareableWindow( + _ candidates: [Window], + front: OnScreenWindow?, + windowID: (Window) -> Int +) -> Result { + guard let front else { return .failure(.windowNotFound) } + guard front.sharingState != 0 else { + return .failure(.frontWindowCaptureExcluded(frontWindowId: front.number)) + } + guard let shareable = candidates.first(where: { windowID($0) == front.number }) else { + return .failure(.frontWindowNotShareable(frontWindowId: front.number)) + } + return .success(shareable) +} + @available(macOS 14.0, *) private func awaitShareableContent() throws -> SCShareableContent { let semaphore = DispatchSemaphore(value: 0) diff --git a/apple/macos-helper/Sources/AgentDeviceMacOSHelper/BackgroundInteraction.swift b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/BackgroundInteraction.swift index 349426f54c..f906ac2a15 100644 --- a/apple/macos-helper/Sources/AgentDeviceMacOSHelper/BackgroundInteraction.swift +++ b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/BackgroundInteraction.swift @@ -206,7 +206,7 @@ func scrollInBackground( guard isVertical || direction == "left" || direction == "right" else { throw HelperError.invalidArgs("scroll requires --direction ") } - guard let frame = frontWindowFrame(pid: app.processIdentifier) else { + guard let frame = try frontWindowFrame(pid: app.processIdentifier) else { throw HelperError.commandFailed( "scroll could not resolve an on-screen window", details: ["reason": "window-not-found", "bundleId": app.bundleIdentifier ?? ""] @@ -306,10 +306,7 @@ private func perform(_ target: PressTarget) -> BackgroundDeliveryMechanism? { /// own on-screen windows, its menus and sheets included, and never on a menu bar element: the app /// owns its menu bar's window too, so the window check alone does not exclude it. private func requireAppWindowPoint(_ point: CGPoint, app: NSRunningApplication) throws { - guard - let info = CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID) - as? [[String: Any]] - else { + guard let info = appWindowInfo() else { throw refusal(.noAccessibleTarget, "the app's on-screen windows could not be read", app: app) } let inAppWindow = info.contains { entry in @@ -363,13 +360,14 @@ private func pressSearchChain(from hit: AXUIElement) -> [AXUIElement] { /// The one window a point action may act in: the window the app's hit test landed in, else the /// app's front on-screen window. A fallback target never comes from a window behind it, and the -/// response names the window so the host can check it. +/// response names the window so the host can check it. The caller's `requireAppWindowPoint` has +/// already read the window list, so a list that turned unreadable here is treated as no window. private func actionWindow(app: NSRunningApplication, hit: AXUIElement?) -> AXUIElement? { if let hit { if role(of: hit) == "AXWindow" { return hit } if let window = elementAttribute(hit, attribute: kAXWindowAttribute as String) { return window } } - guard let front = frontWindowFrame(pid: app.processIdentifier) else { return nil } + guard let front = try? frontWindowFrame(pid: app.processIdentifier) else { return nil } return windows(of: AXUIElementCreateApplication(app.processIdentifier)).first { window in guard let rect = rectAttribute(window) else { return false } return abs(rect.x - front.minX) < 1 && abs(rect.y - front.minY) < 1 @@ -482,39 +480,57 @@ private func numberAttribute(_ element: AXUIElement, attribute: String) -> Doubl return number.doubleValue } -private struct OnScreenWindow { +struct OnScreenWindow { let number: Int let bounds: CGRect + let sharingState: Int? } -/// The pid's normal-layer windows, front to back. Bounds and owners need no Screen Recording -/// permission; only titles do. -private func onScreenWindows(pid: pid_t) -> [OnScreenWindow] { - guard - let info = CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID) - as? [[String: Any]] - else { - return [] - } - return info.compactMap { entry in +/// Only normal-layer windows qualify; app-modal alerts at level 8 are excluded. +let frontWindowLayer = 0 + +/// The pid's visible, nonempty normal-layer windows, front to back. +func onScreenWindows(pid: pid_t, windowInfo: [[String: Any]]) -> [OnScreenWindow] { + windowInfo.compactMap { entry in guard (entry[kCGWindowOwnerPID as String] as? Int32) == pid, - (entry[kCGWindowLayer as String] as? Int) == 0, + (entry[kCGWindowLayer as String] as? Int) == frontWindowLayer, + (entry[kCGWindowIsOnscreen as String] as? Bool) == true, + let alpha = entry[kCGWindowAlpha as String] as? Double, alpha > 0, let number = entry[kCGWindowNumber as String] as? Int, let boundsDict = entry[kCGWindowBounds as String] as? NSDictionary, - let bounds = CGRect(dictionaryRepresentation: boundsDict as CFDictionary) + let bounds = CGRect(dictionaryRepresentation: boundsDict as CFDictionary), + bounds.width > 0, bounds.height > 0 else { return nil } - return OnScreenWindow(number: number, bounds: bounds) + return OnScreenWindow( + number: number, bounds: bounds, sharingState: entry[kCGWindowSharingState as String] as? Int) } } -func frontWindowNumber(pid: pid_t) -> Int? { - onScreenWindows(pid: pid).first?.number +func appWindowInfo() -> [[String: Any]]? { + CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID) + as? [[String: Any]] +} + +/// The app's front on-screen window: frontmost among its eligible windows, or nil when it has +/// none. A window list that cannot be read is a refusal, not an empty list: the caller must not +/// report "no window" for a WindowServer it could not ask. +func frontWindow(pid: pid_t, windowInfo: [[String: Any]]? = appWindowInfo()) throws -> OnScreenWindow? { + guard let windowInfo else { + throw HelperError.commandFailed( + "the app's on-screen windows could not be read", + details: [ + "reason": "window-list-unreadable", + "hint": "The app's on-screen window list could not be read. Retry the command.", + ] + ) + } + return onScreenWindows(pid: pid, windowInfo: windowInfo).first } -private func frontWindowFrame(pid: pid_t) -> CGRect? { - onScreenWindows(pid: pid).first?.bounds +func frontWindowFrame(pid: pid_t, windowInfo: [[String: Any]]? = appWindowInfo()) throws -> CGRect? { + try frontWindow(pid: pid, windowInfo: windowInfo)?.bounds } private let returnVirtualKey: CGKeyCode = 36 diff --git a/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/AppWindowScreenshotTests.swift b/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/AppWindowScreenshotTests.swift new file mode 100644 index 0000000000..f8698ad597 --- /dev/null +++ b/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/AppWindowScreenshotTests.swift @@ -0,0 +1,188 @@ +import CoreGraphics +import XCTest + +@testable import AgentDeviceMacOSHelper + +private struct Candidate: Equatable { + let id: Int + let area: Double +} + +/// An app screenshot captures the front window or refuses; it never substitutes another window. +final class AppWindowScreenshotTests: XCTestCase { + func testCaptureIdentityAndScrollAndClickFallbackFrameSelectTheSameEligibleFrontWindow() throws { + let windows = [ + windowInfo(id: 1, pid: 99), + windowInfo(id: 2, layer: 1), + windowInfo(id: 3, onScreen: false), + windowInfo(id: 4, alpha: 0), + windowInfo(id: 5, width: 0), + windowInfo(id: 6, height: 0), + windowInfo(id: 7, alpha: 0.5, width: 20, height: 20), + windowInfo(id: 8), + ] + + let front = try frontWindow(pid: 42, windowInfo: windows) + let selected = frontShareableWindow([7, 8], front: front, windowID: { $0 }) + + // Scroll and the click actionWindow fallback both consume frontWindowFrame. + XCTAssertEqual( + try frontWindowFrame(pid: 42, windowInfo: windows), + CGRect(x: 10, y: 10, width: 20, height: 20)) + XCTAssertEqual(front?.number, 7) + XCTAssertEqual(try selected.get(), 7) + } + + func testSkipsAppModalAlertInFrontOfNormalLayerWindow() throws { + let windows = [windowInfo(id: 8, layer: 8), windowInfo(id: 3)] + let front = try frontWindow(pid: 42, windowInfo: windows) + let selected = frontShareableWindow([8, 3], front: front, windowID: { $0 }) + + XCTAssertEqual(front?.number, 3) + XCTAssertEqual(try selected.get(), 3) + XCTAssertEqual(try frontWindowFrame(pid: 42, windowInfo: windows), front?.bounds) + } + + func testRefusesWhenOnlyInvisibleOrEmptyWindowsRemain() throws { + let windows = [ + windowInfo(id: 1, onScreen: false), + windowInfo(id: 2, alpha: 0), + windowInfo(id: 3, width: 0), + windowInfo(id: 4, height: 0), + ] + let front = try frontWindow(pid: 42, windowInfo: windows) + let selected = frontShareableWindow([5], front: front, windowID: { $0 }) + + XCTAssertNil(try frontWindowFrame(pid: 42, windowInfo: windows)) + XCTAssertNil(front) + XCTAssertThrowsError(try selected.get()) { error in + assertRefusal(error, reason: "window-not-found", hint: "Open or unminimize") + } + } + + func testUnreadableWindowListRefusesBeforeCaptureOrScrollCanReportNoWindow() { + XCTAssertThrowsError(try frontWindow(pid: 42, windowInfo: nil)) { error in + assertUnreadableList(error) + } + XCTAssertThrowsError(try frontWindowFrame(pid: 42, windowInfo: nil)) { error in + assertUnreadableList(error) + } + XCTAssertNil(try frontWindow(pid: 42, windowInfo: [])) + XCTAssertNil(try frontWindowFrame(pid: 42, windowInfo: [])) + } + + func testSelectsTheFrontWindowEvenWhenAnotherIsLarger() { + let candidates = [Candidate(id: 7, area: 10_000), Candidate(id: 3, area: 100)] + + let selected = frontShareableWindow(candidates, front: onScreenWindow(id: 3), windowID: \.id) + + XCTAssertEqual(try selected.get(), Candidate(id: 3, area: 100)) + } + + func testRefusesWhenTheFrontWindowIsNotShareable() { + let candidates = [Candidate(id: 7, area: 10_000), Candidate(id: 8, area: 100)] + + let selected = frontShareableWindow(candidates, front: onScreenWindow(id: 3), windowID: \.id) + + XCTAssertThrowsError(try selected.get()) { error in + assertRefusal( + error, + reason: "front-window-not-shareable", + hint: "retry after it settles", + frontWindowId: "3") + } + } + + func testCaptureExclusionSurvivesWindowSelectionAndNeverSubstitutesAnotherWindow() throws { + let windows = [windowInfo(id: 3, sharingState: 0), windowInfo(id: 7)] + let front = try frontWindow(pid: 42, windowInfo: windows) + XCTAssertEqual(front?.number, 3) + XCTAssertEqual(front?.sharingState, 0) + + for candidates in [[3, 7], [7]] { + let selected = frontShareableWindow(candidates, front: front, windowID: { $0 }) + XCTAssertThrowsError(try selected.get()) { error in + assertRefusal( + error, + reason: "front-window-capture-excluded", + hint: "Run `snapshot` to read the window", + frontWindowId: "3") + } + } + + var entry = windowInfo(id: 3) + entry.removeValue(forKey: kCGWindowSharingState as String) + let unknown = try frontWindow(pid: 42, windowInfo: [entry]) + let selected = frontShareableWindow([3], front: unknown, windowID: { $0 }) + XCTAssertEqual(try selected.get(), 3) + } + + func testRefusesWhenTheAppHasNoOnScreenWindow() { + let selected = frontShareableWindow( + [Candidate(id: 7, area: 10_000)], front: nil, windowID: \.id) + + XCTAssertThrowsError(try selected.get()) { error in + assertRefusal(error, reason: "window-not-found", hint: "Open or unminimize") + } + } +} + +/// The refusal carries the reason and the remedy the agent runs next, alongside the front window +/// it refused. +private func assertRefusal( + _ error: Error, + reason: String, + hint: String, + frontWindowId: String? = nil, + file: StaticString = #filePath, + line: UInt = #line +) { + guard let refusal = error as? AppWindowScreenshotRefusal else { + return XCTFail("expected an AppWindowScreenshotRefusal, got \(error)", file: file, line: line) + } + guard + case HelperError.commandFailed(_, let details) = refusal.error( + bundleId: "com.apple.TextEdit") + else { + return XCTFail("expected a commandFailed refusal error", file: file, line: line) + } + XCTAssertEqual(details["reason"], reason, file: file, line: line) + XCTAssertEqual(details["bundleId"], "com.apple.TextEdit", file: file, line: line) + XCTAssertEqual(details["frontWindowId"], frontWindowId, file: file, line: line) + XCTAssertTrue( + (details["hint"] ?? "").contains(hint), + "hint \(details["hint"] ?? "") lacks \(hint)", + file: file, line: line) +} + +private func assertUnreadableList( + _ error: Error, file: StaticString = #filePath, line: UInt = #line +) { + guard case HelperError.commandFailed(_, let details) = error else { + return XCTFail("expected a typed window-list refusal, got \(error)", file: file, line: line) + } + XCTAssertEqual(details["reason"], "window-list-unreadable", file: file, line: line) + XCTAssertEqual( + details["hint"], + "The app's on-screen window list could not be read. Retry the command.", + file: file, line: line) +} + +private func onScreenWindow(id: Int) -> OnScreenWindow { + OnScreenWindow(number: id, bounds: .zero, sharingState: 1) +} + +private func windowInfo( + id: Int, pid: Int32 = 42, layer: Int = 0, onScreen: Bool = true, + alpha: Double = 1, sharingState: Int = 1, width: Double = 100, height: Double = 100 +) -> [String: Any] { + [ + kCGWindowNumber as String: id, + kCGWindowOwnerPID as String: pid, + kCGWindowLayer as String: layer, + kCGWindowIsOnscreen as String: onScreen, + kCGWindowAlpha as String: alpha, + kCGWindowSharingState as String: sharingState, + kCGWindowBounds as String: CGRect(x: 10, y: 10, width: width, height: height).dictionaryRepresentation, + ] +} diff --git a/packages/platform-apple/src/core/__tests__/display-inventory.test.ts b/packages/platform-apple/src/core/__tests__/display-inventory.test.ts index a476dafec9..416048de0f 100644 --- a/packages/platform-apple/src/core/__tests__/display-inventory.test.ts +++ b/packages/platform-apple/src/core/__tests__/display-inventory.test.ts @@ -266,15 +266,26 @@ describe('buildInventory', () => { } }); - test('names a capture even when no panel reports primary', () => { + test('names the sole lit panel when no panel reports primary', () => { const inventory = buildInventory([ - { ...outerPanel, primary: false }, + { ...outerPanel, power: 'dark', primary: false }, { ...innerPanel, power: 'lit', primary: false }, ]); assert.equal(inventory.multiScreen, true); assert.equal(inventory.ambiguous, true); - // A capture is still named: never the implicit black-producing default. - assert.ok(inventory.activeDisplay); + assert.equal(inventory.activeDisplay?.name, 'LCD-1'); + }); + + test('names no panel when neither primary nor power identifies one', () => { + for (const power of ['lit', 'dark', 'unknown'] as const) { + const inventory = buildInventory([ + { ...outerPanel, power, primary: false }, + { ...innerPanel, power, primary: false }, + ]); + assert.equal(inventory.multiScreen, true); + assert.equal(inventory.ambiguous, true); + assert.equal(inventory.activeDisplay, undefined, power); + } }); test('keeps choosing the lit panel when a third panel is present', () => { @@ -476,6 +487,58 @@ describe('resolveAppleCaptureTarget', () => { assert.equal(target.displayRotation, undefined); }); + /** The closed-Duo payload with each panel's `primary` flag and backlight replaced. */ + function duoReporting(panels: Record) { + return JSON.stringify({ + result: { + displays: IPHONE_DUO_CLOSED.result.displays.map((entry) => ({ + ...entry, + ...panels[entry.displayId], + })), + }, + }); + } + + test('names the primary panel when panel power is ambiguous', async () => { + mockReadHostTextFile.mockResolvedValue( + duoReporting({ + 1: { primary: true, backlightState: 'activeOn' }, + 3: { primary: false, backlightState: 'activeOn' }, + }), + ); + assert.equal((await resolveAppleCaptureTarget(IOS_TEST_SIMULATOR)).display?.name, 'LCD'); + }); + + test('names the sole lit panel when no panel reports primary', async () => { + mockReadHostTextFile.mockResolvedValue( + duoReporting({ + 1: { primary: false, backlightState: 'off' }, + 3: { primary: false, backlightState: 'activeOn' }, + }), + ); + assert.equal((await resolveAppleCaptureTarget(IOS_TEST_SIMULATOR)).display?.name, 'LCD-1'); + }); + + test('refuses instead of picking a panel by list order when no fact identifies one', async () => { + for (const backlightState of ['activeOn', 'off', 'futureState']) { + mockReadHostTextFile.mockResolvedValue( + duoReporting({ + 1: { primary: false, backlightState }, + 3: { primary: false, backlightState }, + }), + ); + await assert.rejects( + resolveAppleCaptureTarget(IOS_TEST_SIMULATOR), + (error: unknown) => + error instanceof AppError && + error.code === 'COMMAND_FAILED' && + error.details?.reason === 'capture-panel-unidentified' && + typeof error.details?.hint === 'string', + backlightState, + ); + } + }); + test('reports nothing when CoreDevice cannot be asked', async () => { mockRunXcrun.mockResolvedValueOnce({ exitCode: 1, diff --git a/packages/platform-apple/src/core/display-inventory.ts b/packages/platform-apple/src/core/display-inventory.ts index 6dd4fbee2a..1eee41c9f1 100644 --- a/packages/platform-apple/src/core/display-inventory.ts +++ b/packages/platform-apple/src/core/display-inventory.ts @@ -1,5 +1,6 @@ import type { DeviceRotation } from '@agent-device/contracts/device'; import type { DeviceInfo } from '@agent-device/kernel/device'; +import { AppError } from '@agent-device/kernel/errors'; import { isCommandTimeoutError } from '@agent-device/host-kit/command'; import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; @@ -45,6 +46,11 @@ export type AppleDisplayInventory = { * single-screen behavior and never route through a display selector. */ multiScreen: boolean; + /** + * The panel a multi-panel capture names. Absent on a multi-panel device when + * neither panel power nor CoreDevice `primary` identifies a panel: list order is + * not identity, so the capture refuses instead. + */ activeDisplay?: AppleDeviceDisplay; /** * True on a multi-panel device whose lit panel could not be decided, so the @@ -258,7 +264,7 @@ export function buildInventory(displays: AppleDeviceDisplay[]): AppleDisplayInve // choice is the highest screen ID, which is exactly how a foldable capture comes // back all black and exits 0. When panel power is ambiguous the primary panel is // named explicitly and the ambiguity is reported, not hidden. - const captureDisplay = decided ? litPanels[0]! : (primaryPanel ?? litPanels[0] ?? panels[0]!); + const captureDisplay = decided ? litPanels[0]! : primaryPanel; const ambiguous = !decided || primaryPanel === undefined; if (ambiguous) { emitDiagnostic({ @@ -267,20 +273,27 @@ export function buildInventory(displays: AppleDeviceDisplay[]): AppleDisplayInve data: { reason: primaryPanel === undefined ? 'no-primary-panel' : 'lit-panel-count', litPanelCount: litPanels.length, - capturedDisplay: captureDisplay.name, - hint: 'Panel power did not identify exactly one lit panel, so the capture names the primary panel without claiming to know which panel the device is showing. Read the device pose from the app under test with UIHinge.status.', + ...(captureDisplay + ? { + capturedDisplay: captureDisplay.name, + hint: 'Panel power did not identify exactly one lit panel, so the capture names the primary panel without claiming to know which panel the device is showing. Read the device pose from the app under test with UIHinge.status.', + } + : { hint: UNIDENTIFIED_PANEL_HINT }), }, }); } return { displays: panels, multiScreen: true, - activeDisplay: captureDisplay, + ...(captureDisplay ? { activeDisplay: captureDisplay } : {}), ambiguous, unresolved: false, }; } +const UNIDENTIFIED_PANEL_HINT = + 'CoreDevice reported no primary panel and panel power did not identify exactly one lit panel. Run `xcrun devicectl device info displays --device `, then retry after one panel reports a lit backlightState.'; + /** * Sanity-checks the panel geometry against a foldable's shape. * @@ -315,8 +328,9 @@ function warnWhenPrimaryPanelIsNotTheSmallest(panels: AppleDeviceDisplay[]): voi * `simctl io screenshot` without `--display` silently picks the highest screen * ID, which on a closed foldable is the *dark* inner panel: the capture succeeds * and comes back black. A multi-panel device therefore always names a display - * explicitly — even when panel power is ambiguous — and only a genuinely - * single-panel device (or an unreadable toolchain) keeps the historic argv. + * explicitly — even when panel power is ambiguous — or refuses when CoreDevice + * identifies no panel. Only a genuinely single-panel device (or an unreadable + * toolchain) keeps the historic argv. */ export async function resolveAppleCaptureDisplay( device: DeviceInfo, @@ -336,7 +350,17 @@ export async function resolveAppleCaptureTarget( const inventory = await queryAppleDisplayInventory(device, options); if (inventory.unresolved) return {}; if (inventory.multiScreen) { - return inventory.activeDisplay ? { display: inventory.activeDisplay } : {}; + if (inventory.activeDisplay) return { display: inventory.activeDisplay }; + throw new AppError( + 'COMMAND_FAILED', + `CoreDevice did not identify which panel of ${device.name} to capture`, + { + deviceId: device.id, + reason: 'capture-panel-unidentified', + litPanelCount: inventory.displays.filter((display) => display.power === 'lit').length, + hint: UNIDENTIFIED_PANEL_HINT, + }, + ); } // Only a built-in panel is the screen simctl captures; an attached display says nothing about it. const panel = inventory.displays.find((entry) => entry.integrated); diff --git a/packages/platform-apple/src/os/macos/helper.test.ts b/packages/platform-apple/src/os/macos/helper.test.ts index 777c0caa92..12186afe9f 100644 --- a/packages/platform-apple/src/os/macos/helper.test.ts +++ b/packages/platform-apple/src/os/macos/helper.test.ts @@ -1,6 +1,7 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; import { macOsHelperSurface } from '@agent-device/contracts/session'; +import { AppError } from '@agent-device/kernel/errors'; import { createLocalAppleToolProvider, withAppleToolProvider } from '../../core/tool-provider.ts'; import { readFileSync } from 'node:fs'; import { @@ -16,6 +17,7 @@ import { const desktop = macOsHelperSurface('desktop', 'xctest')!; const menubar = macOsHelperSurface('menubar', 'xctest')!; const frontmostApp = macOsHelperSurface('frontmost-app', 'xctest')!; +const nativeApp = macOsHelperSurface('app', 'native')!; test('macOS helper snapshot passes cancellation to the helper process', async () => { const controller = new AbortController(); @@ -241,6 +243,51 @@ test('macOS helper screenshot argv carries only --out and --surface', async () = assert.deepEqual(receivedArgs, ['screenshot', '--out', '/tmp/out.png', '--surface', 'desktop']); }); +for (const hint of [ + 'The front window is unavailable to screen capture. If it is opening or closing, retry after it settles.', + undefined, +]) { + test(`a helper refusal carries its details and hint through untouched (${hint === undefined ? 'no hint' : 'with hint'})`, async () => { + const provider = createLocalAppleToolProvider({ + macosHelper: { + run: async () => ({ + exitCode: 1, + stdout: JSON.stringify({ + ok: false, + error: { + message: + "screenshot could not capture the app's front window: ScreenCaptureKit does not offer it", + details: { + reason: 'front-window-not-shareable', + bundleId: 'com.apple.TextEdit', + frontWindowId: '4242', + ...(hint ? { hint } : {}), + }, + }, + }), + stderr: '', + }), + }, + }); + + const error = await withAppleToolProvider(provider, async () => + runMacOsScreenshotAction('/tmp/out.png', { + surface: nativeApp, + bundleId: 'com.apple.TextEdit', + }).then( + () => undefined, + (error: unknown) => error, + ), + ); + + assert.ok(error instanceof AppError); + assert.equal(error.code, 'COMMAND_FAILED'); + assert.equal(error.details?.reason, 'front-window-not-shareable'); + assert.equal(error.details?.frontWindowId, '4242'); + assert.equal(error.details?.hint, hint); + }); +} + test('helper entry points accept only an owner-routed surface', () => { // Never invoked: each directive fails typecheck once its entry point widens back to an // unbranded or optional surface. @@ -268,7 +315,10 @@ test('the helper vocabulary matches the table the Swift helper is held to', () = ), 'utf8', ), - ) as { refusalReasons: string[]; deliveryMechanisms: string[] }; + ) as { + refusalReasons: string[]; + deliveryMechanisms: string[]; + }; assert.deepEqual([...MACOS_HELPER_REFUSAL_REASONS], fixture.refusalReasons); assert.deepEqual([...MACOS_DELIVERY_MECHANISMS], fixture.deliveryMechanisms); }); diff --git a/packages/platform-apple/src/os/macos/helper.ts b/packages/platform-apple/src/os/macos/helper.ts index 97cc51f146..62da53e366 100644 --- a/packages/platform-apple/src/os/macos/helper.ts +++ b/packages/platform-apple/src/os/macos/helper.ts @@ -354,6 +354,8 @@ async function runMacOsHelper>( parsed && !parsed.ok ? (parsed.error?.message ?? `macOS helper exited with code ${result.exitCode}`) : stdout || result.stderr.trim() || `macOS helper exited with code ${result.exitCode}`; + // The helper's `details` carry its typed `reason` and, where a remedy exists, its `hint`; + // `normalizeError` reads `details.hint`, so the refusal reaches the agent untouched. // exec-guard-allow: the message is already built from the helper's JSON error // envelope (or its output); a stderr excerpt would only degrade it, and the // throw is reachable at exit 0 when the envelope reports ok=false. diff --git a/src/commands/capture/screenshot.ts b/src/commands/capture/screenshot.ts index 13ab745d18..1843d4b50c 100644 --- a/src/commands/capture/screenshot.ts +++ b/src/commands/capture/screenshot.ts @@ -74,7 +74,7 @@ export const screenshotCommandFacet = defineCommandFacet({ text: { summary: 'Capture a screenshot', cliDetail: - 'Web defaults to the viewport; use --fullscreen, --full, or -f for the entire page. iOS simulators default to 1x logical-point output; use --pixel-density to request a different screenshot density. macOS app sessions default to the app window; use --fullscreen for full desktop, --scale to downscale, --crop-on to crop the capture to the frame the selector resolves on the same screen (currently iOS simulators and Android emulators), --overlay-refs to annotate current refs, --normalize-status-bar for deterministic iOS simulator chrome, or --no-stabilize for low-latency Android capture loops. On macOS, any --surface other than app (desktop, menubar, frontmost-app) always captures the main display and refuses an explicit --fullscreen. On single-display iOS simulators and Android devices, the result reports displayRotation, the rotation the display rendered the capture in, when the device returns a reading in time; otherwise the field is absent.', + "Web defaults to the viewport; use --fullscreen, --full, or -f for the entire page. iOS simulators default to 1x logical-point output; use --pixel-density to request a different screenshot density. macOS app sessions default to the app window; use --fullscreen for full desktop, --scale to downscale, --crop-on to crop the capture to the frame the selector resolves on the same screen (currently iOS simulators and Android emulators), --overlay-refs to annotate current refs, --normalize-status-bar for deterministic iOS simulator chrome, or --no-stabilize for low-latency Android capture loops. With the native macOS backend, the front window is the app's frontmost visible, nonempty normal-level (layer-0) window on the current Space: floating palettes and app-modal alerts at window level 8 are skipped, and a sheet is captured as its own window. On macOS, any --surface other than app (desktop, menubar, frontmost-app) always captures the main display and refuses an explicit --fullscreen. On single-display iOS simulators and Android devices, the result reports displayRotation, the rotation the display rendered the capture in, when the device returns a reading in time; otherwise the field is absent.", }, metadata: screenshotCommandMetadata, run: (client, input) => client.capture.screenshot(input), diff --git a/src/commands/system/index.ts b/src/commands/system/index.ts index 7f17fd3da9..16aad73a84 100644 --- a/src/commands/system/index.ts +++ b/src/commands/system/index.ts @@ -278,7 +278,7 @@ export const foldableHelpTopic = { A foldable Apple device (iPhone Duo) carries two integrated panels, which Apple calls the outer display and the inner display. Only one is lit at a time, and which one is lit is the device pose. Screens are handled for you: - Each iOS simulator capture resolves the CoreDevice display table, captures the lit panel explicitly, and normalizes density with that panel's own point scale. Do not add a screen flag to the normal loop; there is none, because the lit panel is always the only capturable one: the dark panel yields an all-black PNG. + Each iOS simulator capture resolves the CoreDevice display table, selects the sole lit panel when power identifies it, and normalizes density with that panel's own point scale. There is no screen flag. When panel power names no single lit panel, the capture names CoreDevice's primary panel, which may be dark and yield an all-black PNG. Check the pose with UIHinge.status before trusting that capture. With no primary panel either, capture fails with COMMAND_FAILED and reason capture-panel-unidentified rather than pick a panel by list order. The two panels are different sizes (iPhone Duo: 466x678 points closed on the outer panel, 669x951 open on the inner). A pose change therefore invalidates every ref and coordinate. Re-snapshot after any pose change and never carry coordinates or refs across one. Check which panel is lit before trusting a geometry claim: agent-device screenshot reports its point size, and 466x678 versus 669x951 says which panel you captured. diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index e0014faa7f..58e7eff9ac 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -321,7 +321,7 @@ agent-device snapshot -i --platform apple --target desktop Set `AGENT_DEVICE_MACOS_APP_BACKEND=native` to run macOS app sessions without XCTest. There is no Automation Mode overlay, the app can stay behind other windows, and your pointer stays yours while a drawn pointer shows each action. -- `click`, `press`, and `fill` use accessibility actions on the element at the target point, which must be inside one of the app's on-screen windows, not its menu bar. `type` inserts at the focused control or falls back to key events. `scroll` moves the scroll bar under the front window's center. Chromium-based apps expose their full tree. `screenshot` captures only the app's front window, even when it is covered. +- `click`, `press`, and `fill` use accessibility actions on the element at the target point, which must be inside one of the app's on-screen windows, not its menu bar. `type` inserts at the focused control or falls back to key events. `scroll` moves the scroll bar under the front window's center. Chromium-based apps expose their full tree. `screenshot` captures only the app's front window, even when it is covered. The front window is the app's frontmost visible, nonempty normal-level (layer-0) window on the current Space: floating palettes and app-modal alerts at window level 8 are skipped, and a sheet is captured as its own window. If that window cannot be captured, `screenshot` fails; it never captures another window. - `record`, `prepare`, `back`, `longpress`, `--hold-ms`, `--double-tap`, `--button secondary`, gestures, and clicks on an element with no accessibility action fail with `UNSUPPORTED_OPERATION` (`reason: unsupported-device-backend`). Use the default XCTest backend for those. ```bash