From cc650f0ebae78dd2f0c63ea2e6f674e9e3ef6233 Mon Sep 17 00:00:00 2001 From: Mikolaj Adamowicz Date: Thu, 15 Oct 2026 14:25:50 +0000 Subject: [PATCH 01/26] feat(hardware-key): press volume, power, and media keys on Android Add `agent-device hardware-key ` and `client.command.hardwareKey()`: one press of a volume (`volume-up`, `volume-down`, `mute`), power (`power`, `sleep`, `wake`), or media (`media-play-pause`, `media-next`, `media-previous`, `media-rewind`, `media-fast-forward`) key. It covers QA checks such as volume and mute handling, standby and wake during playback, and media-key handling in a video-streaming app. Android emulators and devices send the key as `adb shell input keyevent` with the numeric KeyEvent code, locally and on Limrun Android sessions, which ride the same Android interactor as `home` and `app-switcher`. The key vocabulary and its Android keycodes sit beside the TV remote table in `contracts/tv-remote.ts`, and the press joins the system-button family as its one control with an argument, so every owner that does not name the cell refuses it with that family's denial. Both homes are modules the CLI, daemon, and runtime closures already evaluate, so no eager closure grows. The Apple runner exposes only the home and Action Button presses, so iOS, tvOS, and macOS refuse the key, as do web, Linux, Vega OS, HarmonyOS, Limrun iOS, and the WebDriver (Appium) backend. Like `action-button`, the command declares no post-action observation: the key goes to the system and the media session, and power or sleep leaves the display off. Session events summarize the press with its key, and `help tv` points Android TV playback checks at it. --- .../src/__tests__/command-result.test.ts | 4 + .../command-registry/src/command-result.ts | 2 + packages/command-registry/src/registry.ts | 12 ++ packages/contracts/src/client-system.ts | 6 +- packages/contracts/src/facades/client.ts | 1 + ...ctor-operation-catalog-conformance.test.ts | 1 + .../src/interactor-operation-catalog.ts | 8 +- packages/contracts/src/interactor-types.ts | 9 +- packages/contracts/src/navigation.ts | 9 +- .../src/platform-runtime-operations.ts | 1 + .../contracts/src/runtime-operation-names.ts | 1 + .../src/system-button-runtime.test.ts | 44 +++++ .../contracts/src/system-button-runtime.ts | 72 +++++++-- packages/contracts/src/tv-remote.test.ts | 37 +++++ packages/contracts/src/tv-remote.ts | 44 +++++ .../src/core/runtime-operation-effects.ts | 1 + .../__tests__/system-button-runtime.test.ts | 151 ++++++++++++++++++ .../execution/generic-runtime-execution.ts | 14 +- .../src/execution/system-button-runtime.ts | 34 ++++ .../src/__tests__/input-actions.test.ts | 17 ++ .../platform-android/src/input-actions.ts | 11 +- packages/platform-android/src/mechanics.ts | 1 + packages/platform-android/src/runtime.test.ts | 18 +++ packages/platform-android/src/runtime.ts | 11 +- .../platform-apple/src/navigation/runtime.ts | 2 +- packages/platform-apple/src/runner-demand.ts | 2 + packages/platform-apple/src/runtime.test.ts | 23 +++ packages/platform-linux/src/runtime.test.ts | 2 + packages/platform-web/src/runtime.test.ts | 2 + .../src/interaction-operations.test.ts | 5 +- .../src/interaction-operations.ts | 9 +- .../src/platform-runtime.ts | 6 + .../__tests__/session-event-action.test.ts | 17 ++ .../src/session-event-action-presentation.ts | 11 +- ...gent-device-client-system-commands.test.ts | 6 + src/agent-device-client.ts | 2 + src/client/client-types.ts | 2 + src/commands/replay/replay-script.fixtures.ts | 1 + src/commands/schema/cli-help.ts | 1 + src/commands/system/index.test.ts | 26 +++ src/commands/system/index.ts | 68 ++++++++ src/commands/system/output.ts | 1 + src/core/interactors/android.ts | 2 + website/docs/docs/client-api.md | 2 + website/docs/docs/commands.md | 4 + 45 files changed, 672 insertions(+), 31 deletions(-) diff --git a/packages/command-registry/src/__tests__/command-result.test.ts b/packages/command-registry/src/__tests__/command-result.test.ts index 2884e64567..d50eb5010f 100644 --- a/packages/command-registry/src/__tests__/command-result.test.ts +++ b/packages/command-registry/src/__tests__/command-result.test.ts @@ -13,6 +13,7 @@ import type { import type { AppSwitcherCommandResult, BackCommandResult, + HardwareKeyCommandResult, HomeCommandResult, OrientationCommandResult, TvRemoteCommandResult, @@ -62,6 +63,7 @@ test('seeded CommandResult entries resolve to their existing contract result typ const keyboard: Equal, KeyboardCommandResult> = true; const tvRemote: Equal, TvRemoteCommandResult> = true; const telephony: Equal, TelephonyCommandResult> = true; + const hardwareKey: Equal, HardwareKeyCommandResult> = true; const wait: Equal, WaitCommandResult> = true; const prepare: Equal, PrepareCommandResult> = true; const push: Equal, PushCommandResult> = true; @@ -93,6 +95,7 @@ test('seeded CommandResult entries resolve to their existing contract result typ keyboard, tvRemote, telephony, + hardwareKey, wait, prepare, push, @@ -136,6 +139,7 @@ test('CommandResultMap is seeded only from already-existing contract result type | 'keyboard' | 'tv-remote' | 'telephony' + | 'hardware-key' | 'wait' | 'scroll' | 'prepare' diff --git a/packages/command-registry/src/command-result.ts b/packages/command-registry/src/command-result.ts index 56e2805d3c..a994be800e 100644 --- a/packages/command-registry/src/command-result.ts +++ b/packages/command-registry/src/command-result.ts @@ -15,6 +15,7 @@ import type { AppSwitcherCommandResult, BackCommandResult, FoldCommandResult, + HardwareKeyCommandResult, HomeCommandResult, OrientationCommandResult, TvRemoteCommandResult, @@ -72,6 +73,7 @@ export interface CommandResultMap { fill: FillCommandResponseData; find: FindCommandResponseData; fold: FoldCommandResult; + 'hardware-key': HardwareKeyCommandResult; home: HomeCommandResult; hover: HoverCommandResponseData; keyboard: KeyboardCommandResult; diff --git a/packages/command-registry/src/registry.ts b/packages/command-registry/src/registry.ts index b3c319ce16..213064c6f0 100644 --- a/packages/command-registry/src/registry.ts +++ b/packages/command-registry/src/registry.ts @@ -58,6 +58,7 @@ import { focusRuntimeUse, gestureRuntimePlanUses, gestureViewportRuntimeUse, + hardwareKeyRuntimeUse, homeRuntimeUse, hoverRuntimeUses, keyboardRuntimePlanUses, @@ -1494,6 +1495,17 @@ export const RAW_COMMAND_DESCRIPTORS = [ ...GENERIC_MUTATING_COMMAND_TRAITS, platformExecution: { kind: 'device-runtime', uses: [tvRemoteRuntimeUse] }, }, + { + name: 'hardware-key', + ...(ownerFilesEnabled ? { ownerFiles: ['src/commands/system/index.ts'] as const } : {}), + catalog: { group: 'public', key: 'hardwareKey' }, + frameworkTier: 'extended', + // Admission is the owner's `hardwareKey` fact, a member of the system-button family. No + // post-action observation, as for `action-button`: the key goes to the system and the media + // session rather than the session app, and power/sleep leave the display off. + ...GENERIC_MUTATING_COMMAND_TRAITS, + platformExecution: { kind: 'device-runtime', uses: [hardwareKeyRuntimeUse] }, + }, { name: 'orientation', ...(ownerFilesEnabled ? { ownerFiles: ['src/commands/system/index.ts'] as const } : {}), diff --git a/packages/contracts/src/client-system.ts b/packages/contracts/src/client-system.ts index 62932a49de..1777eba1f8 100644 --- a/packages/contracts/src/client-system.ts +++ b/packages/contracts/src/client-system.ts @@ -7,7 +7,7 @@ import type { DeviceCommandBaseOptions } from './client-connection.ts'; import type { SettleCommandOptions } from './client-gesture.ts'; import type { DeviceRotation, SetFoldPoseInput } from './device-rotation.ts'; import type { TelephonyEventInput } from './telephony-input.ts'; -import type { TvRemoteButton } from './tv-remote.ts'; +import type { HardwareKey, TvRemoteButton } from './tv-remote.ts'; export type WaitCommandTarget = | { @@ -104,6 +104,10 @@ export type TvRemoteCommandOptions = DeviceCommandBaseOptions & { export type TelephonyCommandOptions = DeviceCommandBaseOptions & TelephonyEventInput; +export type HardwareKeyCommandOptions = DeviceCommandBaseOptions & { + key: HardwareKey; +}; + export type KeyboardCommandOptions = DeviceCommandBaseOptions & { action?: 'status' | 'dismiss' | 'enter' | 'return'; }; diff --git a/packages/contracts/src/facades/client.ts b/packages/contracts/src/facades/client.ts index 5af90eef2f..4b4bfd096f 100644 --- a/packages/contracts/src/facades/client.ts +++ b/packages/contracts/src/facades/client.ts @@ -120,6 +120,7 @@ export type { ClipboardCommandOptions, FoldCommandOptions, DoctorCommandOptions, + HardwareKeyCommandOptions, HomeCommandOptions, KeyboardCommandOptions, OrientationCommandOptions, diff --git a/packages/contracts/src/interactor-operation-catalog-conformance.test.ts b/packages/contracts/src/interactor-operation-catalog-conformance.test.ts index bca8508288..3dc1c54ec5 100644 --- a/packages/contracts/src/interactor-operation-catalog-conformance.test.ts +++ b/packages/contracts/src/interactor-operation-catalog-conformance.test.ts @@ -28,6 +28,7 @@ const EXPECTATIONS: { home: { method: 'home', input: {} }, setOrientation: { method: 'setOrientation', input: { rotation: 'portrait' } }, tvRemote: { method: 'tvRemote', input: { button: 'select' } }, + hardwareKey: { method: 'hardwareKey', input: { key: 'volume-up' } }, keyboardStatus: { method: 'keyboardStatus', input: {} }, keyboardDismiss: { method: 'keyboardDismiss', input: {} }, keyboardEnter: { method: 'keyboardEnter', input: {} }, diff --git a/packages/contracts/src/interactor-operation-catalog.ts b/packages/contracts/src/interactor-operation-catalog.ts index 80be944e06..d020ffa79d 100644 --- a/packages/contracts/src/interactor-operation-catalog.ts +++ b/packages/contracts/src/interactor-operation-catalog.ts @@ -20,7 +20,12 @@ import { import type { Interactor, RunnerContext } from './interactor-types.ts'; import type { PlatformRuntimeOperations } from './platform-runtime-operations.ts'; import type { RuntimeOperationFact } from './platform-runtime.ts'; -import { bindSystemButton, SYSTEM_BUTTON_LABELS } from './system-button-runtime.ts'; +import { + bindHardwareKey, + bindSystemButton, + HARDWARE_KEY_LABEL, + SYSTEM_BUTTON_LABELS, +} from './system-button-runtime.ts'; /** * How a facet turns one resolved interactor into its own typed operations. Every catalog member @@ -63,6 +68,7 @@ export const INTERACTOR_OPERATIONS = [ }, { operation: 'setOrientation', label: 'orientation', bind: bindOrientation }, { operation: 'tvRemote', label: TV_REMOTE_LABEL, bind: bindTvRemote }, + { operation: 'hardwareKey', label: HARDWARE_KEY_LABEL, bind: bindHardwareKey }, { operation: 'keyboardStatus', label: KEYBOARD_ACTION_LABELS.keyboardStatus, diff --git a/packages/contracts/src/interactor-types.ts b/packages/contracts/src/interactor-types.ts index a17438a9aa..6f6c8f0a8c 100644 --- a/packages/contracts/src/interactor-types.ts +++ b/packages/contracts/src/interactor-types.ts @@ -6,7 +6,7 @@ import type { DeviceRotation } from './device-rotation.ts'; import type { FillUnconfirmedVerification } from './fill-evidence.ts'; import type { ScrollDirection } from './scroll-gesture.ts'; import type { ScrollExecutionOptions } from './scroll-command.ts'; -import type { TvRemoteButton } from './tv-remote.ts'; +import type { HardwareKey, TvRemoteButton } from './tv-remote.ts'; import type { GesturePlan } from './gesture-plan-types.ts'; import type { ReadSettingResult, SettingOptions } from './settings.ts'; import type { ReadableSetting } from './settings-catalog.ts'; @@ -419,6 +419,13 @@ export type Interactor = { * no-op. */ tvRemote?(button: TvRemoteButton, durationMs?: number): Promise; + /** + * Optional (parity with `tvRemote`): presses one volume, power, or media key, a control only + * owners with a key-event channel carry. An owner without it leaves it undefined; its fact + * refuses the press before binding, and the hardware-key binder fails closed rather than + * resolving an absent member as a silent no-op. + */ + hardwareKey?(key: HardwareKey): Promise; /** * Optional (parity with `keyboardDismiss`): presses the iPhone/iPad Action Button, hardware only * the Apple owner carries. An owner without the button leaves it undefined; its fact refuses the diff --git a/packages/contracts/src/navigation.ts b/packages/contracts/src/navigation.ts index fca6600b99..4382f40bdc 100644 --- a/packages/contracts/src/navigation.ts +++ b/packages/contracts/src/navigation.ts @@ -2,7 +2,7 @@ import type { BackMode } from './back-mode.ts'; import type { DeviceRotation, FoldPose } from './device-rotation.ts'; import type { FoldScreenReport } from './fold-runtime.ts'; import type { SettleObservation } from './interaction.ts'; -import type { TvRemoteButton } from './tv-remote.ts'; +import type { HardwareKey, TvRemoteButton } from './tv-remote.ts'; /** * Closed results of the navigation/global action commands. Each mirrors its @@ -89,3 +89,10 @@ export type TvRemoteCommandResult = { durationMs?: number; message: string; }; + +/** `hardware-key` — `{ action: 'hardware-key', key, message }`. */ +export type HardwareKeyCommandResult = { + action: 'hardware-key'; + key: HardwareKey; + message: string; +}; diff --git a/packages/contracts/src/platform-runtime-operations.ts b/packages/contracts/src/platform-runtime-operations.ts index 06b378487c..03cdcbf405 100644 --- a/packages/contracts/src/platform-runtime-operations.ts +++ b/packages/contracts/src/platform-runtime-operations.ts @@ -117,6 +117,7 @@ export const orientationRuntimeUse = defineUse({ required: ['setOrientation'] }) export const foldRuntimeUse = defineUse({ required: ['setFoldPose'] }); export const telephonyRuntimeUse = defineUse({ required: ['sendTelephonyEvent'] }); export const tvRemoteRuntimeUse = defineUse({ required: ['tvRemote'] }); +export const hardwareKeyRuntimeUse = defineUse({ required: ['hardwareKey'] }); export const keyboardStatusUse = defineUse({ required: ['keyboardStatus'] }); export const keyboardDismissUse = defineUse({ required: ['keyboardDismiss'] }); export const keyboardEnterUse = defineUse({ required: ['keyboardEnter'] }); diff --git a/packages/contracts/src/runtime-operation-names.ts b/packages/contracts/src/runtime-operation-names.ts index 32cdf3b31a..0c2f3c30e3 100644 --- a/packages/contracts/src/runtime-operation-names.ts +++ b/packages/contracts/src/runtime-operation-names.ts @@ -41,6 +41,7 @@ export const RUNTIME_OPERATION_NAMES = [ 'findText', 'focusPoint', 'gestureViewport', + 'hardwareKey', 'home', 'hoverPoint', 'hoverRef', diff --git a/packages/contracts/src/system-button-runtime.test.ts b/packages/contracts/src/system-button-runtime.test.ts index 13468254d1..3536d10f82 100644 --- a/packages/contracts/src/system-button-runtime.test.ts +++ b/packages/contracts/src/system-button-runtime.test.ts @@ -3,6 +3,7 @@ import type { Interactor } from './interactor-types.ts'; import { localInteractorSource } from './interactor-operation-binding.ts'; import { SYSTEM_BUTTONS, + bindHardwareKey, bindSystemButton, systemButtonRuntimeOperationFacts, } from './system-button-runtime.ts'; @@ -22,11 +23,19 @@ test('an omitted button reports the family denial, a named one its own cell', () home: available, appSwitcher: unsupported, actionButton: unsupported, + hardwareKey: unsupported, + }); + expect(systemButtonRuntimeOperationFacts({ unsupported, hardwareKey: available })).toEqual({ + home: unsupported, + appSwitcher: unsupported, + actionButton: unsupported, + hardwareKey: available, }); expect(systemButtonRuntimeOperationFacts({ unsupported })).toEqual({ home: unsupported, appSwitcher: unsupported, actionButton: unsupported, + hardwareKey: unsupported, }); }); @@ -71,3 +80,38 @@ test('an admitted button whose interactor lacks the member fails closed instead details: { reason: 'interactor-method-missing' }, }); }); + +test('a local hardware-key binding presses the interactor once with the parsed key', async () => { + const hardwareKey = vi.fn(async () => undefined); + const resolveInteractor = vi.fn(async () => ({ hardwareKey }) as unknown as Interactor); + const signal = new AbortController().signal; + + const operations = bindHardwareKey(signal, localInteractorSource({ device, resolveInteractor })); + await operations.hardwareKey({ + key: 'media-play-pause', + options: { appBundleId: 'com.example.app' }, + execution: { logPath: '/tmp/daemon.log', requestId: 'hardware-key-1' }, + }); + + expect(resolveInteractor).toHaveBeenCalledWith(device, { + logPath: '/tmp/daemon.log', + requestId: 'hardware-key-1', + appBundleId: 'com.example.app', + signal, + }); + expect(hardwareKey).toHaveBeenCalledExactlyOnceWith('media-play-pause'); +}); + +test('an admitted hardware key whose interactor lacks the member fails closed', async () => { + const resolveInteractor = vi.fn(async () => ({}) as unknown as Interactor); + const operations = bindHardwareKey( + new AbortController().signal, + localInteractorSource({ device, resolveInteractor }), + ); + + await expect(operations.hardwareKey({ key: 'volume-up' })).rejects.toMatchObject({ + code: 'COMMAND_FAILED', + message: 'hardware-key was admitted but its bound interactor has no implementation.', + details: { reason: 'interactor-method-missing' }, + }); +}); diff --git a/packages/contracts/src/system-button-runtime.ts b/packages/contracts/src/system-button-runtime.ts index 9b17bd4dfd..41b921bd6b 100644 --- a/packages/contracts/src/system-button-runtime.ts +++ b/packages/contracts/src/system-button-runtime.ts @@ -2,17 +2,28 @@ import { requireInteractorMethod } from './interactor-operation-binding.ts'; import type { Interactor, RunnerContext } from './interactor-types.ts'; import type { RuntimeOperationFact, RuntimeOperationUnavailability } from './platform-runtime.ts'; import type { SnapshotRuntimeExecution } from './snapshot-runtime.ts'; +import type { HardwareKey } from './tv-remote.ts'; /** * The system buttons: one press each, no arguments, nothing returned. `home` and `appSwitcher` * reach a springboard or recents surface; `actionButton` presses iPhone/iPad hardware. What * varies between them is which owners carry the control, and that is the fact table's job, not * a per-button module's: a button joins this list and its owners state a cell. + * + * `hardwareKey` is the family's one control that takes an argument: a single cell admits every + * volume, power, and media key, because an owner either has a key-event channel or has none. */ export const SYSTEM_BUTTONS = ['home', 'appSwitcher', 'actionButton'] as const; export type SystemButton = (typeof SYSTEM_BUTTONS)[number]; +type SystemControl = SystemButton | 'hardwareKey'; + +const SYSTEM_CONTROLS = [ + ...SYSTEM_BUTTONS, + 'hardwareKey', +] as const satisfies readonly SystemControl[]; + /** How a provider fail-closed refusal names each button to the caller. */ export const SYSTEM_BUTTON_LABELS = { home: 'home', @@ -27,13 +38,18 @@ export type SystemButtonInput = Readonly<{ execution?: SnapshotRuntimeExecution; }>; +/** One hardware key press: the parsed key plus the same runner metadata a button carries. */ +type HardwareKeyInput = SystemButtonInput & Readonly<{ key: HardwareKey }>; + /** A press returns nothing; every leaf discarded whatever the interactor answered. */ -export type SystemButtonRuntimeOperations = Readonly<{ - [Button in SystemButton]: (input: SystemButtonInput) => Promise; -}>; +export type SystemButtonRuntimeOperations = Readonly< + { + [Button in SystemButton]: (input: SystemButtonInput) => Promise; + } & { hardwareKey: (input: HardwareKeyInput) => Promise } +>; export type SystemButtonRuntimeOperationFacts = Readonly<{ - [Button in SystemButton]: RuntimeOperationFact; + [Control in SystemControl]: RuntimeOperationFact; }>; /** @@ -49,18 +65,31 @@ export type SystemButtonRuntimeOperationFacts = Readonly<{ */ export type SystemButtonRuntimeOperationFactsInput = Readonly< { unsupported: RuntimeOperationUnavailability } & { - [Button in SystemButton]?: RuntimeOperationFact; + [Control in SystemControl]?: RuntimeOperationFact; } >; export function systemButtonRuntimeOperationFacts( input: SystemButtonRuntimeOperationFactsInput, ): SystemButtonRuntimeOperationFacts { - const facts = {} as Record; - for (const button of SYSTEM_BUTTONS) facts[button] = input[button] ?? input.unsupported; + const facts = {} as Record; + for (const control of SYSTEM_CONTROLS) facts[control] = input[control] ?? input.unsupported; return Object.freeze(facts); } +async function resolvePressInteractor( + input: SystemButtonInput, + signal: AbortSignal, + resolveInteractor: (runner: RunnerContext) => Promise, +): Promise { + signal.throwIfAborted(); + return await resolveInteractor({ + ...input.execution, + appBundleId: input.options?.appBundleId, + signal, + }); +} + /** * Captures one selected owner's interactor authority for the lifetime of a request binding and * presses one button through it. `Interactor` members for hardware only some owners carry are @@ -73,12 +102,7 @@ export function bindSystemButton