diff --git a/packages/command-registry/src/flag-definitions-action.ts b/packages/command-registry/src/flag-definitions-action.ts index 0a760234ee..5634505506 100644 --- a/packages/command-registry/src/flag-definitions-action.ts +++ b/packages/command-registry/src/flag-definitions-action.ts @@ -147,7 +147,7 @@ export const ACTION_FLAG_DEFINITIONS: readonly FlagDefinition[] = [ min: 0, max: 100, usageLabel: '--jitter-px ', - usageDescription: 'Deterministic coordinate jitter radius for press', + usageDescription: 'Deterministic per-axis coordinate jitter in pixels', projectConfig: true, recorded: true, }, diff --git a/scripts/help-conformance-bench.mjs b/scripts/help-conformance-bench.mjs index 06bcf178f8..1f0bd7ae3a 100644 --- a/scripts/help-conformance-bench.mjs +++ b/scripts/help-conformance-bench.mjs @@ -22,7 +22,7 @@ const RUN_TIMEOUT_MS = Number(process.env.HELP_BENCH_TIMEOUT_MS ?? 90_000); // Runner x case pairs run concurrently, capped low: these are paid LLM calls // and the CLI help subprocess calls behind loadDocs share this same machine. const CONCURRENCY = Number(process.env.HELP_BENCH_CONCURRENCY ?? 4); -const DEFAULT_RUNNERS = ['codex:gpt-5.4-mini', 'claude:claude-haiku-4-5']; +const DEFAULT_RUNNERS = ['codex:gpt-6-luna', 'claude:claude-haiku-4-5']; const USAGE = `Usage: node scripts/help-conformance-bench.mjs [options] Feeds a help slice + task into one non-agentic LLM call per runner x case and diff --git a/src/commands/command-input.ts b/src/commands/command-input.ts index fbce60a059..c618e2a85b 100644 --- a/src/commands/command-input.ts +++ b/src/commands/command-input.ts @@ -387,7 +387,10 @@ export function repeatedFields() { count: integerField('Number of press/click repetitions.', { min: 1 }), intervalMs: integerField('Delay between repeated press/click actions.', { min: 0 }), holdMs: integerField('Hold duration for each action.', { min: 0 }), - jitterPx: integerField('Randomization radius in pixels.', { min: 0 }), + jitterPx: integerField( + 'Deterministic per-axis coordinate jitter in pixels (each press offsets x and y by up to this amount).', + { min: 0 }, + ), doubleTap: booleanField('Request a double-tap action.'), }; } diff --git a/src/commands/interaction/index.ts b/src/commands/interaction/index.ts index 0ebaf14cc6..21c635cb6e 100644 --- a/src/commands/interaction/index.ts +++ b/src/commands/interaction/index.ts @@ -229,7 +229,9 @@ const tapInteractionResponseDataSchema = interactionResponseDataSchema({ count: numberSchema('Number of press/click repetitions.'), intervalMs: numberSchema('Delay between repeated press/click actions.'), holdMs: numberSchema('Hold duration for each action.'), - jitterPx: numberSchema('Randomization radius in pixels.'), + jitterPx: numberSchema( + 'Deterministic per-axis coordinate jitter in pixels (each press offsets x and y by up to this amount).', + ), doubleTap: booleanSchema('Whether the command requested a double-tap action.'), }, }); diff --git a/src/commands/management/app.ts b/src/commands/management/app.ts index 17f4e9fbce..8a035fc7a7 100644 --- a/src/commands/management/app.ts +++ b/src/commands/management/app.ts @@ -44,7 +44,9 @@ const openCommandMetadata = defineFieldCommandMetadata( 'macOS presentation surface to open: the app itself, the frontmost app, the desktop, or the menu bar.', ), activity: stringField('Android activity name.'), - launchConsole: stringField('Launch console mode.'), + launchConsole: stringField( + 'File path on the daemon host for the stdout and stderr of the initial app launch. iOS simulator direct app launches only; not for URL opens.', + ), launchArgs: stringArrayField( 'Launch arguments forwarded verbatim to the platform launch command.', ), diff --git a/src/commands/schema/cli-help-first-screen.test.ts b/src/commands/schema/cli-help-first-screen.test.ts new file mode 100644 index 0000000000..f11eafce1a --- /dev/null +++ b/src/commands/schema/cli-help-first-screen.test.ts @@ -0,0 +1,208 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { usage, usageForCommand } from '../../cli/parser/args.ts'; +import { helpTopicIds } from './cli-help.ts'; + +// Agents often read only the first screen of a help text, and the help benchmark caps the root +// help at 30 lines (`--help:first30`). Each entry pins the rules that keep an agent safe on that +// screen: prerequisites, refusals, verification steps, and recovery. A new topic must declare its +// own entry. +const FIRST_SCREEN_LINES = 30; +const ROOT_HELP = '--help'; + +const FIRST_SCREEN_RULES: Record = { + [ROOT_HELP]: [ + 'agent-device open --foreground', + 'Do not probe first with devices, apps, appstate, snapshot, or screenshot', + 'Unknown app id: devices, then apps, then open . Never invent ids', + 'is, get, or find. A bare screenshot is not verification', + 'End with: agent-device close', + ], + commands: [ + 'Full command catalog. Use agent-device help for exact flags and behavior', + 'Commands:', + ], + 'manual-qa': [ + 'Execute the script. Do not explore unrelated screens', + 'Stop and report ambiguity', + '1. Open the requested app; use --relaunch only when the script needs fresh state', + '5. If --settle prints not settled, follow its hint', + '6. Verify named expectations with wait text/selector/absent, get, is, find, or the settled diff. A bare screenshot/snapshot is not verification', + ], + maestro: [ + 'Run Maestro compatibility flows with replay --maestro', + '- Flows: launchApp (with clearState, permissions, and Apple-only launch arguments; permissions apply after state clearing but before launch, and a launchApp without permissions touches nothing — there is no silent all: allow default', + '- Failure diagnostics: resolved targets and runFlow paths are rendered, while inputText payloads remain hidden; do not place secrets in diagnostic identifiers', + '- Trust: runScript, evalScript, and JavaScript condition true: fields execute flow scripts in-process via node:vm, which is not a security sandbox', + 'Unsupported syntax fails loudly', + ], + workflow: [ + 'Refs look like @e12; use the exact ref from the latest snapshot -i, never a placeholder (@ref, @eN, @Label_Name)', + 'Refs look like @e12; use the exact ref from the latest snapshot -i, never a placeholder (@ref, @eN, @Label_Name). Pin with ~s (press @e12~s4); every platform rejects a stale pinned ref', + 'Unknown app id: devices, then apps, then open . Never open artifact paths or invent package ids', + 'Truncated preview: snapshot -s @e12 (the current concrete ref), not get text. Missing target: scroll --until . TV/D-pad focus: help tv', + 'Mutating selector ambiguity: press/click/fill/longpress collapse duplicate accessibility wrappers only when every match is one ancestor-descendant chain resolving to the same actionable node. Matches in distinct subtrees fail with AMBIGUOUS_MATCH', + ], + scripting: [ + 'Reaching an off-screen target is viewport-independent in a script the same way it is at the CLI: write scroll down --until ', + 'press/click/longpress resolve their target once. Before interacting with a loading screen, add an explicit landmark wait', + 'agent-device fill \'id="password"\' "$AD_VAR_PASSWORD" --record-as PASSWORD', + 'The live app receives the value; recording state and the published script contain only ${PASSWORD}', + '--text-stdin reads at most 64 KiB, removes exactly one trailing newline, and refuses a terminal, empty input, invalid UTF-8', + ], + gestures: [ + 'longpress accepts coordinates, @refs, or selectors; prefer @ref/selector, coordinates only as a fallback', + 'iOS simulator transform/pinch/rotate use private XCTest synthesis for a continuous two-finger pan/scale/rotation path; verify app metrics instead of assuming requested values map exactly to recognizer output', + 'Rare iOS accessibility gap: a row shown disabled/hittable:false where press reports success but no UI change, or a collapsed composite control with no child refs -- run snapshot -i --json, compute the target center from rects, press x y, then diff snapshot -i. Coordinates are fallback-only; document why you used them.', + ], + tv: [ + 'Use this when the target is Android TV, Apple TV/tvOS, or an Amazon Vega OS TV app running in the Vega Virtual Device (VVD). TV surfaces are focus-first: move focus with remote/D-pad buttons, then activate the focused control', + 'Vega OS is driven through the SDK-matched Vega CLI and VDA, not ADB', + 'Initial support is VVD-only. Physical Fire TV devices remain unsupported', + 'The VVD is never booted implicitly', + 'Snapshot, screenshot, selectors, install, touch/text/gesture, logs, and performance commands remain unsupported until their Vega backends are implemented', + ], + debugging: [ + 'Do not cat a full stale log into agent context', + 'logs clear --restart is the compact command to clear old logs and start a fresh capture; do not split it into logs stop, logs clear, logs start', + '--launch-console is only for direct iOS simulator app launches, not URL opens', + 'network log is a supported alias, but network dump --include headers is the clearest plan form. Do not write network log headers', + ], + 'react-devtools': [ + "Every command except profile report and profile commit, which read the saved profile, attaches to the app's JavaScript runtime through its Metro dev server: a development build on React Native 0.74 or newer", + 'Refs are DevTools element ids: they change when a component remounts or the app reloads, so find again after either', + 'With a React DevTools window connected to the app, tree, find, errors, and inspect by name fail with tree-unavailable', + '1. Run profile start immediately before the interaction', + ], + cdp: [ + 'state. Do not use this as the default React Native profiler', + 'Start Metro and open the app first. For Android devices/emulators, make sure Metro is reachable from the app, typically with adb reverse tcp:8081 tcp:8081', + 'agent-device cdp target select ', + 'agent-device cdp memory snapshot leak-triplet', + ], + 'react-native': [ + 'Do not run doctor as routine QA/dogfood prep', + 'For "start from screen X" flows, prefer open --relaunch before the first snapshot', + 'Do not use agent-device reload', + 'Verify Metro/Re.Pack from the same host context that owns the dev server. If a sandboxed shell cannot curl localhost:8081/status but an unrestricted host shell can, the dev server is running and the sandbox probe is not authoritative', + 'Expo Go/dev clients are host shells. Use provided project URLs, verify with snapshot -i after opening, and ask instead of inventing app ids or URLs', + ], + 'physical-device': [ + 'The device must be paired/trusted, connected, unlocked when needed, and have Developer Mode enabled', + 'The AgentDeviceRunner XCTest host must be signed before commands can run on a physical device', + 'A runner startup failure names its class in error details.reason rather than only in prose:', + 'Enable USB debugging', + ], + 'ios-system-ui': [ + '1. Reach the app state you want to prepare (for example, arrange the widget/Live Activity data the app should show) with normal app automation, then agent-device open com.apple.springboard --platform ios', + 'This works today by opening SpringBoard as the session app; there is no separate widget/system command. System labels vary by iOS version and locale, so discover them from the current snapshot instead of relying on the literal strings', + '3. agent-device longpress on an empty area of the home screen to enter edit mode. This is the one deliberate coordinate step', + 'Do not hard-code Edit/Done/Add Widget', + '9. Reopen the app bundle under test', + ], + foldable: [ + "Each iOS simulator capture resolves the CoreDevice display table, captures the lit panel explicitly, and normalizes density with that panel's own point scale. Do not add a screen flag to the normal loop", + 'The two panels are different sizes (iPhone Duo: 466x678 points closed on the outer panel, 669x951 open on the inner). A pose change therefore invalidates every ref and coordinate. Re-snapshot after any pose change and never carry coordinates or refs across', + 'Expect a fold to take 10-16 seconds: each hinge read is a five-second devicectl stream, and half-open waits for the hinge to stop moving. Re-snapshot after every fold', + 'fold sends a private HID hinge event inside the selected simulator', + 'If a task asserts behavior for more than one pose, fold to each pose and re-snapshot, and report which poses the run covered', + ], + remote: [ + 'connect -> install/open -> commands -> close -> disconnect', + 'BrowserStack, AWS Device Farm, and TestMu AI create the hosted session on open. open needs the installed package or bundle identifier, not the app artifact name, ARN, or lt:// id', + 'AWS Device Farm cannot install after allocation. If connect reports no attached app, run its printed reconnect command', + 'Do not run devices as a pre-open catalog probe for direct providers', + 'MCP exposes operational tools such as open, snapshot, click, close, and artifacts. It does not expose connect/disconnect', + ], + macos: [ + 'Use open and snapshot -i for menu bar inspection. Do not output inspect as a command', + 'agent-device click @e66 --button secondary', + 'Do not let iOS simulator-set scoping hide macOS desktop targets', + 'macOS snapshot rects are window-space; use current refs or overlay refs instead of guessing coordinates', + 'On the default XCTest backend a read (snapshot, get, find) of a session app that sits behind other windows is answered from where it sits, without raising the app', + ], + web: [ + 'Use --platform web only for the minimal browser command loop', + 'agent-device web doctor', + 'Web automation requires Node 24+', + 'For web command plans, output only agent-device command lines', + ], + dogfood: [ + 'Find user-visible issues from runtime behavior. Do not read app source or invent findings from code', + '1. Identify target app/platform; ask only if missing', + 'Keep stateful commands serial within the same session', + 'React Native warning/error overlays can be real findings or test blockers. Capture them, use react-native dismiss-overlay', + ], + validate: [ + 'Prove the changed behavior through public agent-device surfaces. Do not validate against stale dist output', + 'For a TypeScript runtime or CLI output change, start with pnpm build', + 'Before local Android verification, run pnpm build:android before pnpm clean:daemon', + 'For an Apple runner change, run pnpm build:xcuitest and avoid inherited retained runners from older source. Do not build the Apple runner for TypeScript-only changes', + '6. Close sessions and release leases before finishing', + ], +}; + +// A rule counts only when a whole rendered line carries it: the trimmed line equals the +// rule, or starts with it and the rule ends at a word boundary. A substring match would +// let a short generic rule like 'Commands:' pass from inside unrelated prose. +function ruleMatchesLine(line: string, rule: string): boolean { + const trimmed = line.trim(); + if (trimmed === rule) return true; + if (!trimmed.startsWith(rule)) return false; + return !/[\w-]/.test(trimmed[rule.length] ?? ''); +} + +function assertRulesInFirstScreen(lines: readonly string[], rules: readonly string[]): void { + for (const rule of rules) { + const line = lines.findIndex((text) => ruleMatchesLine(text, rule)) + 1; + assert.ok( + line > 0 && line <= FIRST_SCREEN_LINES, + line > 0 + ? `"${rule}" is on line ${line}, below the first ${FIRST_SCREEN_LINES} lines` + : `"${rule}" has no matching line`, + ); + } +} + +async function renderHelp(id: string): Promise { + if (id === ROOT_HELP) return await usage(); + const text = await usageForCommand(id); + if (text === null) throw new Error(`Expected help text for topic ${id}`); + return text; +} + +test('every help topic and the root help declare first-screen rules', () => { + assert.deepEqual(Object.keys(FIRST_SCREEN_RULES).sort(), [ROOT_HELP, ...helpTopicIds()].sort()); +}); + +test.each(Object.entries(FIRST_SCREEN_RULES))( + 'help %s keeps its key rules in the first 30 lines', + async (id, rules) => { + assertRulesInFirstScreen((await renderHelp(id)).split('\n'), rules); + }, +); + +test('a rule matches only whole lines, not prose, a moved rule, or a deleted rule', () => { + // Short generic rules must not pass from inside unrelated prose. + const prose = [ + 'Some unrelated sentence that merely quotes Commands: in passing, far from any list.', + ]; + assert.equal(ruleMatchesLine(prose[0]!, 'Commands:'), false); + assert.throws(() => assertRulesInFirstScreen(prose, ['Commands:']), /no matching line/); + + // A rule pushed below the window fails. + const filler = Array.from({ length: 40 }, (_, i) => `filler line ${i + 1}`); + filler[34] = 'Commands:'; + assert.throws( + () => assertRulesInFirstScreen(filler, ['Commands:']), + /"Commands:" is on line 35, below the first 30 lines/, + ); + + // A deleted rule fails. + filler[34] = 'filler line 35'; + assert.throws(() => assertRulesInFirstScreen(filler, ['Commands:']), /no matching line/); + + // Prefixes count only at a word boundary, not mid-word. + assert.equal(ruleMatchesLine(' Commands: are listed below', 'Commands:'), true); + assert.equal(ruleMatchesLine('SubCommands: here', 'Commands:'), false); +}); diff --git a/src/commands/schema/cli-help.ts b/src/commands/schema/cli-help.ts index 7a42f33207..2619c82eb5 100644 --- a/src/commands/schema/cli-help.ts +++ b/src/commands/schema/cli-help.ts @@ -967,15 +967,21 @@ function buildHelpTopicUsageText(topicName: string): string | null { const topic = HELP_TOPICS[topicName as keyof typeof HELP_TOPICS]; if (!topic) return null; const body = topicName === 'commands' ? renderFullCommandReferenceText() : topic.body; + const related = RELATED_HELP_LINES.filter((entry) => entry.topic !== topicName) + .map((entry) => ` ${entry.line}\n`) + .join(''); return `${withVersionHeader(topicName, body)} Related: - agent-device help command-specific flags - agent-device help manual-qa routine QA loop - agent-device help workflow full automation reference -`; +${related}`; } +const RELATED_HELP_LINES = [ + { topic: null, line: 'agent-device help command-specific flags' }, + { topic: 'manual-qa', line: 'agent-device help manual-qa routine QA loop' }, + { topic: 'workflow', line: 'agent-device help workflow full automation reference' }, +] as const; + // Every topic body's first line is authored as `agent-device help `. Swapping in the // installed version here (instead of hand-editing every topic string) gives the skill router a // single, reliable header to read the CLI version from without a separate --version call: a diff --git a/src/mcp/server-guide.ts b/src/mcp/server-guide.ts index b7dba19c8c..a6d55e9c66 100644 --- a/src/mcp/server-guide.ts +++ b/src/mcp/server-guide.ts @@ -46,7 +46,7 @@ export function terminalOnlyCommandNames(): string[] { */ export const HELP_TOOL: { name: string; description: string; inputSchema: JsonSchema } = { name: HELP_TOOL_NAME, - description: `Usage guides. No topic: the full workflow card. topic = one of ${helpTopicIds().join(', ')} for that guide, or a tool name for its complete flag reference. For specialized work or an unclear command shape only; not a startup step, not needed after an error that carries a hint.`, + description: `Usage guides. No topic: the root decision card. topic = one of ${helpTopicIds().join(', ')} for that guide, or a tool name for its complete flag reference. For specialized work or an unclear command shape only; not a startup step, not needed after an error that carries a hint.`, inputSchema: { type: 'object', properties: {