From 0bc7e83407136fc47a0aa9e3264a41263ad594a5 Mon Sep 17 00:00:00 2001 From: Kit Langton Date: Sun, 19 Jul 2026 16:45:04 -0400 Subject: [PATCH] feat: add semantic UI snapshots --- .changeset/semantic-ui-snapshots.md | 5 + packages/drive/README.md | 33 ++- packages/drive/docs/open-code-driver-api.md | 16 ++ packages/drive/src/cli/commands.ts | 35 +++- packages/drive/src/cli/send.ts | 2 +- packages/drive/src/cli/types.ts | 2 +- packages/drive/src/driver/index.ts | 2 + packages/drive/src/driver/llm-responder.ts | 8 +- packages/drive/src/driver/ui.ts | 120 +++++++++-- packages/drive/src/script/errors.ts | 2 + packages/drive/src/simulation/connector.ts | 18 +- packages/drive/src/simulation/protocol.ts | 59 +++++- packages/drive/src/simulation/rpc.ts | 1 + packages/drive/test/cli/integration.test.ts | 75 +++++++ packages/drive/test/driver/ui.test.ts | 193 ++++++++++++++++++ packages/drive/test/fixtures/fake-opencode.ts | 15 ++ packages/drive/test/simulation/ui.test.ts | 39 ++-- skills/opencode-drive/SKILL.md | 2 + 18 files changed, 582 insertions(+), 45 deletions(-) create mode 100644 .changeset/semantic-ui-snapshots.md diff --git a/.changeset/semantic-ui-snapshots.md b/.changeset/semantic-ui-snapshots.md new file mode 100644 index 0000000..06ba605 --- /dev/null +++ b/.changeset/semantic-ui-snapshots.md @@ -0,0 +1,5 @@ +--- +"opencode-drive": minor +--- + +Expose semantic UI snapshots, exact semantic node polling, and safe semantic-node clicks for compatible OpenCode endpoints. diff --git a/packages/drive/README.md b/packages/drive/README.md index aff6c7e..01f6047 100644 --- a/packages/drive/README.md +++ b/packages/drive/README.md @@ -453,15 +453,38 @@ Finish a tool-calling response with `Llm.finish("tool-calls")`. Streamed calls drive OpenCode's normal tool-input start, delta, and end lifecycle; `Llm.raw()` remains available for provider-wire scenarios not covered by these helpers. +Current OpenCode simulation endpoints expose a semantic UI tree alongside +renderer state and terminal capture. Use `ui.snapshot()` for the complete +versioned tree or `ui.getNode()` to poll for one exact semantic match. Semantic +nodes carry stable IDs, optional occurrence identity, role, label, hierarchy, +component-owned state, and a transient element handle that `ui.click()` can +resolve safely: + +```ts +const allow = yield* ui.getNode({ + role: "option", + label: "Allow once", + selected: true, + disabled: false, +}) + +yield* ui.click(allow) +``` + +`ui.snapshot` and atomic semantic clicks are negotiated as optional +capabilities so ordinary operations remain compatible with older OpenCode +checkouts. Calling `ui.snapshot()`, `ui.getNode()`, or `ui.click(node)` when its +required capability is unavailable fails locally with `UiCapabilityError`. + Capability errors are typed and the concrete classes are grouped under `Errors`. UI timeouts remain owner-fatal even when caught; recover locally from errors for which the script has a truthful fallback: -Polling timeouts from `ui.waitFor` and `ui.getElement` make one best-effort, -bounded `ui.capture` request. When it succeeds, the resulting normalized -terminal frame is available as `error.frame` without creating or retaining a -screenshot file. RPC-level timeouts and failed diagnostic captures leave -`error.frame` undefined. +Polling timeouts from `ui.waitFor`, `ui.getElement`, and `ui.getNode` make one +best-effort, bounded `ui.capture` request. When it succeeds, the resulting +normalized terminal frame is available as `error.frame` without creating or +retaining a screenshot file. RPC-level timeouts and failed diagnostic captures +leave `error.frame` undefined. ```ts import { Effect } from "effect" diff --git a/packages/drive/docs/open-code-driver-api.md b/packages/drive/docs/open-code-driver-api.md index 37d2687..3715dc0 100644 --- a/packages/drive/docs/open-code-driver-api.md +++ b/packages/drive/docs/open-code-driver-api.md @@ -401,6 +401,22 @@ Predicates passed to `ui.waitFor` may return a boolean or an Effect. Capability methods expose typed error channels. Concrete tagged errors are available from the `Errors` namespace. +`ui.snapshot()` returns the endpoint's versioned semantic tree. `ui.getNode()` +polls for one exact match and fails with `UiNodeAmbiguousError` when more than +one node matches. Semantic snapshots and identity-checked semantic clicks are +optional during negotiation, so older OpenCode checkouts retain ordinary UI +control while unsupported semantic operations fail locally with +`UiCapabilityError`. + +```ts +const option = yield* ui.getNode({ + role: "option", + label: "Allow once", + selected: true, +}) +yield* ui.click(option) +``` + ### Additional TUI ```ts diff --git a/packages/drive/src/cli/commands.ts b/packages/drive/src/cli/commands.ts index de65c8f..206f6f0 100644 --- a/packages/drive/src/cli/commands.ts +++ b/packages/drive/src/cli/commands.ts @@ -35,6 +35,10 @@ export const commandInfo = { value: false, description: "Return focus, elements, and available UI actions", }, + "ui.snapshot": { + value: false, + description: "Return the semantic UI tree as JSON", + }, "ui.matches": { value: true, description: "Check for literal screen text using JSON params", @@ -44,15 +48,17 @@ export const commandInfo = { description: "Finish recording and return the timeline path", }, } as const satisfies Record< - Frontend.Capability, + Exclude, { readonly value: boolean | "optional"; readonly description: string } > -export function isCommandName(operation: string): operation is Frontend.Capability { +type CommandName = Exclude + +export function isCommandName(operation: string): operation is CommandName { return Object.hasOwn(commandInfo, operation) } -export function commandAcceptsValue(operation: Frontend.Capability) { +export function commandAcceptsValue(operation: CommandName) { return commandInfo[operation].value } @@ -179,6 +185,27 @@ function dispatch( connection: SimulationConnector.UiConnection, request: Frontend.Request, ): Effect.Effect { + if ( + request.method === "ui.snapshot" && + !SimulationConnector.supportsCapability(connection.compatibility, "ui.snapshot") + ) + return Effect.fail( + new SimulationError( + "ui.snapshot is not available on this OpenCode endpoint", + request.method, + ), + ) + if ( + request.method === "ui.click" && + request.params.semantic !== undefined && + !SimulationConnector.supportsCapability(connection.compatibility, "ui.click.semantic") + ) + return Effect.fail( + new SimulationError( + "semantic ui.click is not available on this OpenCode endpoint", + request.method, + ), + ) switch (request.method) { case "ui.type": return connection.rpc["ui.type"](request.params) @@ -200,6 +227,8 @@ function dispatch( return connection.rpc["ui.capture"]() case "ui.state": return connection.rpc["ui.state"]() + case "ui.snapshot": + return connection.rpc["ui.snapshot"]() case "ui.matches": return connection.rpc["ui.matches"](request.params) case "ui.recording.finish": diff --git a/packages/drive/src/cli/send.ts b/packages/drive/src/cli/send.ts index 2fee86f..2ecf476 100644 --- a/packages/drive/src/cli/send.ts +++ b/packages/drive/src/cli/send.ts @@ -19,7 +19,7 @@ export async function send(options: SendOptions) { } if ( options.commands.length === 1 && - ["ui.state", "ui.capture"].includes( + ["ui.state", "ui.snapshot", "ui.capture"].includes( options.commands[0]?.operation ?? "", ) ) { diff --git a/packages/drive/src/cli/types.ts b/packages/drive/src/cli/types.ts index d55727e..532e23d 100644 --- a/packages/drive/src/cli/types.ts +++ b/packages/drive/src/cli/types.ts @@ -1,7 +1,7 @@ import type { Frontend } from "../client/index.js" export interface DriveCommand { - readonly operation: Frontend.Capability + readonly operation: Exclude readonly value?: string } diff --git a/packages/drive/src/driver/index.ts b/packages/drive/src/driver/index.ts index f9a8bad..d2d699c 100644 --- a/packages/drive/src/driver/index.ts +++ b/packages/drive/src/driver/index.ts @@ -184,7 +184,9 @@ export { LlmSettlementError, } from "./llm-controller.js" export { + UiCapabilityError, UiElementAmbiguousError, + UiNodeAmbiguousError, UiPredicateError, UiTimeoutError, UiWaitOptionsError, diff --git a/packages/drive/src/driver/llm-responder.ts b/packages/drive/src/driver/llm-responder.ts index 2a3ce8c..59826a3 100644 --- a/packages/drive/src/driver/llm-responder.ts +++ b/packages/drive/src/driver/llm-responder.ts @@ -4,7 +4,10 @@ import * as Schema from "effect/Schema" import * as Stream from "effect/Stream" import * as Llm from "../llm/index.js" import { chunkText } from "../llm/internal.js" -import type { BackendConnection } from "../simulation/connector.js" +import { + supportsCapability, + type BackendConnection, +} from "../simulation/connector.js" import { controllerError, LlmControllerError } from "./llm-errors.js" /** @@ -70,8 +73,7 @@ export const make = ({ requestTimeout }: Options): Responder => { ): Effect.Effect => Effect.gen(function* () { if ( - backend.compatibility._tag !== "Negotiated" || - !backend.compatibility.capabilities.includes("llm.pending") + !supportsCapability(backend.compatibility, "llm.pending") ) return yield* Effect.fail(error) const pending = yield* Effect.exit( diff --git a/packages/drive/src/driver/ui.ts b/packages/drive/src/driver/ui.ts index efe6c44..bab25a5 100644 --- a/packages/drive/src/driver/ui.ts +++ b/packages/drive/src/driver/ui.ts @@ -2,7 +2,10 @@ import * as Effect from "effect/Effect" import * as Schedule from "effect/Schedule" import * as Schema from "effect/Schema" import type { RpcClientError } from "effect/unstable/rpc" -import type { UiConnection } from "../simulation/connector.js" +import { + supportsCapability, + type UiConnection, +} from "../simulation/connector.js" import { Frontend } from "../client/protocol.js" import type { SimulationRequestError } from "../simulation/rpc.js" @@ -22,6 +25,8 @@ export interface ElementQuery { readonly editor?: boolean } +export type SemanticQuery = Partial + export type Position = Pick export type Predicate = (state: Frontend.State) => boolean @@ -50,6 +55,25 @@ export class UiElementAmbiguousError extends Schema.TaggedErrorClass()( + "UiNodeAmbiguousError", + { + count: Schema.Number, + }, +) { + override get message() { + return `ui.getNode matched ${this.count} semantic nodes` + } +} + +export class UiCapabilityError extends Schema.TaggedErrorClass()( + "UiCapabilityError", + { + capability: Schema.Literals(["ui.snapshot", "ui.click.semantic"]), + message: Schema.String, + }, +) {} + export class UiWaitOptionsError extends Schema.TaggedErrorClass()( "UiWaitOptionsError", { @@ -77,9 +101,11 @@ const RequestTimeout = Schema.Finite.check(Schema.isGreaterThanOrEqualTo(0)) export type WaitError = UiTimeoutError | UiWaitOptionsError type RpcError = SimulationRequestError | RpcClientError.RpcClientError export type OperationError = RpcError | UiTimeoutError +export type SemanticOperationError = OperationError | UiCapabilityError export interface Ui { readonly state: () => Effect.Effect + readonly snapshot: () => Effect.Effect readonly capture: () => Effect.Effect readonly matches: (text: string) => Effect.Effect readonly screenshot: ( @@ -98,11 +124,11 @@ export interface Ui { target: number | Frontend.Element, ) => Effect.Effect readonly click: ( - target: number | Frontend.Element, + target: number | Frontend.Element | Frontend.SemanticNode, position?: Position, ) => Effect.Effect< Frontend.State, - OperationError | UiElementAmbiguousError | UiWaitOptionsError + OperationError | UiCapabilityError | UiElementAmbiguousError | UiWaitOptionsError > readonly resize: ( viewport: Frontend.ResizeParams, @@ -124,6 +150,13 @@ export interface Ui { Frontend.Element, OperationError | WaitError | UiElementAmbiguousError > + readonly getNode: ( + target: string | SemanticQuery, + options?: WaitOptions, + ) => Effect.Effect< + Frontend.SemanticNode, + SemanticOperationError | WaitError | UiNodeAmbiguousError + > } interface Control extends Ui { @@ -137,6 +170,7 @@ export interface Transform { /** Applies one Effect transformation to every operation without changing the UI interface. */ export const transform = (ui: Ui, apply: Transform): Ui => ({ state: () => apply(ui.state()), + snapshot: () => apply(ui.snapshot()), capture: () => apply(ui.capture()), matches: (text) => apply(ui.matches(text)), screenshot: (name) => apply(ui.screenshot(name)), @@ -150,6 +184,7 @@ export const transform = (ui: Ui, apply: Transform): Ui => ({ submit: (text) => apply(ui.submit(text)), waitFor: (target, options) => apply(ui.waitFor(target, options)), getElement: (target, options) => apply(ui.getElement(target, options)), + getNode: (target, options) => apply(ui.getNode(target, options)), }) export const make = (connection: UiConnection, options?: Options): Control => { @@ -173,6 +208,18 @@ export const make = (connection: UiConnection, options?: Options): Control => { }) const state = Effect.fn("Ui.state")(() => call("state", rpc["ui.state"]())) + const snapshot = Effect.fn("Ui.snapshot")(function* () { + if ( + !supportsCapability(connection.compatibility, "ui.snapshot") + ) + return yield* Effect.fail( + new UiCapabilityError({ + capability: "ui.snapshot", + message: "ui.snapshot is not available on this OpenCode endpoint", + }), + ) + return yield* call("snapshot", rpc["ui.snapshot"]()) + }) const capture = Effect.fn("Ui.capture")(() => call("capture", rpc["ui.capture"]()), ) @@ -315,7 +362,7 @@ export const make = (connection: UiConnection, options?: Options): Control => { ? element.num === target : typeof target === "string" ? element.id === target - : matchesElement(element, target), + : matchesQuery(element, target), ) if (elements.length > 1) return Effect.fail( @@ -328,12 +375,60 @@ export const make = (connection: UiConnection, options?: Options): Control => { ), ) + const getNode = Effect.fn("Ui.getNode")( + (target: string | SemanticQuery, options?: WaitOptions) => + poll( + "getNode", + Effect.flatMap(snapshot(), (value) => { + const nodes = value.nodes.filter((node) => + typeof target === "string" + ? node.id === target + : matchesQuery(node, target), + ) + if (nodes.length > 1) + return Effect.fail( + new UiNodeAmbiguousError({ count: nodes.length }), + ) + return Effect.succeed(nodes[0]) + }), + options, + "timed out waiting for the semantic UI node", + ), + ) + const click = Effect.fn("Ui.click")(function* ( - target: number | Frontend.Element, + target: number | Frontend.Element | Frontend.SemanticNode, position?: Position, ) { + if (typeof target !== "number" && "element" in target) { + if (!supportsCapability(connection.compatibility, "ui.click.semantic")) + return yield* Effect.fail( + new UiCapabilityError({ + capability: "ui.click.semantic", + message: "semantic ui.click is not available on this OpenCode endpoint", + }), + ) + const element = position === undefined + ? (yield* state()).elements.find((candidate) => candidate.num === target.element) + : undefined + return yield* call( + "click", + rpc["ui.click"]({ + target: target.element, + x: position?.x ?? (element === undefined ? 0 : Math.floor(element.width / 2)), + y: position?.y ?? (element === undefined ? 0 : Math.floor(element.height / 2)), + semantic: { + id: target.id, + ...(target.instance === undefined ? {} : { instance: target.instance }), + element: target.element, + }, + }), + ) + } const element = - typeof target === "number" ? yield* getElement(target) : target + typeof target === "number" + ? yield* getElement(target) + : target return yield* call( "click", rpc["ui.click"]({ @@ -346,6 +441,7 @@ export const make = (connection: UiConnection, options?: Options): Control => { return { state, + snapshot, capture, matches, screenshot, @@ -360,6 +456,7 @@ export const make = (connection: UiConnection, options?: Options): Control => { submit, waitFor, getElement, + getNode, } } @@ -387,14 +484,9 @@ function predicateEffect( }) } -function matchesElement(element: Frontend.Element, query: ElementQuery) { - return ( - (query.id === undefined || element.id === query.id) && - (query.num === undefined || element.num === query.num) && - (query.focusable === undefined || element.focusable === query.focusable) && - (query.focused === undefined || element.focused === query.focused) && - (query.clickable === undefined || element.clickable === query.clickable) && - (query.editor === undefined || element.editor === query.editor) +function matchesQuery(value: Value, query: Partial) { + return Object.entries(query).every( + ([key, expected]) => expected === undefined || Reflect.get(value, key) === expected, ) } diff --git a/packages/drive/src/script/errors.ts b/packages/drive/src/script/errors.ts index 44a56b2..7f709b7 100644 --- a/packages/drive/src/script/errors.ts +++ b/packages/drive/src/script/errors.ts @@ -8,7 +8,9 @@ export { FileSystemError } from "../project.js" export { OpenCodeDriverError } from "../driver/error.js" export { LlmControllerError, LlmModeError } from "../driver/llm-controller.js" export { + UiCapabilityError, UiElementAmbiguousError, + UiNodeAmbiguousError, UiPredicateError, UiTimeoutError, UiWaitOptionsError, diff --git a/packages/drive/src/simulation/connector.ts b/packages/drive/src/simulation/connector.ts index 8f86153..ceb9037 100644 --- a/packages/drive/src/simulation/connector.ts +++ b/packages/drive/src/simulation/connector.ts @@ -52,6 +52,14 @@ export const EndpointCompatibility = Schema.TaggedUnion({ }) export type EndpointCompatibility = typeof EndpointCompatibility.Type +export function supportsCapability( + compatibility: EndpointCompatibility, + capability: Handshake.Capability, +) { + return compatibility._tag === "Negotiated" && + compatibility.capabilities.includes(capability) +} + export type CompatibilityPolicy = "required" | "preferred" export interface Options { @@ -104,16 +112,20 @@ export const ui = Effect.fn("SimulationConnector.ui")(function* ( const rpc = yield* RpcClient.make(UiRpcs).pipe( Effect.provideService(RpcClient.Protocol, protocol), ) + const required = FrontendProtocol.Capabilities.filter( + (capability) => + capability !== "ui.snapshot" && capability !== "ui.click.semantic", + ) const compatibility = yield* negotiate( endpoint, "ui", - FrontendProtocol.Capabilities, + required, rpc["simulation.handshake"]({ client: { name: "opencode-drive", version: packageJson.version }, expectedRole: "ui", offeredVersions: [1], - requiredCapabilities: [...FrontendProtocol.Capabilities], - optionalCapabilities: [], + requiredCapabilities: required, + optionalCapabilities: ["ui.snapshot", "ui.click.semantic"], }), options?.compatibility, ) diff --git a/packages/drive/src/simulation/protocol.ts b/packages/drive/src/simulation/protocol.ts index 4071e93..66b8b70 100644 --- a/packages/drive/src/simulation/protocol.ts +++ b/packages/drive/src/simulation/protocol.ts @@ -104,10 +104,12 @@ export namespace Frontend { "ui.arrow", "ui.focus", "ui.click", + "ui.click.semantic", "ui.resize", "ui.matches", "ui.screenshot", "ui.state", + "ui.snapshot", "ui.capture", "ui.recording.finish", ] as const satisfies ReadonlyArray @@ -123,6 +125,14 @@ export namespace Frontend { export interface KeyModifiers extends Schema.Schema.Type {} + export const SemanticClickTarget = Schema.Struct({ + id: Schema.NonEmptyString, + instance: Schema.optionalKey(Schema.NonEmptyString), + element: Schema.Number.check(Schema.isInt(), Schema.isGreaterThan(0)), + }) + export interface SemanticClickTarget + extends Schema.Schema.Type {} + export const Action = Schema.Union([ Schema.Struct({ type: Schema.Literal("ui.type"), text: Schema.String }), Schema.Struct({ @@ -141,6 +151,7 @@ export namespace Frontend { target: Schema.Number, x: Schema.Number, y: Schema.Number, + semantic: Schema.optionalKey(SemanticClickTarget), }), Schema.Struct({ type: Schema.Literal("ui.resize"), @@ -173,6 +184,46 @@ export namespace Frontend { }) export interface State extends Schema.Schema.Type {} + export const SemanticNode = Schema.Struct({ + id: Schema.NonEmptyString, + instance: Schema.optionalKey(Schema.NonEmptyString), + parent: Schema.optionalKey(Schema.NonEmptyString), + role: Schema.NonEmptyString, + label: Schema.optionalKey(Schema.NonEmptyString), + element: Schema.Number.check(Schema.isInt(), Schema.isGreaterThan(0)), + focused: Schema.optionalKey(Schema.Boolean), + selected: Schema.optionalKey(Schema.Boolean), + expanded: Schema.optionalKey(Schema.Boolean), + disabled: Schema.optionalKey(Schema.Boolean), + }) + export interface SemanticNode extends Schema.Schema.Type {} + + export const SemanticSnapshot = Schema.Struct({ + format: Schema.Literal("opencode-ui-snapshot-v1"), + nodes: Schema.Array(SemanticNode).check( + Schema.makeFilter((nodes) => { + const ids = new Set(nodes.map((node) => node.id)) + if (ids.size !== nodes.length) return "semantic node ids must be unique" + if (new Set(nodes.map((node) => node.element)).size !== nodes.length) + return "semantic node elements must be unique" + if (nodes.some((node) => node.parent !== undefined && !ids.has(node.parent))) + return "semantic node parents must reference another node" + const parents = new Map(nodes.map((node) => [node.id, node.parent])) + for (const node of nodes) { + const visited = new Set() + let current: string | undefined = node.id + while (current !== undefined) { + if (visited.has(current)) return "semantic node hierarchy must be acyclic" + visited.add(current) + current = parents.get(current) + } + } + return undefined + }), + ), + }) + export interface SemanticSnapshot extends Schema.Schema.Type {} + export const Screenshot = Schema.String export type Screenshot = Schema.Schema.Type @@ -250,6 +301,7 @@ export namespace Frontend { target: Schema.Number, x: Schema.Number, y: Schema.Number, + semantic: Schema.optionalKey(SemanticClickTarget), }) export interface ClickParams extends Schema.Schema.Type {} @@ -304,7 +356,12 @@ export namespace Frontend { }), Schema.Struct({ ...JsonRpc.RequestFields, - method: Schema.Literals(["ui.enter", "ui.state", "ui.recording.finish"]), + method: Schema.Literals([ + "ui.enter", + "ui.state", + "ui.snapshot", + "ui.recording.finish", + ]), }), Schema.Struct({ ...JsonRpc.RequestFields, diff --git a/packages/drive/src/simulation/rpc.ts b/packages/drive/src/simulation/rpc.ts index 6491955..f544311 100644 --- a/packages/drive/src/simulation/rpc.ts +++ b/packages/drive/src/simulation/rpc.ts @@ -34,6 +34,7 @@ export const UiRpcs = RpcGroup.make( success: Handshake.Response, }), request("ui.state", { success: Frontend.State }), + request("ui.snapshot", { success: Frontend.SemanticSnapshot }), request("ui.capture", { success: Frontend.CapturedFrame }), request("ui.matches", { payload: Frontend.MatchesParams, diff --git a/packages/drive/test/cli/integration.test.ts b/packages/drive/test/cli/integration.test.ts index 3719a2c..14ce542 100644 --- a/packages/drive/test/cli/integration.test.ts +++ b/packages/drive/test/cli/integration.test.ts @@ -11,8 +11,10 @@ import { type InstanceManifest, } from "../../src/instance/registry.js" import { createResponseSettings, generateResponse } from "../../src/cli/response-generator.js" +import { CommandBatchError, executeCommands } from "../../src/cli/commands.js" import { splitText } from "../../src/cli/mock-backend.js" import { resolveSendEndpoint } from "../../src/cli/send.js" +import { sendError, sendResult, startTransportPeer } from "../simulation/transport-peer.js" const roots: string[] = [] const instances: Array<{ root: string; name: string }> = [] @@ -29,6 +31,72 @@ afterEach(async () => { }) describe("opencode-drive", () => { + test("rejects unsupported optional UI commands without sending them", async () => { + const peers = [ + startTransportPeer(({ request, socket }) => { + if (request.method !== "simulation.handshake") return + const params = request.params as { + readonly requiredCapabilities: ReadonlyArray + } + sendResult(socket, request, { + protocolVersion: 1, + role: "ui", + server: { name: "opencode", version: "older" }, + capabilities: params.requiredCapabilities, + }) + }, { handshake: false }), + startTransportPeer(({ request, socket }) => { + if (request.method === "simulation.handshake") + sendError(socket, request, "method not found", -32601) + }, { handshake: false }), + ] + + try { + for (const peer of peers) { + for (const expected of [ + { + command: { operation: "ui.snapshot" as const }, + method: "ui.snapshot", + message: "ui.snapshot is not available on this OpenCode endpoint", + }, + { + command: { + operation: "ui.click" as const, + value: JSON.stringify({ + target: 3, + x: 1, + y: 0, + semantic: { + id: "session.permission.action.once", + instance: "permission-1", + element: 3, + }, + }), + }, + method: "ui.click", + message: "semantic ui.click is not available on this OpenCode endpoint", + }, + ]) { + const result = executeCommands(peer.url, [expected.command]) + await expect(result).rejects.toMatchObject({ + name: "CommandBatchError", + reason: { + name: "SimulationError", + method: expected.method, + message: expected.message, + }, + } satisfies Partial) + } + expect(peer.received.map(({ request }) => request.method)).toEqual([ + "simulation.handshake", + "simulation.handshake", + ]) + } + } finally { + await Promise.all(peers.map((peer) => peer.stop())) + } + }) + test("requires an explicit name to initialize or start headless", async () => { const root = await temporary() for (const command of ["init", "start"]) { @@ -115,6 +183,13 @@ describe("opencode-drive", () => { expect(await state.exited).toBe(0) expect(JSON.parse(await new Response(state.stdout).text()).focused.editor).toBe(true) + const snapshot = spawn(["send", "--name", name, "--command.ui.snapshot"], root) + expect(await snapshot.exited).toBe(0) + expect(JSON.parse(await new Response(snapshot.stdout).text())).toMatchObject({ + format: "opencode-ui-snapshot-v1", + nodes: [{ id: "prompt", role: "textbox", element: 1 }], + }) + const matches = spawn(["send", "--name", name, "--command.ui.matches", '{"text":"Fake OpenCode"}'], root) expect(await matches.exited).toBe(0) expect(await new Response(matches.stdout).text()).toBe("true\n") diff --git a/packages/drive/test/driver/ui.test.ts b/packages/drive/test/driver/ui.test.ts index dde8eb8..8138a69 100644 --- a/packages/drive/test/driver/ui.test.ts +++ b/packages/drive/test/driver/ui.test.ts @@ -22,6 +22,31 @@ const state = { elements: [editor], } +const snapshot = { + format: "opencode-ui-snapshot-v1", + nodes: [ + { + id: "session.permission", + instance: "permission-1", + role: "dialog", + label: "Permission required: Edit fixture.txt", + element: 2, + expanded: false, + }, + { + id: "session.permission.action.once", + instance: "permission-1", + parent: "session.permission", + role: "option", + label: "Allow once", + element: 3, + focused: true, + selected: true, + disabled: false, + }, + ], +} + const frame = { cols: 2, rows: 1, @@ -115,6 +140,174 @@ describe("OpenCodeUi", () => { }) }) + it.live("selects and clicks semantic UI nodes", () => { + let snapshotCalls = 0 + const peer = startTransportPeer(({ request, socket }) => { + if (request.method === "ui.snapshot") { + snapshotCalls++ + sendResult( + socket, + request, + snapshotCalls === 2 + ? { format: "opencode-ui-snapshot-v1", nodes: [] } + : snapshot, + ) + return + } + sendResult(socket, request, state) + }) + + return Effect.gen(function* () { + yield* Effect.addFinalizer(() => Effect.promise(() => peer.stop())) + const connection = yield* SimulationConnector.ui(peer.url) + const ui = OpenCodeUi.make(connection) + + expect(yield* ui.snapshot()).toEqual(snapshot) + const option = yield* ui.getNode({ + instance: "permission-1", + role: "option", + selected: true, + disabled: false, + }, { interval: 1 }) + expect(option).toEqual(snapshot.nodes[1]) + expect(yield* ui.click(option)).toEqual(state) + expect(peer.received.map(({ request }) => request)).toEqual([ + { jsonrpc: "2.0", id: 1, method: "ui.snapshot" }, + { jsonrpc: "2.0", id: 2, method: "ui.snapshot" }, + { jsonrpc: "2.0", id: 3, method: "ui.snapshot" }, + { jsonrpc: "2.0", id: 4, method: "ui.state" }, + { + jsonrpc: "2.0", + id: 5, + method: "ui.click", + params: { + target: 3, + x: 10, + y: 3, + semantic: { + id: "session.permission.action.once", + instance: "permission-1", + element: 3, + }, + }, + }, + ]) + }) + }) + + it.live("sends stale semantic handles to the endpoint without polling", () => { + const peer = startTransportPeer(({ request, socket }) => { + if (request.method === "ui.state") { + sendResult(socket, request, { ...state, elements: [] }) + return + } + sendError(socket, request, "target is stale") + }) + + return Effect.gen(function* () { + yield* Effect.addFinalizer(() => Effect.promise(() => peer.stop())) + const connection = yield* SimulationConnector.ui(peer.url) + const error = yield* OpenCodeUi.make(connection) + .click(snapshot.nodes[1]!) + .pipe(Effect.flip) + + expect(error).toMatchObject({ + _tag: "SimulationRequestError", + method: "ui.click", + message: "target is stale", + }) + expect(peer.received.map(({ request }) => request)).toEqual([ + { jsonrpc: "2.0", id: 1, method: "ui.state" }, + { + jsonrpc: "2.0", + id: 2, + method: "ui.click", + params: { + target: 3, + x: 0, + y: 0, + semantic: { + id: "session.permission.action.once", + instance: "permission-1", + element: 3, + }, + }, + }, + ]) + }) + }) + + it.live("reports unavailable semantic snapshots without breaking older endpoints", () => { + const peer = startTransportPeer(({ request, socket }) => { + if (request.method === "simulation.handshake") { + const params = request.params as { + readonly requiredCapabilities: ReadonlyArray + } + sendResult(socket, request, { + protocolVersion: 1, + role: "ui", + server: { name: "opencode", version: "older" }, + capabilities: params.requiredCapabilities, + }) + return + } + sendResult(socket, request, state) + }, { handshake: false }) + + return Effect.gen(function* () { + yield* Effect.addFinalizer(() => Effect.promise(() => peer.stop())) + const connection = yield* SimulationConnector.ui(peer.url) + const ui = OpenCodeUi.make(connection) + + expect(yield* ui.state()).toEqual(state) + const error = yield* ui.snapshot().pipe(Effect.flip) + expect(error).toBeInstanceOf(OpenCodeUi.UiCapabilityError) + expect(error).toMatchObject({ + capability: "ui.snapshot", + message: "ui.snapshot is not available on this OpenCode endpoint", + }) + const clickError = yield* ui.click(snapshot.nodes[1]!).pipe(Effect.flip) + expect(clickError).toMatchObject({ + capability: "ui.click.semantic", + message: "semantic ui.click is not available on this OpenCode endpoint", + }) + expect(peer.received.map(({ request }) => request.method)).toEqual([ + "simulation.handshake", + "ui.state", + ]) + }) + }) + + it.live("reports ambiguous semantic nodes as typed UI failures", () => { + const peer = startTransportPeer(({ request, socket }) => + sendResult(socket, request, { + ...snapshot, + nodes: [ + snapshot.nodes[0], + snapshot.nodes[1], + { + ...snapshot.nodes[1], + id: "session.permission.action.always", + element: 4, + }, + ], + }), + ) + + return Effect.gen(function* () { + yield* Effect.addFinalizer(() => Effect.promise(() => peer.stop())) + const connection = yield* SimulationConnector.ui(peer.url) + const error = yield* OpenCodeUi.make(connection) + .getNode({ role: "option" }) + .pipe(Effect.flip) + expect(error).toBeInstanceOf(OpenCodeUi.UiNodeAmbiguousError) + expect(error).toMatchObject({ + count: 2, + message: "ui.getNode matched 2 semantic nodes", + }) + }) + }) + it.live("reports ambiguous elements as typed UI failures", () => { const peer = startTransportPeer(({ request, socket }) => sendResult(socket, request, { diff --git a/packages/drive/test/fixtures/fake-opencode.ts b/packages/drive/test/fixtures/fake-opencode.ts index 2e5fd08..9438a24 100644 --- a/packages/drive/test/fixtures/fake-opencode.ts +++ b/packages/drive/test/fixtures/fake-opencode.ts @@ -232,6 +232,21 @@ function frontend(method: string, params: unknown) { lines: [{ spans: [{ text: screen.value, fg: [255, 255, 255, 255], bg: [0, 0, 0, 255], attributes: 0, width: screen.value.length }] }], } } + if (method === "ui.snapshot") { + return { + format: "opencode-ui-snapshot-v1", + nodes: [ + { + id: "prompt", + role: "textbox", + label: "Prompt", + element: 1, + focused: true, + disabled: false, + }, + ], + } + } if (method === "ui.screenshot") { const name = isRecord(params) && typeof params.name === "string" ? params.name diff --git a/packages/drive/test/simulation/ui.test.ts b/packages/drive/test/simulation/ui.test.ts index f3f63ab..ed42ca8 100644 --- a/packages/drive/test/simulation/ui.test.ts +++ b/packages/drive/test/simulation/ui.test.ts @@ -9,10 +9,19 @@ const state: Frontend.State = { elements: [], } +const snapshot: Frontend.SemanticSnapshot = { + format: "opencode-ui-snapshot-v1", + nodes: [{ id: "prompt", role: "textbox", element: 1, focused: true }], +} + describe("OpenCode UI simulation transport", () => { it.live("preserves every UI call's exact JSON-RPC frame", () => Effect.gen(function* () { const peer = startTransportPeer(({ request, socket }) => { + if (request.method === "ui.snapshot") { + sendResult(socket, request, snapshot) + return + } if (request.method === "ui.matches") { sendResult(socket, request, true) return @@ -36,6 +45,7 @@ describe("OpenCode UI simulation transport", () => { const { rpc } = yield* SimulationConnector.ui(peer.url) expect(yield* rpc["ui.state"]()).toEqual(state) + expect(yield* rpc["ui.snapshot"]()).toEqual(snapshot) expect(yield* rpc["ui.matches"]({ text: "needle" })).toBe(true) expect(yield* rpc["ui.screenshot"](undefined)).toBe("/tmp/screenshot.png") expect(yield* rpc["ui.screenshot"]({ name: "home" })).toBe("/tmp/home.png") @@ -61,72 +71,73 @@ describe("OpenCode UI simulation transport", () => { expect(peer.received.map(({ request }) => request)).toEqual([ { jsonrpc: "2.0", id: 1, method: "ui.state" }, + { jsonrpc: "2.0", id: 2, method: "ui.snapshot" }, { jsonrpc: "2.0", - id: 2, + id: 3, method: "ui.matches", params: { text: "needle" }, }, - { jsonrpc: "2.0", id: 3, method: "ui.screenshot" }, + { jsonrpc: "2.0", id: 4, method: "ui.screenshot" }, { jsonrpc: "2.0", - id: 4, + id: 5, method: "ui.screenshot", params: { name: "home" }, }, - { jsonrpc: "2.0", id: 5, method: "ui.recording.finish" }, + { jsonrpc: "2.0", id: 6, method: "ui.recording.finish" }, { jsonrpc: "2.0", - id: 6, + id: 7, method: "ui.type", params: { text: "hello" }, }, { jsonrpc: "2.0", - id: 7, + id: 8, method: "ui.press", params: { key: "x" }, }, { jsonrpc: "2.0", - id: 8, + id: 9, method: "ui.press", params: { key: "x", modifiers: { ctrl: true, shift: false } }, }, { jsonrpc: "2.0", - id: 9, + id: 10, method: "ui.press", params: { key: "escape" }, }, - { jsonrpc: "2.0", id: 10, method: "ui.enter" }, + { jsonrpc: "2.0", id: 11, method: "ui.enter" }, { jsonrpc: "2.0", - id: 11, + id: 12, method: "ui.arrow", params: { direction: "left" }, }, { jsonrpc: "2.0", - id: 12, + id: 13, method: "ui.focus", params: { target: 7 }, }, { jsonrpc: "2.0", - id: 13, + id: 14, method: "ui.click", params: { target: 7, x: 3, y: 2 }, }, { jsonrpc: "2.0", - id: 14, + id: 15, method: "ui.resize", params: { cols: 120, rows: 40 }, }, { jsonrpc: "2.0", - id: 15, + id: 16, method: "ui.screenshot", params: { name: "fail" }, }, diff --git a/skills/opencode-drive/SKILL.md b/skills/opencode-drive/SKILL.md index 9e5e0ec..d549bc9 100644 --- a/skills/opencode-drive/SKILL.md +++ b/skills/opencode-drive/SKILL.md @@ -129,6 +129,7 @@ UI operations are Effects: - `ui.submit(text)` types and presses Enter. - `ui.state()`, `ui.capture()`, and `ui.matches(text)` inspect the terminal. +- `ui.snapshot()` returns the versioned semantic tree; `ui.getNode(query, options?)` polls for one exact semantic match. - `ui.waitFor(textOrPredicate, options?)` polls until a match. - `ui.getElement(query, options?)`, `ui.focus(...)`, and `ui.click(...)` target interactive elements. - `ui.screenshot(name?)` exports an image and returns its absolute path. @@ -353,6 +354,7 @@ opencode-drive stop --name demo - `--command.ui.resize '{"cols":120,"rows":40}'` - `--command.ui.screenshot` or `--command.ui.screenshot '{"name":"home"}'` - `--command.ui.state` +- `--command.ui.snapshot` - `--command.ui.capture` - `--command.ui.matches '{"text":"OpenCode"}'` - `--command.ui.recording.finish`