diff --git a/README.md b/README.md index cb9f75f..ac51e2a 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,8 @@
-View and control Android (for now) emulators from OpenCode Desktop on Windows, Linux, and -macOS. +View and control Android emulators from OpenCode Desktop on Windows, Linux, and macOS, and iOS +simulators on macOS. ## Requirements @@ -13,6 +13,7 @@ macOS. - Node 22.19+ on the 22.x line, or Node 24+, with npm. - Android SDK Platform-Tools and Emulator, a configured Android Virtual Device, and hardware virtualization enabled. - ffmpeg with libx264 on the desktop process's PATH. +- For iOS (macOS only): Xcode with an iOS Simulator runtime, selected with `xcode-select`. Set `ANDROID_HOME` or `ANDROID_SDK_ROOT` if your SDK is outside the standard Android Studio location. Set `OPENCODE_DEVICE_NODE` to your Node executable if the desktop cannot find it. @@ -23,7 +24,7 @@ Add this entry to your OpenCode configuration. ```json { - "plugins": ["github:viralcodex/opencode-demul#v0.1.4"] + "plugins": ["github:viralcodex/opencode-demul#v0.2.0"] } ``` @@ -33,18 +34,19 @@ Android SDK tools, Node and ffmpeg must already be installed. ## Use -The panel opens the first configured emulator and starts it if needed. Use the -picker to switch devices and refresh to rescan. +The panel opens the first running device, or else the first listed one, and starts it if +needed. Use the picker to switch between Android emulators and iOS simulators and refresh to +rescan. Either platform's toolchain may be missing; the picker lists what the host can run. Agent access starts enabled. Use the switch to revoke access for the current session. Agents can only use a device you have opened. -Installing an APK or shutting down an emulator requires a separate permission prompt. +Installing an app or shutting down a device requires a separate permission prompt. Tools use `device_*` names, or `device.*` in Code Mode. They can inspect the screen, launch apps, enter text, interact with elements and take screenshots. -Android emulators only (for now). -[iOS, physical devices, remote servers and direct keyboard input into the panel coming soon.] +The first iOS automation command builds the agent's XCTest runner, which can take a few minutes. +[Physical devices, remote servers and direct keyboard input into the panel coming soon.] Prebuilt plugin files are included. diff --git a/dist/gui.cjs b/dist/gui.cjs index b95884d..74c0c49 100644 --- a/dist/gui.cjs +++ b/dist/gui.cjs @@ -1,23 +1,24 @@ -"use strict";Object.defineProperties(exports,{__esModule:{value:!0},[Symbol.toStringTag]:{value:"Module"}});const Z=require("solid-js/web"),le=require("solid-js"),ft=require("@opencode/plugin/gui"),Is=require("@opencode/schema/rpc"),u=require("effect"),Os=8*1024*1024,je=u.Schema.optional,we=u.Schema.String.check(u.Schema.isMaxLength(2048)),Er=u.Schema.String.check(u.Schema.isMaxLength(1e5)),mt=u.Schema.Int.check(u.Schema.isGreaterThanOrEqualTo(0)),Lr=u.Schema.Int.check(u.Schema.isBetween({minimum:0,maximum:1e6})),Fe=u.Schema.Literals(["ios","android"]).annotate({identifier:"Device.Platform"}),Ps=u.Schema.Literals(["shutdown","booting","booted","unavailable"]).annotate({identifier:"Device.DeviceState"}),Bs=u.Schema.Literals(["available","owned-by-this-host","owned-by-other","unknown"]).annotate({identifier:"Device.Ownership"}),Dr=u.Schema.Literals(["apps","launch","install","snapshot","press","fill","swipe","key","screenshot","stream"]).annotate({identifier:"Device.Capability"}),Ns=u.Schema.Literals(["home","back","recents","lock","volumeUp","volumeDown"]).annotate({identifier:"Device.NavigationKey"}),fn=mt.annotate({identifier:"Device.Generation"}),mn=u.Schema.String.check(u.Schema.isPattern(/^dh_[a-f0-9-]{36}$/u)).pipe(u.Schema.brand("Device.HandleToken")).annotate({identifier:"Device.HandleToken"}),ke=u.Schema.Struct({token:mn,platform:Fe,deviceId:we,generation:fn}).annotate({identifier:"Device.DeviceHandle"}),Nr=u.Schema.String.check(u.Schema.isPattern(/^@?e[1-9][0-9]*$/u)).pipe(u.Schema.brand("Device.ElementRef")).annotate({identifier:"Device.ElementRef"}),Tr=u.Schema.Struct({token:mn,generation:fn,revision:mt,ref:Nr}).annotate({identifier:"Device.SnapshotReference"}),Ms=u.Schema.Struct({ref:je(Nr),role:we,name:je(Er),value:je(Er),enabled:je(u.Schema.Boolean),focused:je(u.Schema.Boolean)}).annotate({identifier:"Device.AccessibilityNode"}),As=u.Schema.Struct({token:mn,generation:fn,revision:mt,interactiveOnly:u.Schema.Boolean,complete:u.Schema.Boolean,nodes:u.Schema.Array(Ms)}).annotate({identifier:"Device.Snapshot"}),hn=u.Schema.Struct({x:Lr,y:Lr}).annotate({identifier:"Device.LogicalPoint"}),Us=u.Schema.Struct({id:we,name:we,platform:Fe,osVersion:je(we),state:Ps,ownership:Bs}).annotate({identifier:"Device.DeviceInfo"}),Rs=u.Schema.Struct({supported:u.Schema.Array(Dr),unsupported:u.Schema.Array(u.Schema.Struct({capability:Dr,reason:we}))}).annotate({identifier:"Device.Capabilities"}),js=u.Schema.Struct({id:we,name:je(we)}).annotate({identifier:"Device.AppInfo"}),$s=u.Schema.Struct({app:we,state:u.Schema.Literals(["launched","foreground","unknown"])}).annotate({identifier:"Device.LaunchResult"}),Ws=u.Schema.Struct({installed:u.Schema.Boolean,app:je(we)}).annotate({identifier:"Device.InstallResult"}),Fs=u.Schema.Struct({data:u.Schema.Uint8ArrayFromBase64.check(u.Schema.isMaxLength(Os)),width:mt,height:mt,deviceId:we,platform:Fe}).annotate({identifier:"Device.Screenshot"}),zs=u.Schema.Literals(["mjpeg","h264"]).annotate({identifier:"Device.StreamProtocol"}),Vs=u.Schema.Struct({platform:Fe,deviceId:we,baseUrl:we,token:we,protocol:zs,loopback:u.Schema.Literal(!0)}).annotate({identifier:"Device.StreamDescriptor"}),Gs=u.Schema.Literals(["missing-prerequisite","permission-denied","device-busy","detached","stale-reference","unsupported","not-found","invalid-artifact","timeout","helper-failed","internal"]).annotate({identifier:"Device.DeviceErrorCode"}),Sn=u.Schema.Struct({code:Gs,message:we,mayHaveExecuted:u.Schema.Boolean}).annotate({identifier:"Device.DeviceError"}),Qe=u.Schema.optional,$e=u.Schema.String.check(u.Schema.isMaxLength(2048)),Mr=$e.check(u.Schema.isMinLength(1)),pn={sessionID:Mr,connectionID:u.Schema.String},Cr={...pn,requestID:u.Schema.String},Ir=u.Schema.Number.check(u.Schema.isInt(),u.Schema.isBetween({minimum:1,maximum:1e6})),Or=u.Schema.optional(u.Schema.Struct({width:Ir,height:Ir})),Ke=u.Schema.Struct({sessionID:Mr,serverID:$e,location:u.Schema.optionalKey(u.Schema.Struct({directory:$e}))}),Ot=u.Schema.Struct({handle:ke,capabilities:Rs,agentAccess:u.Schema.Boolean}).annotate({identifier:"DeviceBridge.Attachment"}),Ar=u.Schema.Union([u.Schema.Struct({type:u.Schema.Literal("list"),platform:Qe(Fe)}),u.Schema.Struct({type:u.Schema.Literal("state")}),u.Schema.Struct({type:u.Schema.Literal("open"),platform:Fe,deviceId:$e,boot:Qe(u.Schema.Boolean)}),u.Schema.Struct({type:u.Schema.Literal("apps"),handle:ke}),u.Schema.Struct({type:u.Schema.Literal("launch"),handle:ke,app:$e,deepLink:Qe($e),relaunch:Qe(u.Schema.Boolean)}),u.Schema.Struct({type:u.Schema.Literal("install"),handle:ke,artifactPath:$e}),u.Schema.Struct({type:u.Schema.Literal("snapshot"),handle:ke,interactiveOnly:Qe(u.Schema.Boolean)}),u.Schema.Struct({type:u.Schema.Literal("press"),handle:ke,reference:Tr}),u.Schema.Struct({type:u.Schema.Literal("tap"),handle:ke,point:hn,viewport:Or}),u.Schema.Struct({type:u.Schema.Literal("fill"),handle:ke,reference:Tr,text:$e}),u.Schema.Struct({type:u.Schema.Literal("swipe"),handle:ke,from:hn,to:hn,viewport:Or}),u.Schema.Struct({type:u.Schema.Literal("key"),handle:ke,key:Ns}),u.Schema.Struct({type:u.Schema.Literal("screenshot"),handle:ke}),u.Schema.Struct({type:u.Schema.Literal("close"),handle:ke,shutdown:Qe(u.Schema.Boolean)})]).pipe(u.Schema.toTaggedUnion("type")).annotate({identifier:"DeviceBridge.Operation"}),qs=u.Schema.Union([u.Schema.Struct({type:u.Schema.Literal("devices"),devices:u.Schema.Array(Us)}),u.Schema.Struct({type:u.Schema.Literal("state"),state:u.Schema.NullOr(Ot)}),u.Schema.Struct({type:u.Schema.Literal("attachment"),attachment:Ot}),u.Schema.Struct({type:u.Schema.Literal("apps"),apps:u.Schema.Array(js)}),u.Schema.Struct({type:u.Schema.Literal("launch"),result:$s}),u.Schema.Struct({type:u.Schema.Literal("install"),result:Ws}),u.Schema.Struct({type:u.Schema.Literal("snapshot"),snapshot:As}),u.Schema.Struct({type:u.Schema.Literal("screenshot"),screenshot:Fs}),u.Schema.Struct({type:u.Schema.Literal("done")})]).pipe(u.Schema.toTaggedUnion("type")).annotate({identifier:"DeviceBridge.Value"}),It=u.Schema.Union([u.Schema.Struct({type:u.Schema.Literal("success"),value:qs}),u.Schema.Struct({type:u.Schema.Literal("failure"),error:Sn})]).pipe(u.Schema.toTaggedUnion("type")).annotate({identifier:"DeviceBridge.Outcome"}),Hs=u.Schema.Union([u.Schema.Struct({type:u.Schema.Literal("success"),stream:Vs}),u.Schema.Struct({type:u.Schema.Literal("failure"),error:Sn})]).pipe(u.Schema.toTaggedUnion("type")).annotate({identifier:"DeviceBridge.StreamOutcome"}),Js=u.Schema.Union([u.Schema.Struct({type:u.Schema.Literal("attached"),connectionID:u.Schema.String,version:u.Schema.Literal(1)}),u.Schema.Struct({type:u.Schema.Literal("command"),connectionID:u.Schema.String,requestID:u.Schema.String}),u.Schema.Struct({type:u.Schema.Literal("cancel"),connectionID:u.Schema.String,requestID:u.Schema.String})]).pipe(u.Schema.toTaggedUnion("type")).annotate({identifier:"DeviceBridge.Control"}),Ie=u.Schema.toStandardSchemaV1,Tt={unavailable:Ie(Sn)};Is.Rpc.define({id:"device.control",methods:{attach:{input:Ie(u.Schema.Struct({...pn,version:u.Schema.Literal(1)})),output:Ie(u.Schema.Literals(["closed","replaced"])),errors:Tt},state:{input:Ie(u.Schema.Struct({...pn,state:u.Schema.NullOr(Ot)})),output:Ie(u.Schema.Void),errors:Tt},command:{input:Ie(u.Schema.Struct(Cr)),output:Ie(Ar),errors:Tt},result:{input:Ie(u.Schema.Struct({...Cr,outcome:It})),output:Ie(u.Schema.Void),errors:Tt}},events:{control:{schema:Ie(Js)}}});const Xs=u.Schema.String.check(u.Schema.isMaxLength(2048)),Ys=ft.Ipc.define({id:"demul.device.bridge.v2",state:u.Schema.Array(u.Schema.Struct({scope:Ke,state:u.Schema.NullOr(Ot)})),methods:{connect:{input:Ke},disconnect:{input:Ke},list:{input:u.Schema.Struct({platform:u.Schema.optional(Fe),serverID:Xs}),output:It},access:{input:u.Schema.Struct({scope:Ke,enabled:u.Schema.Boolean}),output:It},manual:{input:u.Schema.Struct({scope:Ke,operation:Ar}),output:It},stream:{input:u.Schema.Struct({scope:Ke,handle:ke}),output:Hs}}});function Qs(g,T,E,N){if(!(!g||g!==T||E!=="playing"||!Number.isSafeInteger(N.width)||!Number.isSafeInteger(N.height)||N.width<=0||N.height<=0))return{token:g,width:N.width,height:N.height}}function Ks(g,T){return!!T&&g.token===T.token&&g.width===T.width&&g.height===T.height}const Zs=1397050709,Ur=16*1024*1024;function eo(g){let T=!1,E;const N=()=>{const n=T&&document.visibilityState==="visible";if(n&&!E){E=g.stream.protocol==="h264"?oo(g):ao(g);return}n||!E||(E(),E=void 0,g.onState(T?"paused":"idle"))},V=()=>N();return document.addEventListener("visibilitychange",V),{active(n){T=n,N()},destroy(){T=!1,document.removeEventListener("visibilitychange",V),E?.(),E=void 0,g.canvas.width=0,g.canvas.height=0,g.onState("idle")}}}function to(g){const T=new Uint8Array(g),E=new DataView(g);if(T.byteLength<4)return;if(E.getUint32(0)!==Zs)return Pr(T)?{bytes:T,key:Ct(T,5)||Ct(T,7),timestamp:performance.now()*1e3}:void 0;if(T.byteLength<16)return;const N=E.getUint8(4),V=N===1?16:N===2?24:0;if(!V||T.byteLength<=V)return;const n=E.getBigUint64(8);if(n>BigInt(Number.MAX_SAFE_INTEGER))return;const re=T.subarray(V);if(Pr(re))return{bytes:re,key:(E.getUint8(5)&1)===1||Ct(re,5)||Ct(re,7),timestamp:Number(n),serverTimestamp:N===2?Number(E.getBigUint64(16))/1e3:void 0}}function no(g){const T=jr(g).find(E=>(E[0]&31)===7);if(!(!T||T.byteLength<4))return`avc1.${[T[1],T[2],T[3]].map(E=>E.toString(16).padStart(2,"0")).join("")}`}function ro(g,T,E){if(![g.left,g.top,g.width,g.height,T.width,T.height,E.x,E.y].every(Number.isFinite)||g.width<=0||g.height<=0||!Number.isSafeInteger(T.width)||!Number.isSafeInteger(T.height)||T.width<=0||T.height<=0)return;const N=Math.min(g.width/T.width,g.height/T.height),V=T.width*N,n=T.height*N,re=E.x-g.left-(g.width-V)/2,F=E.y-g.top-(g.height-n)/2;if(!(re<0||F<0||re>=V||F>=n))return{x:Math.min(T.width-1,Math.floor(re/N)),y:Math.min(T.height-1,Math.floor(F/N))}}function io(g){if(!g||/[\r\n]/.test(g))return;const T=g.split(";");if(T.shift()?.trim().toLowerCase()!=="multipart/x-mixed-replace")return;const E=T.map(n=>n.trim()).find(n=>n.toLowerCase().startsWith("boundary="))?.slice(9).trim();if(!E)return;const N=E.startsWith('"')&&E.endsWith('"')?E.slice(1,-1):E,V=N.startsWith("--")?N.slice(2):N;if(!(!V||/[\r\n]/.test(V)))return V}function so(g,T){const E=new TextEncoder().encode(`--${T}`),N=new Uint8Array([13,10,13,10]),V=[];let n=0,re=0;for(;ng.byteLength)return{frames:V,rest:g.slice(F)};if(g[q]!==13||g[q+1]!==10){n=q;continue}const G=Br(g,N,q+2);if(G<0)return{frames:V,rest:g.slice(F)};const ie=new TextDecoder("ascii").decode(g.subarray(q+2,G)),j=ie.match(/(?:^|\r\n)Content-Length:\s*(\d+)(?:\r\n|$)/i)?.[1],ce=ie.match(/(?:^|\r\n)Content-Type:\s*([^\r\n]+)(?:\r\n|$)/i)?.[1].trim().toLowerCase(),se=j?Number(j):0,de=G+N.byteLength,he=de+se;if(!Number.isSafeInteger(se)||se<=0||se>Ur||ce!=="image/jpeg"){n=G+N.byteLength;continue}if(he>g.byteLength)return{frames:V,rest:g.slice(F)};const pe=g.slice(de,he);pe.byteLength>=4&&pe[0]===255&&pe[1]===216&&pe.at(-2)===255&&pe.at(-1)===217&&V.push(pe),re=he,n=he}return{frames:V,rest:g.slice(re)}}function oo(g){if(typeof VideoDecoder!="function"||typeof EncodedVideoChunk!="function")return g.onState("paused"),g.onError("H.264 device video requires WebCodecs support in this browser"),()=>{};const T=new URL("/video?frame-meta=1",new URL(g.stream.baseUrl).origin);T.protocol=T.protocol==="https:"?"wss:":"ws:";const E=new WebSocket(T,`opencode-device.${g.stream.token}`);let N,V,n=0,re=!0,F=!1,q=!1;E.binaryType="arraybuffer",g.onState("connecting");const G=()=>{if(E.readyState===WebSocket.OPEN)try{E.send(JSON.stringify({type:"reset-video",ack:!1}))}catch{}},ie=(se=!1)=>{n++;const de=N,he=q;if(N=void 0,V=void 0,re=!0,q=!1,de&&de.state!=="closed")try{de.reset()}finally{de.close()}se&&he&&!F&&g.onState("connecting"),se&&G()},j=se=>{F||(g.onError(se),g.onState("paused"))},ce=(se,de)=>{if(!(F||N!==de||de.state!=="configured"))try{de.decode(new EncodedVideoChunk({type:se.key?"key":"delta",timestamp:se.timestamp,data:se.bytes})),re=!1}catch(he){ie(!0),j(`H.264 decode failed: ${he instanceof Error?he.message:String(he)}`)}};return E.addEventListener("open",()=>G()),E.addEventListener("message",se=>{if(F)return;if(typeof se.data=="string"){co(se.data)&&ie(!0);return}if(!(se.data instanceof ArrayBuffer))return;const de=to(se.data);if(!de)return;if(N&&N.decodeQueueSize>2){ie(!0);return}if(re&&!de.key)return;const he=no(de.bytes)??V;if(N&&he===V){ce(de,N);return}if(!he){G();return}ie(),V=he;const pe=n,Be={codec:he,optimizeForLatency:!0,hardwareAcceleration:"prefer-hardware"};VideoDecoder.isConfigSupported(Be).then(xe=>{if(F||pe!==n)return;if(!xe.supported){j(`H.264 codec ${he} is not supported by this browser`);return}const _e=new VideoDecoder({output(ge){if(F||N!==_e){ge.close();return}try{Rr(g,ge,ge.displayWidth,ge.displayHeight)&&!q&&(q=!0,g.onState("playing"))}catch(Oe){ie(!0),j(`Device video draw failed: ${Oe instanceof Error?Oe.message:String(Oe)}`)}finally{ge.close()}},error(ge){F||N!==_e||(ie(!0),j(`H.264 decoder failed: ${ge.message}`))}});try{_e.configure(xe.config??Be)}catch(ge){_e.close(),j(`H.264 decoder configuration failed: ${ge instanceof Error?ge.message:String(ge)}`);return}if(F||pe!==n){_e.close();return}N=_e,ce(de,_e)},xe=>j(`H.264 decoder support check failed: ${xe instanceof Error?xe.message:String(xe)}`))}),E.addEventListener("error",()=>j("Device video connection failed")),E.addEventListener("close",()=>{ie(),F||g.onState("paused")}),()=>{F=!0,E.close(),ie()}}function ao(g){const T=new AbortController;let E,N,V=!1,n=!1;return g.onState("connecting"),fetch(new URL("/video",new URL(g.stream.baseUrl).origin),{method:"GET",headers:{Authorization:`Bearer ${g.stream.token}`},signal:T.signal}).then(async re=>{if(!re.ok||!re.body)throw new Error(`Device video returned ${re.status}`);const F=io(re.headers.get("content-type"));if(!F)throw new Error("Device video returned an invalid MJPEG content type");E=re.body.getReader();let q=new Uint8Array;for(;!V;){const G=await E.read();if(G.done)break;q=lo(q,G.value);const ie=so(q,F);q=ie.rest;for(const j of ie.frames){const ce=await createImageBitmap(new Blob([new Uint8Array(j).buffer],{type:"image/jpeg"}));N=ce;try{if(V)continue;Rr(g,ce,ce.width,ce.height)&&!n&&(n=!0,g.onState("playing"))}finally{ce.close(),N===ce&&(N=void 0)}}if(q.byteLength>Ur)throw new Error("Device MJPEG stream exceeded 16 MiB without a complete frame")}V||g.onState("paused")}).catch(re=>{V||(g.onState("paused"),g.onError(re instanceof Error?re.message:String(re)))}).finally(()=>{E?.releaseLock(),E=void 0}),()=>{V=!0,T.abort(),E?.cancel().catch(()=>{}),N?.close(),N=void 0}}function Rr(g,T,E,N){if(!Number.isSafeInteger(E)||!Number.isSafeInteger(N)||E<=0||N<=0)return!1;(g.canvas.width!==E||g.canvas.height!==N)&&(g.canvas.width=E,g.canvas.height=N);const V=g.canvas.getContext("2d",{alpha:!1});return V?(V.drawImage(T,0,0,E,N),!0):!1}function co(g){try{const T=JSON.parse(g);return!T||typeof T!="object"||!("type"in T)||T.type!=="video-session"||!("size"in T)||!T.size||typeof T.size!="object"||!("width"in T.size)||!("height"in T.size)||typeof T.size.width!="number"||typeof T.size.height!="number"||!Number.isSafeInteger(T.size.width)||!Number.isSafeInteger(T.size.height)||T.size.width<=0||T.size.height<=0?void 0:{width:T.size.width,height:T.size.height}}catch{return}}function jr(g){const T=[];for(let E=0;Eg.subarray(E.nal,T[N+1]?.code??g.byteLength))}function Pr(g){return g.byteLength>=4&&g[0]===0&&g[1]===0&&g[2]===1?!0:g.byteLength>=5&&g[0]===0&&g[1]===0&&g[2]===0&&g[3]===1}function Ct(g,T){return jr(g).some(E=>(E[0]&31)===T)}function lo(g,T){const E=new Uint8Array(g.byteLength+T.byteLength);return E.set(g),E.set(T,g.byteLength),E}function Br(g,T,E){for(let N=E;N<=g.byteLength-T.byteLength;N++){let V=!0;for(let n=0;n"),po=Z.template("