diff --git a/README.md b/README.md index ac51e2a..e8a2662 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ Add this entry to your OpenCode configuration. ```json { - "plugins": ["github:viralcodex/opencode-demul#v0.2.0"] + "plugins": ["github:viralcodex/opencode-demul#v0.3.0"] } ``` @@ -34,9 +34,17 @@ Android SDK tools, Node and ffmpeg must already be installed. ## Use -The panel opens the first running device, or else the first listed one, and starts it if -needed. Use the picker to switch between Android emulators and iOS simulators and refresh to -rescan. Either platform's toolchain may be missing; the picker lists what the host can run. +The panel opens the first available running device, or else the first available listed one, +and starts it if needed. Use the platform and device pickers to switch between Android +emulators and iOS simulators, and refresh to rescan. Either platform's toolchain may be +missing; the picker lists what the host can run. + +Each session controls its own device. Run iOS and Android side by side in separate sessions; +a device already controlled by another session cannot be attached. Tabs and windows showing +the same session share its device and agent-access setting. Closing a viewer stops its video +without detaching the device, so agents can keep working. Detach the device in the panel to +release it; deleting or moving the session also releases it. A failed device switch keeps the +previous attachment. Agent access starts enabled. Use the switch to revoke access for the current session. Agents can only use a device you have opened. diff --git a/dist/gui.cjs b/dist/gui.cjs index 74c0c49..3b10df1 100644 --- a/dist/gui.cjs +++ b/dist/gui.cjs @@ -1,24 +1,24 @@ -"use strict";Object.defineProperties(exports,{__esModule:{value:!0},[Symbol.toStringTag]:{value:"Module"}});const J=require("solid-js/web"),ce=require("solid-js"),mt=require("@opencode/plugin/gui"),ys=require("@opencode/schema/rpc"),u=require("effect"),ws=8*1024*1024,Fe=u.Schema.optional,ye=u.Schema.String.check(u.Schema.isMaxLength(2048)),Ur=u.Schema.String.check(u.Schema.isMaxLength(1e5)),pt=u.Schema.Int.check(u.Schema.isGreaterThanOrEqualTo(0)),Rr=u.Schema.Int.check(u.Schema.isBetween({minimum:0,maximum:1e6})),He=u.Schema.Literals(["ios","android"]).annotate({identifier:"Device.Platform"}),bs=u.Schema.Literals(["shutdown","booting","booted","unavailable"]).annotate({identifier:"Device.DeviceState"}),_s=u.Schema.Literals(["available","owned-by-this-host","owned-by-other","unknown"]).annotate({identifier:"Device.Ownership"}),Ar=u.Schema.Literals(["apps","launch","install","snapshot","press","fill","swipe","key","screenshot","stream"]).annotate({identifier:"Device.Capability"}),ks=u.Schema.Literals(["home","back","recents","lock","volumeUp","volumeDown"]).annotate({identifier:"Device.NavigationKey"}),Sn=pt.annotate({identifier:"Device.Generation"}),vn=u.Schema.String.check(u.Schema.isPattern(/^dh_[a-f0-9-]{36}$/u)).pipe(u.Schema.brand("Device.HandleToken")).annotate({identifier:"Device.HandleToken"}),Ee=u.Schema.Struct({token:vn,platform:He,deviceId:ye,generation:Sn}).annotate({identifier:"Device.DeviceHandle"}),Gr=u.Schema.String.check(u.Schema.isPattern(/^@?e[1-9][0-9]*$/u)).pipe(u.Schema.brand("Device.ElementRef")).annotate({identifier:"Device.ElementRef"}),$r=u.Schema.Struct({token:vn,generation:Sn,revision:pt,ref:Gr}).annotate({identifier:"Device.SnapshotReference"}),xs=u.Schema.Struct({ref:Fe(Gr),role:ye,name:Fe(Ur),value:Fe(Ur),enabled:Fe(u.Schema.Boolean),focused:Fe(u.Schema.Boolean)}).annotate({identifier:"Device.AccessibilityNode"}),Es=u.Schema.Struct({token:vn,generation:Sn,revision:pt,interactiveOnly:u.Schema.Boolean,complete:u.Schema.Boolean,nodes:u.Schema.Array(xs)}).annotate({identifier:"Device.Snapshot"}),fn=u.Schema.Struct({x:Rr,y:Rr}).annotate({identifier:"Device.LogicalPoint"}),Ls=u.Schema.Struct({id:ye,name:ye,platform:He,osVersion:Fe(ye),state:bs,ownership:_s}).annotate({identifier:"Device.DeviceInfo"}),Ds=u.Schema.Struct({supported:u.Schema.Array(Ar),unsupported:u.Schema.Array(u.Schema.Struct({capability:Ar,reason:ye}))}).annotate({identifier:"Device.Capabilities"}),Os=u.Schema.Struct({id:ye,name:Fe(ye)}).annotate({identifier:"Device.AppInfo"}),Ts=u.Schema.Struct({app:ye,state:u.Schema.Literals(["launched","foreground","unknown"])}).annotate({identifier:"Device.LaunchResult"}),Is=u.Schema.Struct({installed:u.Schema.Boolean,app:Fe(ye)}).annotate({identifier:"Device.InstallResult"}),Cs=u.Schema.Struct({data:u.Schema.Uint8ArrayFromBase64.check(u.Schema.isMaxLength(ws)),width:pt,height:pt,deviceId:ye,platform:He}).annotate({identifier:"Device.Screenshot"}),Ps=u.Schema.Literals(["mjpeg","h264"]).annotate({identifier:"Device.StreamProtocol"}),Bs=u.Schema.Struct({platform:He,deviceId:ye,baseUrl:ye,token:ye,protocol:Ps,loopback:u.Schema.Literal(!0)}).annotate({identifier:"Device.StreamDescriptor"}),Ms=u.Schema.Literals(["missing-prerequisite","permission-denied","device-busy","detached","stale-reference","unsupported","not-found","invalid-artifact","timeout","helper-failed","internal"]).annotate({identifier:"Device.DeviceErrorCode"}),gn=u.Schema.Struct({code:Ms,message:ye,mayHaveExecuted:u.Schema.Boolean}).annotate({identifier:"Device.DeviceError"}),et=u.Schema.optional,ze=u.Schema.String.check(u.Schema.isMaxLength(2048)),qr=ze.check(u.Schema.isMinLength(1)),pn={sessionID:qr,connectionID:u.Schema.String},jr={...pn,requestID:u.Schema.String},Wr=u.Schema.Number.check(u.Schema.isInt(),u.Schema.isBetween({minimum:1,maximum:1e6})),Fr=u.Schema.optional(u.Schema.Struct({width:Wr,height:Wr})),tt=u.Schema.Struct({sessionID:qr,serverID:ze,location:u.Schema.optionalKey(u.Schema.Struct({directory:ze}))}),Pt=u.Schema.Struct({handle:Ee,capabilities:Ds,agentAccess:u.Schema.Boolean}).annotate({identifier:"DeviceBridge.Attachment"}),Hr=u.Schema.Union([u.Schema.Struct({type:u.Schema.Literal("list"),platform:et(He)}),u.Schema.Struct({type:u.Schema.Literal("state")}),u.Schema.Struct({type:u.Schema.Literal("open"),platform:He,deviceId:ze,boot:et(u.Schema.Boolean)}),u.Schema.Struct({type:u.Schema.Literal("apps"),handle:Ee}),u.Schema.Struct({type:u.Schema.Literal("launch"),handle:Ee,app:ze,deepLink:et(ze),relaunch:et(u.Schema.Boolean)}),u.Schema.Struct({type:u.Schema.Literal("install"),handle:Ee,artifactPath:ze}),u.Schema.Struct({type:u.Schema.Literal("snapshot"),handle:Ee,interactiveOnly:et(u.Schema.Boolean)}),u.Schema.Struct({type:u.Schema.Literal("press"),handle:Ee,reference:$r}),u.Schema.Struct({type:u.Schema.Literal("tap"),handle:Ee,point:fn,viewport:Fr}),u.Schema.Struct({type:u.Schema.Literal("fill"),handle:Ee,reference:$r,text:ze}),u.Schema.Struct({type:u.Schema.Literal("swipe"),handle:Ee,from:fn,to:fn,viewport:Fr}),u.Schema.Struct({type:u.Schema.Literal("key"),handle:Ee,key:ks}),u.Schema.Struct({type:u.Schema.Literal("screenshot"),handle:Ee}),u.Schema.Struct({type:u.Schema.Literal("close"),handle:Ee,shutdown:et(u.Schema.Boolean)})]).pipe(u.Schema.toTaggedUnion("type")).annotate({identifier:"DeviceBridge.Operation"}),Ns=u.Schema.Union([u.Schema.Struct({type:u.Schema.Literal("devices"),devices:u.Schema.Array(Ls)}),u.Schema.Struct({type:u.Schema.Literal("state"),state:u.Schema.NullOr(Pt)}),u.Schema.Struct({type:u.Schema.Literal("attachment"),attachment:Pt}),u.Schema.Struct({type:u.Schema.Literal("apps"),apps:u.Schema.Array(Os)}),u.Schema.Struct({type:u.Schema.Literal("launch"),result:Ts}),u.Schema.Struct({type:u.Schema.Literal("install"),result:Is}),u.Schema.Struct({type:u.Schema.Literal("snapshot"),snapshot:Es}),u.Schema.Struct({type:u.Schema.Literal("screenshot"),screenshot:Cs}),u.Schema.Struct({type:u.Schema.Literal("done")})]).pipe(u.Schema.toTaggedUnion("type")).annotate({identifier:"DeviceBridge.Value"}),Ct=u.Schema.Union([u.Schema.Struct({type:u.Schema.Literal("success"),value:Ns}),u.Schema.Struct({type:u.Schema.Literal("failure"),error:gn})]).pipe(u.Schema.toTaggedUnion("type")).annotate({identifier:"DeviceBridge.Outcome"}),Us=u.Schema.Union([u.Schema.Struct({type:u.Schema.Literal("success"),stream:Bs}),u.Schema.Struct({type:u.Schema.Literal("failure"),error:gn})]).pipe(u.Schema.toTaggedUnion("type")).annotate({identifier:"DeviceBridge.StreamOutcome"}),Rs=u.Schema.Union([u.Schema.Struct({type:u.Schema.Literal("attached"),connectionID:u.Schema.String,version:u.Schema.Literal(1)}),u.Schema.Struct({type:u.Schema.Literal("command"),connectionID:u.Schema.String,requestID:u.Schema.String}),u.Schema.Struct({type:u.Schema.Literal("cancel"),connectionID:u.Schema.String,requestID:u.Schema.String})]).pipe(u.Schema.toTaggedUnion("type")).annotate({identifier:"DeviceBridge.Control"}),Pe=u.Schema.toStandardSchemaV1,Tt={unavailable:Pe(gn)};ys.Rpc.define({id:"device.control",methods:{attach:{input:Pe(u.Schema.Struct({...pn,version:u.Schema.Literal(1)})),output:Pe(u.Schema.Literals(["closed","replaced"])),errors:Tt},state:{input:Pe(u.Schema.Struct({...pn,state:u.Schema.NullOr(Pt)})),output:Pe(u.Schema.Void),errors:Tt},command:{input:Pe(u.Schema.Struct(jr)),output:Pe(Hr),errors:Tt},result:{input:Pe(u.Schema.Struct({...jr,outcome:Ct})),output:Pe(u.Schema.Void),errors:Tt}},events:{control:{schema:Pe(Rs)}}});const As=u.Schema.String.check(u.Schema.isMaxLength(2048)),$s=mt.Ipc.define({id:"demul.device.bridge.v2",state:u.Schema.Array(u.Schema.Struct({scope:tt,state:u.Schema.NullOr(Pt)})),methods:{connect:{input:tt},disconnect:{input:tt},list:{input:u.Schema.Struct({platform:u.Schema.optional(He),serverID:As}),output:Ct},access:{input:u.Schema.Struct({scope:tt,enabled:u.Schema.Boolean}),output:Ct},manual:{input:u.Schema.Struct({scope:tt,operation:Hr}),output:Ct},stream:{input:u.Schema.Struct({scope:tt,handle:Ee}),output:Us}}});function js(g,O,x,M){if(!(!g||g!==O||x!=="playing"||!Number.isSafeInteger(M.width)||!Number.isSafeInteger(M.height)||M.width<=0||M.height<=0))return{token:g,width:M.width,height:M.height}}function Ws(g,O){return!!O&&g.token===O.token&&g.width===O.width&&g.height===O.height}const Fs=1397050709,Jr=16*1024*1024;function zs(g){let O=!1,x;const M=()=>{const n=O&&document.visibilityState==="visible";if(n&&!x){x=g.stream.protocol==="h264"?Xs(g):Ks(g);return}n||!x||(x(),x=void 0,g.onState(O?"paused":"idle"))},V=()=>M();return document.addEventListener("visibilitychange",V),{active(n){O=n,M()},destroy(){O=!1,document.removeEventListener("visibilitychange",V),x?.(),x=void 0,g.canvas.width=0,g.canvas.height=0,g.onState("idle")}}}function Vs(g){const O=new Uint8Array(g),x=new DataView(g);if(O.byteLength<4)return;if(x.getUint32(0)!==Fs)return zr(O)?{bytes:O,key:It(O,5)||It(O,7),timestamp:performance.now()*1e3}:void 0;if(O.byteLength<16)return;const M=x.getUint8(4),V=M===1?16:M===2?24:0;if(!V||O.byteLength<=V)return;const n=x.getBigUint64(8);if(n>BigInt(Number.MAX_SAFE_INTEGER))return;const re=O.subarray(V);if(zr(re))return{bytes:re,key:(x.getUint8(5)&1)===1||It(re,5)||It(re,7),timestamp:Number(n),serverTimestamp:M===2?Number(x.getBigUint64(16))/1e3:void 0}}function Gs(g){const O=Kr(g).find(x=>(x[0]&31)===7);if(!(!O||O.byteLength<4))return`avc1.${[O[1],O[2],O[3]].map(x=>x.toString(16).padStart(2,"0")).join("")}`}function qs(g,O,x){if(![g.left,g.top,g.width,g.height,O.width,O.height,x.x,x.y].every(Number.isFinite)||g.width<=0||g.height<=0||!Number.isSafeInteger(O.width)||!Number.isSafeInteger(O.height)||O.width<=0||O.height<=0)return;const M=Math.min(g.width/O.width,g.height/O.height),V=O.width*M,n=O.height*M,re=x.x-g.left-(g.width-V)/2,W=x.y-g.top-(g.height-n)/2;if(!(re<0||W<0||re>=V||W>=n))return{x:Math.min(O.width-1,Math.floor(re/M)),y:Math.min(O.height-1,Math.floor(W/M))}}function Hs(g){if(!g||/[\r\n]/.test(g))return;const O=g.split(";");if(O.shift()?.trim().toLowerCase()!=="multipart/x-mixed-replace")return;const x=O.map(n=>n.trim()).find(n=>n.toLowerCase().startsWith("boundary="))?.slice(9).trim();if(!x)return;const M=x.startsWith('"')&&x.endsWith('"')?x.slice(1,-1):x,V=M.startsWith("--")?M.slice(2):M;if(!(!V||/[\r\n]/.test(V)))return V}function Js(g,O){const x=new TextEncoder().encode(`--${O}`),M=new Uint8Array([13,10,13,10]),V=[];let n=0,re=0;for(;ng.byteLength)return{frames:V,rest:g.slice(W)};if(g[H]!==13||g[H+1]!==10){n=H;continue}const q=Vr(g,M,H+2);if(q<0)return{frames:V,rest:g.slice(W)};const ie=new TextDecoder("ascii").decode(g.subarray(H+2,q)),A=ie.match(/(?:^|\r\n)Content-Length:\s*(\d+)(?:\r\n|$)/i)?.[1],de=ie.match(/(?:^|\r\n)Content-Type:\s*([^\r\n]+)(?:\r\n|$)/i)?.[1].trim().toLowerCase(),oe=A?Number(A):0,le=q+M.byteLength,he=le+oe;if(!Number.isSafeInteger(oe)||oe<=0||oe>Jr||de!=="image/jpeg"){n=q+M.byteLength;continue}if(he>g.byteLength)return{frames:V,rest:g.slice(W)};const fe=g.slice(le,he);fe.byteLength>=4&&fe[0]===255&&fe[1]===216&&fe.at(-2)===255&&fe.at(-1)===217&&V.push(fe),re=he,n=he}return{frames:V,rest:g.slice(re)}}function Xs(g){if(typeof VideoDecoder!="function"||typeof EncodedVideoChunk!="function")return g.onState("paused"),g.onError("H.264 device video requires WebCodecs support in this browser"),()=>{};const O=new URL("/video?frame-meta=1",new URL(g.stream.baseUrl).origin);O.protocol=O.protocol==="https:"?"wss:":"ws:";const x=new WebSocket(O,`opencode-device.${g.stream.token}`);let M,V,n=0,re=!0,W=!1,H=!1;x.binaryType="arraybuffer",g.onState("connecting");const q=()=>{if(x.readyState===WebSocket.OPEN)try{x.send(JSON.stringify({type:"reset-video",ack:!1}))}catch{}},ie=(oe=!1)=>{n++;const le=M,he=H;if(M=void 0,V=void 0,re=!0,H=!1,le&&le.state!=="closed")try{le.reset()}finally{le.close()}oe&&he&&!W&&g.onState("connecting"),oe&&q()},A=oe=>{W||(g.onError(oe),g.onState("paused"))},de=(oe,le)=>{if(!(W||M!==le||le.state!=="configured"))try{le.decode(new EncodedVideoChunk({type:oe.key?"key":"delta",timestamp:oe.timestamp,data:oe.bytes})),re=!1}catch(he){ie(!0),A(`H.264 decode failed: ${he instanceof Error?he.message:String(he)}`)}};return x.addEventListener("open",()=>q()),x.addEventListener("message",oe=>{if(W)return;if(typeof oe.data=="string"){Qs(oe.data)&&ie(!0);return}if(!(oe.data instanceof ArrayBuffer))return;const le=Vs(oe.data);if(!le)return;if(M&&M.decodeQueueSize>2){ie(!0);return}if(re&&!le.key)return;const he=Gs(le.bytes)??V;if(M&&he===V){de(le,M);return}if(!he){q();return}ie(),V=he;const fe=n,Be={codec:he,optimizeForLatency:!0,hardwareAcceleration:"prefer-hardware"};VideoDecoder.isConfigSupported(Be).then(De=>{if(W||fe!==n)return;if(!De.supported){A(`H.264 codec ${he} is not supported by this browser`);return}const xe=new VideoDecoder({output(me){if(W||M!==xe){me.close();return}try{Xr(g,me,me.displayWidth,me.displayHeight)&&!H&&(H=!0,g.onState("playing"))}catch(Oe){ie(!0),A(`Device video draw failed: ${Oe instanceof Error?Oe.message:String(Oe)}`)}finally{me.close()}},error(me){W||M!==xe||(ie(!0),A(`H.264 decoder failed: ${me.message}`))}});try{xe.configure(De.config??Be)}catch(me){xe.close(),A(`H.264 decoder configuration failed: ${me instanceof Error?me.message:String(me)}`);return}if(W||fe!==n){xe.close();return}M=xe,de(le,xe)},De=>A(`H.264 decoder support check failed: ${De instanceof Error?De.message:String(De)}`))}),x.addEventListener("error",()=>A("Device video connection failed")),x.addEventListener("close",()=>{ie(),W||g.onState("paused")}),()=>{W=!0,x.close(),ie()}}function Ks(g){const O=new AbortController;let x,M,V=!1,n=!1;return g.onState("connecting"),fetch(new URL("/video",new URL(g.stream.baseUrl).origin),{method:"GET",headers:{Authorization:`Bearer ${g.stream.token}`},signal:O.signal}).then(async re=>{if(!re.ok||!re.body)throw new Error(`Device video returned ${re.status}`);const W=Hs(re.headers.get("content-type"));if(!W)throw new Error("Device video returned an invalid MJPEG content type");x=re.body.getReader();let H=new Uint8Array;for(;!V;){const q=await x.read();if(q.done)break;H=Ys(H,q.value);const ie=Js(H,W);H=ie.rest;for(const A of ie.frames){const de=await createImageBitmap(new Blob([new Uint8Array(A).buffer],{type:"image/jpeg"}));M=de;try{if(V)continue;Xr(g,de,de.width,de.height)&&!n&&(n=!0,g.onState("playing"))}finally{de.close(),M===de&&(M=void 0)}}if(H.byteLength>Jr)throw new Error("Device MJPEG stream exceeded 16 MiB without a complete frame")}V||g.onState("paused")}).catch(re=>{V||(g.onState("paused"),g.onError(re instanceof Error?re.message:String(re)))}).finally(()=>{x?.releaseLock(),x=void 0}),()=>{V=!0,O.abort(),x?.cancel().catch(()=>{}),M?.close(),M=void 0}}function Xr(g,O,x,M){if(!Number.isSafeInteger(x)||!Number.isSafeInteger(M)||x<=0||M<=0)return!1;(g.canvas.width!==x||g.canvas.height!==M)&&(g.canvas.width=x,g.canvas.height=M);const V=g.canvas.getContext("2d",{alpha:!1});return V?(V.drawImage(O,0,0,x,M),!0):!1}function Qs(g){try{const O=JSON.parse(g);return!O||typeof O!="object"||!("type"in O)||O.type!=="video-session"||!("size"in O)||!O.size||typeof O.size!="object"||!("width"in O.size)||!("height"in O.size)||typeof O.size.width!="number"||typeof O.size.height!="number"||!Number.isSafeInteger(O.size.width)||!Number.isSafeInteger(O.size.height)||O.size.width<=0||O.size.height<=0?void 0:{width:O.size.width,height:O.size.height}}catch{return}}function Kr(g){const O=[];for(let x=0;xg.subarray(x.nal,O[M+1]?.code??g.byteLength))}function zr(g){return g.byteLength>=4&&g[0]===0&&g[1]===0&&g[2]===1?!0:g.byteLength>=5&&g[0]===0&&g[1]===0&&g[2]===0&&g[3]===1}function It(g,O){return Kr(g).some(x=>(x[0]&31)===O)}function Ys(g,O){const x=new Uint8Array(g.byteLength+O.byteLength);return x.set(g),x.set(O,g.byteLength),x}function Vr(g,O,x){for(let M=x;M<=g.byteLength-O.byteLength;M++){let V=!0;for(let n=0;n"),ta=J.template("