Unify device video on authenticated H.264

README.md

@@ -29,7 +29,8 @@ Add this entry to your OpenCode configuration.
 
 Open **Add tab > Device** in a session.
 First use downloads the pinned device helpers from npm.
-Android SDK tools and Node must already be installed. Android video uses the device's H.264 encoder through scrcpy.
+Android SDK tools and Node must already be installed. Both platforms stream H.264 through a shared viewer.
+Android uses the device's encoder through scrcpy; iOS uses the simulator helper's native encoder.
 
 ## Use
 

dist/server.js

@@ -183,7 +183,7 @@ var Screenshot = Schema.Struct({
   deviceId: short,
   platform: Platform
 }).annotate({ identifier: "Device.Screenshot" });
-var StreamProtocol = Schema.Literals(["mjpeg", "h264"]).annotate({
+var StreamProtocol = Schema.Literal("h264").annotate({
   identifier: "Device.StreamProtocol"
 });
 var StreamDescriptor = Schema.Struct({

src/avcc.test.ts

@@ -0,0 +1,126 @@
+import { describe, expect, test } from "bun:test";
+import { AvccStream } from "./avcc.js";
+import { parseFramePacket } from "./h264.js";
+import {
+  avccDescription as description,
+  avccRecord as record,
+  avccSample as sample,
+  sps as annexSps,
+  pps as annexPps,
+  idr as annexIdr,
+  delta as annexDelta,
+  startCode as start,
+} from "../test/video.js";
+
+const sps = annexSps.subarray(4);
+const pps = annexPps.subarray(3);
+const idr = annexIdr.subarray(4);
+const delta = annexDelta.subarray(3);
+
+describe("iOS AVCC normalization", () => {
+  test.each([1, 2, 4])(
+    "converts %i-byte NAL lengths and preserves keyframe parameter sets",
+    (lengthBytes) => {
+      const stream = new AvccStream();
+      const frames = stream.push(
+        Buffer.concat([
+          record(4, Buffer.from([0xff, 0xd8, 0xff, 0xd9])),
+          description(lengthBytes),
+          sample(2, [idr], lengthBytes),
+          sample(3, [delta, delta], lengthBytes),
+        ]),
+      );
+      expect(frames).toEqual([
+        { key: true, bytes: Buffer.concat([start, sps, start, pps, start, idr]) },
+        { key: false, bytes: Buffer.concat([start, delta, start, delta]) },
+      ]);
+      expect(parseFramePacket(frames[0].bytes)).toMatchObject({ key: true, codec: "avc1.640029" });
+      stream.finish();
+    },
+  );
+
+  test("handles every split point and byte-at-a-time delivery", () => {
+    const bytes = Buffer.concat([description(), sample(2, [idr]), sample(3, [delta])]);
+    const expected = new AvccStream().push(bytes);
+    for (let offset = 0; offset <= bytes.length; offset++) {
+      const stream = new AvccStream();
+      expect([
+        ...stream.push(bytes.subarray(0, offset)),
+        ...stream.push(bytes.subarray(offset)),
+      ]).toEqual(expected);
+      stream.finish();
+    }
+    const stream = new AvccStream();
+    expect([...bytes].flatMap((byte) => stream.push(Buffer.from([byte])))).toEqual(expected);
+    stream.finish();
+  });
+
+  test("waits for configuration and a keyframe, including after configuration changes", () => {
+    const stream = new AvccStream();
+    expect(stream.push(sample(2, [idr]))).toEqual([]);
+    expect(stream.push(Buffer.concat([description(), sample(3, [delta])]))).toEqual([]);
+    expect(stream.push(sample(2, [idr]))).toHaveLength(1);
+    expect(stream.push(Buffer.concat([description(2), sample(3, [delta], 2)]))).toEqual([]);
+    expect(stream.push(sample(2, [idr], 2))).toHaveLength(1);
+  });
+
+  test("does not mutate frames after processing more records", () => {
+    const stream = new AvccStream();
+    const frame = stream.push(Buffer.concat([description(), sample(2, [idr])]))[0];
+    const original = Buffer.from(frame.bytes);
+    stream.push(sample(3, [Buffer.alloc(100_000, 0x41)]));
+    expect(frame.bytes).toEqual(original);
+  });
+
+  test.each([0, 8 * 1024 * 1024 + 1, 0xffffffff])(
+    "rejects record size %i before allocating its payload",
+    (size) => {
+      const header = Buffer.alloc(4);
+      header.writeUInt32BE(size);
+      expect(() => new AvccStream().push(header)).toThrow("record size");
+    },
+  );
+
+  test.each([
+    Buffer.alloc(0),
+    Buffer.from([1, 0x64, 0, 0x29, 0xfe, 0xe1, 0]),
+    Buffer.from([1, 0x64, 0, 0x29, 0xff, 0xe1, 0, 255]),
+    Buffer.from([1, 0x64, 0, 0x29, 0xff, 0xe0, 0]),
+  ])("rejects malformed decoder descriptions", (data) => {
+    expect(() => new AvccStream().push(record(1, data))).toThrow("iOS H.264");
+  });
+
+  test.each([
+    Buffer.alloc(0),
+    Buffer.from([0, 0]),
+    Buffer.from([0, 0, 0, 0]),
+    Buffer.from([0, 0, 0, 5, 0x65]),
+  ])("rejects malformed sample NAL units", (data) => {
+    const stream = new AvccStream();
+    stream.push(description());
+    expect(() => stream.push(record(2, data))).toThrow("iOS H.264");
+  });
+
+  test("bounds the number of NAL units and rejects unknown tags", () => {
+    const stream = new AvccStream();
+    stream.push(description());
+    expect(() =>
+      stream.push(
+        sample(
+          2,
+          Array.from({ length: 4_097 }, () => idr),
+        ),
+      ),
+    ).toThrow("Too many");
+    expect(() => new AvccStream().push(record(9, Buffer.alloc(0)))).toThrow("record type");
+  });
+
+  test.each([Buffer.from([0, 0]), Buffer.from([0, 0, 0, 5, 1])])(
+    "detects truncated records at EOF",
+    (data) => {
+      const stream = new AvccStream();
+      stream.push(data);
+      expect(() => stream.finish()).toThrow("Truncated");
+    },
+  );
+});

src/avcc.ts

@@ -0,0 +1,122 @@
+// serve-sim sends length-prefixed records: description=1, keyframe=2, delta=3, JPEG seed=4.
+// Convert AVCC NAL lengths to Annex-B start codes without decoding or re-encoding video.
+const MAX_RECORD_BYTES = 8 * 1024 * 1024;
+const MAX_NALS = 4_096;
+const START_CODE = Buffer.from([0, 0, 0, 1]);
+
+export interface AvccFrame {
+  readonly bytes: Buffer;
+  readonly key: boolean;
+}
+
+interface Configuration {
+  readonly lengthBytes: number;
+  readonly parameters: Buffer;
+}
+
+export class AvccStream {
+  private readonly header = Buffer.alloc(4);
+  private headerBytes = 0;
+  private record: Buffer | undefined;
+  private recordBytes = 0;
+  private configuration: Configuration | undefined;
+  private waitingKeyframe = true;
+
+  push(chunk: Uint8Array): AvccFrame[] {
+    const frames: AvccFrame[] = [];
+    let offset = 0;
+    while (offset < chunk.byteLength) {
+      if (!this.record) {
+        offset += this.readHeader(chunk.subarray(offset));
+        if (!this.record) continue;
+      }
+      const count = Math.min(this.record.length - this.recordBytes, chunk.byteLength - offset);
+      this.record.set(chunk.subarray(offset, offset + count), this.recordBytes);
+      this.recordBytes += count;
+      offset += count;
+      if (this.recordBytes !== this.record.length) continue;
+      const record = this.record;
+      this.record = undefined;
+      this.recordBytes = 0;
+      const frame = this.convert(record);
+      if (frame) frames.push(frame);
+    }
+    return frames;
+  }
+
+  finish(): void {
+    if (this.headerBytes || this.record) throw new Error("Truncated iOS H.264 record");
+  }
+
+  private readHeader(chunk: Uint8Array): number {
+    const count = Math.min(4 - this.headerBytes, chunk.byteLength);
+    this.header.set(chunk.subarray(0, count), this.headerBytes);
+    this.headerBytes += count;
+    if (this.headerBytes !== 4) return count;
+    const length = this.header.readUInt32BE(0);
+    if (length < 1 || length > MAX_RECORD_BYTES) throw new Error("Invalid iOS H.264 record size");
+    this.record = Buffer.allocUnsafe(length);
+    this.headerBytes = 0;
+    return count;
+  }
+
+  private convert(record: Buffer): AvccFrame | undefined {
+    const tag = record[0];
+    const payload = record.subarray(1);
+    if (tag === 4) return;
+    if (tag === 1) {
+      this.configuration = parseConfiguration(payload);
+      this.waitingKeyframe = true;
+      return;
+    }
+    if (tag !== 2 && tag !== 3) throw new Error("Invalid iOS H.264 record type");
+    if (!this.configuration || (this.waitingKeyframe && tag !== 2)) return;
+    const key = tag === 2;
+    const bytes = annexB(payload, this.configuration, key);
+    this.waitingKeyframe = false;
+    return { bytes, key };
+  }
+}
+
+function parseConfiguration(data: Buffer): Configuration {
+  if (data.length < 7 || data[0] !== 1) throw new Error("Invalid iOS H.264 configuration");
+  const lengthBytes = (data[4] & 3) + 1;
+  if (lengthBytes === 3) throw new Error("Invalid iOS H.264 NAL length size");
+  const sps = parameterSets(data, 6, data[5] & 31, 7);
+  const pps = parameterSets(data, sps.offset + 1, data[sps.offset] ?? 0, 8);
+  if (!sps.parts.length || !pps.parts.length) throw new Error("Missing iOS H.264 parameter sets");
+  return { lengthBytes, parameters: Buffer.concat([...sps.parts, ...pps.parts]) };
+}
+
+function parameterSets(data: Buffer, start: number, count: number, type: number) {
+  const parts: Buffer[] = [];
+  let offset = start;
+  for (let index = 0; index < count; index++) {
+    if (offset + 2 > data.length) throw new Error("Truncated iOS H.264 parameter set");
+    const length = data.readUInt16BE(offset);
+    offset += 2;
+    if (!length || offset + length > data.length || (data[offset] & 31) !== type)
+      throw new Error("Invalid iOS H.264 parameter set");
+    parts.push(START_CODE, data.subarray(offset, offset + length));
+    offset += length;
+  }
+  return { parts, offset };
+}
+
+function annexB(data: Buffer, configuration: Configuration, key: boolean): Buffer {
+  const parts: Buffer[] = key ? [configuration.parameters] : [];
+  let offset = 0;
+  let count = 0;
+  while (offset < data.length) {
+    if (++count > MAX_NALS) throw new Error("Too many iOS H.264 NAL units");
+    if (offset + configuration.lengthBytes > data.length)
+      throw new Error("Truncated iOS H.264 NAL length");
+    const length = data.readUIntBE(offset, configuration.lengthBytes);
+    offset += configuration.lengthBytes;
+    if (!length || offset + length > data.length) throw new Error("Invalid iOS H.264 NAL size");
+    parts.push(START_CODE, data.subarray(offset, offset + length));
+    offset += length;
+  }
+  if (!data.length) throw new Error("Empty iOS H.264 frame");
+  return Buffer.concat(parts);
+}

src/controller.test.ts

@@ -84,7 +84,7 @@ function fixture() {
         deviceId: handle.deviceId,
         baseUrl: "http://127.0.0.1:3400",
         token: "private",
-        protocol: "mjpeg",
+        protocol: "h264",
         loopback: true,
       };
     },