perf(player): reduce MPEG-TS playback CPU overhead

web-ui/src/components/player/video-player.tsx

@@ -1995,9 +1995,11 @@ function VideoPlayerComponent({
           className={clsx(
             "player-performance-controls-position player-performance-motion absolute bottom-0 left-[calc(0px_-_env(safe-area-inset-left))] right-[calc(0px_-_env(safe-area-inset-right))] z-10 transition-opacity duration-300",
             showSidebar && "md:right-0",
+            // Invisible pulse animations still wake the compositor. Restore the
+            // live indicator animation when pointer or keyboard controls appear.
             showControls
               ? "opacity-100"
-              : "opacity-0 pointer-events-none has-focus-visible:opacity-100 has-focus-visible:pointer-events-auto",
+              : "opacity-0 pointer-events-none has-focus-visible:opacity-100 has-focus-visible:pointer-events-auto [&_.animate-pulse]:animate-none has-focus-visible:[&_.animate-pulse]:animate-pulse",
           )}
         >
           <PlayerControls

web-ui/src/playback-engine/audio/wasm-stretcher.ts

@@ -19,7 +19,8 @@ export interface Stretcher {
   /** Input frames consumed for emitted output since reset (fractional). */
   readonly position: number;
   setRatio(ratio: number): void;
-  /** Feed interleaved PCM, get stretched interleaved PCM (may be empty). */
+  /** Feed interleaved PCM, get stretched interleaved PCM (may be empty).
+   *  The returned view is borrowed; consume it before the next process call. */
   process(input: Float32Array): Float32Array;
   reset(): void;
   destroy(): void;
@@ -128,10 +129,9 @@ export class WasmStretcher implements Stretcher {
       return new Float32Array(0);
     }
 
-    const view = new Float32Array(this.exports.memory.buffer, this.outPtr, outFrames * ch);
-    const out = new Float32Array(outFrames * ch);
-    out.set(view);
-    return out;
+    // scheduleOutput consumes this synchronously into an AudioBuffer. Keeping
+    // a borrowed view avoids copying every PCM chunk into a temporary JS array.
+    return new Float32Array(this.exports.memory.buffer, this.outPtr, outFrames * ch);
   }
 
   reset(): void {

web-ui/src/playback-engine/decoder/mpeg-audio-decoder.ts

@@ -8,8 +8,7 @@
  * and keeps trailing partial frames in an internal carry buffer, so frames
  * split across PES packets are handled transparently.
  *
- * The WASM is built as standalone (`-o .wasm`) with -O2 to preserve
- * readable export/import names.
+ * The WASM is built as standalone (`-o .wasm`), retaining its public exports.
  */
 
 // Maximum samples per frame for MPEG audio (all channels interleaved)

web-ui/src/playback-engine/demux/ts-demuxer.test.ts

@@ -0,0 +1,56 @@
+import { describe, expect, it, vi } from "vitest";
+import TSDemuxer from "./ts-demuxer";
+
+function section(pid: number, bytes: number[]): Uint8Array {
+  const packet = new Uint8Array(188).fill(0xff);
+  packet.set([0x47, 0x40 | (pid >> 8), pid & 255, 0x10, 0, ...bytes]);
+  return packet;
+}
+function pcrPacket(base: number, discontinuity = false): Uint8Array {
+  const packet = new Uint8Array(188).fill(0xff);
+  packet.set([0x47, 1, 1, 0x20, 183, discontinuity ? 0x90 : 0x10]);
+  packet.set(
+    [
+      Math.floor(base / 2 ** 25),
+      Math.floor(base / 2 ** 17) & 255,
+      Math.floor(base / 2 ** 9) & 255,
+      Math.floor(base / 2) & 255,
+      ((base % 2) << 7) | 0x7e,
+      0,
+    ],
+    6,
+  );
+  return packet;
+}
+function transport(packets: Uint8Array[], stride: number): Uint8Array {
+  const output = new Uint8Array(packets.length * stride);
+  packets.forEach((packet, i) => {
+    output.set(packet, i * stride + (stride === 192 ? 4 : 0));
+  });
+  return output;
+}
+
+describe("TS packet offsets", () => {
+  it.each([188, 192, 204])("reads PCR and discontinuity flags with %i-byte packets", (stride) => {
+    // A single-program PAT/PMT declares PID 257 as the PCR/video PID.
+    const pat = section(0, [0, 0xb0, 13, 0, 1, 0xc1, 0, 0, 0, 1, 0xe1, 0, 0, 0, 0, 0]);
+    const pmt = section(256, [2, 0xb0, 18, 0, 1, 0xc1, 0, 0, 0xe1, 1, 0xf0, 0, 0x1b, 0xe1, 1, 0xf0, 0, 0, 0, 0, 0]);
+    const packets = [pat, pmt, pcrPacket(2 ** 33 - 90000), pcrPacket(0, true), pcrPacket(90000)];
+    const bytes = transport(packets, stride);
+    const demux = new TSDemuxer({ match: true, ts_packet_size: stride, sync_offset: 0 });
+    demux.onError = vi.fn();
+    demux.onTrackMetadata = vi.fn();
+    demux.onDataAvailable = vi.fn();
+    const pcr = vi.fn();
+    demux.onPcr = pcr;
+    // The second call starts at a nonzero byteOffset in the same allocation.
+    expect(demux.parseChunks(bytes.subarray(0, stride * 3), 1000)).toBe(stride * 3);
+    expect(demux.parseChunks(bytes.subarray(stride * 3), 1000 + stride * 3)).toBe(stride * 2);
+    expect(pcr.mock.calls).toEqual([
+      [2 ** 33 - 90000, 1000 + stride * 2, false],
+      [2 ** 33, 1000 + stride * 3, true],
+      [2 ** 33 + 90000, 1000 + stride * 4, false],
+    ]);
+    expect(demux.onError).not.toHaveBeenCalled();
+  });
+});

web-ui/src/playback-engine/demux/ts-demuxer.ts

@@ -49,11 +49,6 @@ interface TSSliceMisc {
   stream_type?: StreamType;
 }
 
-type AdaptationFieldInfo = {
-  discontinuity_indicator?: number;
-  random_access_indicator?: number;
-  elementary_stream_priority_indicator?: number;
-};
 type CommonPidKey = keyof PMT["common_pids"];
 type TSDemuxerOptions = {
   waitForInitialVideoKeyframe?: boolean;
@@ -317,7 +312,11 @@ class TSDemuxer {
 
   private isCommonPid(pid: number, keys: readonly CommonPidKey[]): boolean {
     const commonPids = this.pmt_?.common_pids;
-    return !!commonPids && keys.some((key) => commonPids[key] === pid);
+    if (!commonPids) return false;
+    for (const key of keys) {
+      if (commonPids[key] === pid) return true;
+    }
+    return false;
   }
 
   private isVideoPid(pid: number): boolean {
@@ -460,38 +459,36 @@ class TSDemuxer {
         offset += 4;
       }
 
-      const data = chunk.subarray(offset, offset + 188);
-
-      const sync_byte = data[0];
+      const sync_byte = chunk[offset];
       if (sync_byte !== 0x47) {
         Log.e(this.TAG, `sync_byte = ${sync_byte}, not 0x47`);
         break;
       }
 
-      const payload_unit_start_indicator = (data[1] & 0x40) >>> 6;
-      const pid = ((data[1] & 0x1f) << 8) | data[2];
-      const adaptation_field_control = (data[3] & 0x30) >>> 4;
-      const continuity_conunter = data[3] & 0x0f;
+      const payload_unit_start_indicator = (chunk[offset + 1] & 0x40) >>> 6;
+      const pid = ((chunk[offset + 1] & 0x1f) << 8) | chunk[offset + 2];
+      const adaptation_field_control = (chunk[offset + 3] & 0x30) >>> 4;
+      const continuity_conunter = chunk[offset + 3] & 0x0f;
 
       const is_pcr_pid: boolean = !!(this.pmt_ && this.pmt_.pcr_pid === pid);
-      const adaptation_field_info: AdaptationFieldInfo = {};
+      let discontinuityIndicator: number | undefined;
+      let randomAccessIndicator: number | undefined;
       let ts_payload_start_index = 4;
 
       if (adaptation_field_control === 0x02 || adaptation_field_control === 0x03) {
         // Adaptation field exists along with / without payload
-        const adaptation_field_length = data[4];
+        const adaptation_field_length = chunk[offset + 4];
         if (adaptation_field_length > 0 && (is_pcr_pid || adaptation_field_control === 0x03)) {
           // Parse adaptation field
-          adaptation_field_info.discontinuity_indicator = (data[5] & 0x80) >>> 7;
-          adaptation_field_info.random_access_indicator = (data[5] & 0x40) >>> 6;
-          adaptation_field_info.elementary_stream_priority_indicator = (data[5] & 0x20) >>> 5;
+          discontinuityIndicator = (chunk[offset + 5] & 0x80) >>> 7;
+          randomAccessIndicator = (chunk[offset + 5] & 0x40) >>> 6;
 
-          const PCR_flag = (data[5] & 0x10) >>> 4;
+          const PCR_flag = (chunk[offset + 5] & 0x10) >>> 4;
           if (PCR_flag) {
             // track PCR base for pts/dts wraparound detection
-            const pcrBase = this.getPcrBase(data);
+            const pcrBase = this.getPcrBase(chunk, offset);
             if (is_pcr_pid) {
-              this.onPcr?.(pcrBase, file_position, adaptation_field_info.discontinuity_indicator === 1);
+              this.onPcr?.(pcrBase, file_position, discontinuityIndicator === 1);
             }
           }
         }
@@ -521,7 +518,7 @@ class TSDemuxer {
             file_position,
             payload_unit_start_indicator,
             continuity_conunter,
-            random_access_indicator: adaptation_field_info.random_access_indicator,
+            random_access_indicator: randomAccessIndicator,
           });
         } else if (this.pmt_ !== undefined && this.pmt_.pid_stream_type[pid] !== undefined) {
           // PES
@@ -530,7 +527,7 @@ class TSDemuxer {
 
           // process PES only for known common_pids
           if (this.isMediaPid(pid)) {
-            if (!this.shouldProcessPayload(pid, continuity_conunter, adaptation_field_info.discontinuity_indicator)) {
+            if (!this.shouldProcessPayload(pid, continuity_conunter, discontinuityIndicator)) {
               offset += 188;
               if (this.ts_packet_size_ === 204) {
                 offset += 16;
@@ -543,7 +540,7 @@ class TSDemuxer {
               file_position,
               payload_unit_start_indicator,
               continuity_conunter,
-              random_access_indicator: adaptation_field_info.random_access_indicator,
+              random_access_indicator: randomAccessIndicator,
             });
           }
         }
@@ -2150,13 +2147,13 @@ class TSDemuxer {
     this.video_metadata_changed_ = false;
   }
 
-  private getPcrBase(data: Uint8Array): number {
+  private getPcrBase(data: Uint8Array, offset: number): number {
     let pcr_base =
-      data[6] * 33554432 + // 1 << 25
-      data[7] * 131072 + // 1 << 17
-      data[8] * 512 + // 1 << 9
-      data[9] * 2 + // 1 << 1
-      (data[10] & 0x80) / 128 + // 1 >> 7
+      data[offset + 6] * 33554432 + // 1 << 25
+      data[offset + 7] * 131072 + // 1 << 17
+      data[offset + 8] * 512 + // 1 << 9
+      data[offset + 9] * 2 + // 1 << 1
+      (data[offset + 10] & 0x80) / 128 + // 1 >> 7
       this.timestamp_offset_;
     if (pcr_base + 0x100000000 < this.last_pcr_base_) {
       pcr_base += 0x200000000; // pcr_base wraparound

web-ui/src/playback-engine/wasm/minimp3/Makefile

@@ -3,9 +3,8 @@
 EMCC = emcc
 OUTPUT = mp2_decoder
 
-# -O2 preserves readable export names (-O3 would minify them).
-# -o .wasm produces standalone WASM with no JS glue.
-EMFLAGS = -O2 \
+# Standalone WASM has no JS glue and retains the public KEEPALIVE exports.
+EMFLAGS = -O3 \
 	-s EXPORTED_FUNCTIONS='["_malloc", "_free"]' \
 	-s EXPORTED_RUNTIME_METHODS='[]' \
 	-s ALLOW_MEMORY_GROWTH=1 \

web-ui/src/playback-engine/wasm/minimp3/wsola.c

@@ -310,7 +310,35 @@ int wsola_process(Wsola* w, const float* input, int in_frames, float* output, in
 
         int best_q = p;
         float best_score = -1e30f;
-        for (int q = lo; q <= hi; q++) {
+        /* Score eight adjacent candidates together. Each dot product retains
+         * its original summation order, while independent accumulators let
+         * scalar WASM overlap arithmetic and reuse the reference loads.
+         * Keep the exhaustive search and tie ordering (audio is unchanged). */
+        int q = lo;
+        for (; q + 7 <= hi; q += 8) {
+            const float* cand = w->mix + (q - lo);
+            float dot[8] = {0};
+            for (int i = 0; i < w->overlap; i++) {
+                float ref = w->tail_mono[i];
+                dot[0] += ref * cand[i];
+                dot[1] += ref * cand[i + 1];
+                dot[2] += ref * cand[i + 2];
+                dot[3] += ref * cand[i + 3];
+                dot[4] += ref * cand[i + 4];
+                dot[5] += ref * cand[i + 5];
+                dot[6] += ref * cand[i + 6];
+                dot[7] += ref * cand[i + 7];
+            }
+            for (int j = 0; j < 8; j++) {
+                float cand_energy = w->energy[q + j - lo + w->overlap] - w->energy[q + j - lo];
+                float score = dot[j] / sqrtf(cand_energy + 1e-9f);
+                if (score > best_score) {
+                    best_score = score;
+                    best_q = q + j;
+                }
+            }
+        }
+        for (; q <= hi; q++) {
             const float* cand = w->mix + (q - lo);
             float dot = 0.0f;
             for (int i = 0; i < w->overlap; i++) {

web-ui/src/playback-engine/wasm/minimp3/wsola.test.ts

@@ -0,0 +1,86 @@
+import { readFileSync } from "node:fs";
+import { beforeAll, describe, expect, it } from "vitest";
+
+interface Wsola {
+  memory: WebAssembly.Memory;
+  _initialize(): void;
+  malloc(bytes: number): number;
+  wsola_create(rate: number, channels: number): number;
+  wsola_set_ratio(handle: number, ratio: number): void;
+  wsola_process(handle: number, input: number, frames: number, output: number, capacity: number): number;
+  wsola_reset(handle: number): void;
+  wsola_position(handle: number): number;
+}
+let module: WebAssembly.Module;
+beforeAll(async () => {
+  module = await WebAssembly.compile(readFileSync(new URL("./mp2_decoder.wasm", import.meta.url)));
+});
+
+async function stretch(input: Float32Array, rate: number, channels: number, ratio: number) {
+  const instance = await WebAssembly.instantiate(module, { env: { emscripten_notify_memory_growth() {} } });
+  const x = instance.exports as unknown as Wsola;
+  x._initialize();
+  const handle = x.wsola_create(rate, channels);
+  const src = x.malloc(4096 * channels * 4);
+  const dst = x.malloc(16384 * channels * 4);
+  x.wsola_set_ratio(handle, ratio);
+  const chunks: Float32Array[] = [];
+  let count = 0;
+  const sizes = [1, 97, 1152, 4096, 333];
+  for (let pos = 0, index = 0; pos < input.length; index++) {
+    const frames = Math.min(sizes[index % sizes.length], (input.length - pos) / channels);
+    new Float32Array(x.memory.buffer, src, frames * channels).set(input.subarray(pos, pos + frames * channels));
+    const n = x.wsola_process(handle, src, frames, dst, 16384);
+    const output = new Float32Array(x.memory.buffer, dst, n * channels).slice();
+    chunks.push(output);
+    count += output.length;
+    pos += frames * channels;
+  }
+  const output = new Float32Array(count);
+  let pos = 0;
+  for (const chunk of chunks) {
+    output.set(chunk, pos);
+    pos += chunk.length;
+  }
+  x.wsola_reset(handle);
+  expect(x.wsola_position(handle)).toBe(0);
+  expect(x.wsola_process(handle, src, 0, dst, 16384)).toBe(0);
+  return output;
+}
+
+function tone(rate: number, channels: number): Float32Array {
+  return Float32Array.from(
+    { length: rate * 3 * channels },
+    (_, i) => 0.4 * Math.sin((2 * Math.PI * 440 * Math.floor(i / channels)) / rate),
+  );
+}
+
+describe("WSOLA PCM contract", () => {
+  it.each([8000, 11025, 22050, 32000, 44100, 48000])("preserves samples at 1x, %i Hz", async (rate) => {
+    for (const channels of [1, 2]) {
+      const input = tone(rate, channels);
+      expect(await stretch(input, rate, channels, 1)).toEqual(input);
+    }
+  });
+
+  it.each([0.5, 0.9, 1.01, 1.2, 2])("preserves pitch and requested duration at %f x", async (ratio) => {
+    for (const rate of [11025, 44100, 48000]) {
+      const output = await stretch(tone(rate, 2), rate, 2, ratio);
+      expect(Math.abs(output.length / (rate * 2) - 3 / ratio)).toBeLessThan(0.13);
+      let crossings = 0;
+      const start = Math.floor(rate / 5) * 2;
+      const end = output.length - start;
+      for (let i = start + 2; i < end; i += 2) {
+        if (output[i - 2] < 0 && output[i] >= 0) crossings++;
+      }
+      expect(output.every((sample, i) => Number.isFinite(sample) && sample === output[i - (i % 2)])).toBe(true);
+      expect(Math.abs((crossings * rate * 2) / (end - start) - 440)).toBeLessThan(6);
+    }
+  });
+
+  it("keeps silence silent while correcting drift", async () => {
+    const output = await stretch(new Float32Array(48000 * 2), 48000, 2, 1.01);
+    expect(output.length).toBeGreaterThan(48000);
+    expect(output.every((sample) => sample === 0)).toBe(true);
+  });
+});