diff --git a/web-ui/src/components/player/video-player.tsx b/web-ui/src/components/player/video-player.tsx index cc484a78..84adcf35 100644 --- a/web-ui/src/components/player/video-player.tsx +++ b/web-ui/src/components/player/video-player.tsx @@ -1995,9 +1995,11 @@ function VideoPlayerComponent({ className={clsx( "player-performance-controls-position player-performance-motion absolute bottom-0 left-[calc(0px_-_env(safe-area-inset-left))] right-[calc(0px_-_env(safe-area-inset-right))] z-10 transition-opacity duration-300", showSidebar && "md:right-0", + // Invisible pulse animations still wake the compositor. Restore the + // live indicator animation when pointer or keyboard controls appear. showControls ? "opacity-100" - : "opacity-0 pointer-events-none has-focus-visible:opacity-100 has-focus-visible:pointer-events-auto", + : "opacity-0 pointer-events-none has-focus-visible:opacity-100 has-focus-visible:pointer-events-auto [&_.animate-pulse]:animate-none has-focus-visible:[&_.animate-pulse]:animate-pulse", )} > > 8), pid & 255, 0x10, 0, ...bytes]); + return packet; +} +function pcrPacket(base: number, discontinuity = false): Uint8Array { + const packet = new Uint8Array(188).fill(0xff); + packet.set([0x47, 1, 1, 0x20, 183, discontinuity ? 0x90 : 0x10]); + packet.set( + [ + Math.floor(base / 2 ** 25), + Math.floor(base / 2 ** 17) & 255, + Math.floor(base / 2 ** 9) & 255, + Math.floor(base / 2) & 255, + ((base % 2) << 7) | 0x7e, + 0, + ], + 6, + ); + return packet; +} +function transport(packets: Uint8Array[], stride: number): Uint8Array { + const output = new Uint8Array(packets.length * stride); + packets.forEach((packet, i) => { + output.set(packet, i * stride + (stride === 192 ? 4 : 0)); + }); + return output; +} + +describe("TS packet offsets", () => { + it.each([188, 192, 204])("reads PCR and discontinuity flags with %i-byte packets", (stride) => { + // A single-program PAT/PMT declares PID 257 as the PCR/video PID. + const pat = section(0, [0, 0xb0, 13, 0, 1, 0xc1, 0, 0, 0, 1, 0xe1, 0, 0, 0, 0, 0]); + const pmt = section(256, [2, 0xb0, 18, 0, 1, 0xc1, 0, 0, 0xe1, 1, 0xf0, 0, 0x1b, 0xe1, 1, 0xf0, 0, 0, 0, 0, 0]); + const packets = [pat, pmt, pcrPacket(2 ** 33 - 90000), pcrPacket(0, true), pcrPacket(90000)]; + const bytes = transport(packets, stride); + const demux = new TSDemuxer({ match: true, ts_packet_size: stride, sync_offset: 0 }); + demux.onError = vi.fn(); + demux.onTrackMetadata = vi.fn(); + demux.onDataAvailable = vi.fn(); + const pcr = vi.fn(); + demux.onPcr = pcr; + // The second call starts at a nonzero byteOffset in the same allocation. + expect(demux.parseChunks(bytes.subarray(0, stride * 3), 1000)).toBe(stride * 3); + expect(demux.parseChunks(bytes.subarray(stride * 3), 1000 + stride * 3)).toBe(stride * 2); + expect(pcr.mock.calls).toEqual([ + [2 ** 33 - 90000, 1000 + stride * 2, false], + [2 ** 33, 1000 + stride * 3, true], + [2 ** 33 + 90000, 1000 + stride * 4, false], + ]); + expect(demux.onError).not.toHaveBeenCalled(); + }); +}); diff --git a/web-ui/src/playback-engine/demux/ts-demuxer.ts b/web-ui/src/playback-engine/demux/ts-demuxer.ts index 62f11bfc..5215eb5e 100644 --- a/web-ui/src/playback-engine/demux/ts-demuxer.ts +++ b/web-ui/src/playback-engine/demux/ts-demuxer.ts @@ -49,11 +49,6 @@ interface TSSliceMisc { stream_type?: StreamType; } -type AdaptationFieldInfo = { - discontinuity_indicator?: number; - random_access_indicator?: number; - elementary_stream_priority_indicator?: number; -}; type CommonPidKey = keyof PMT["common_pids"]; type TSDemuxerOptions = { waitForInitialVideoKeyframe?: boolean; @@ -317,7 +312,11 @@ class TSDemuxer { private isCommonPid(pid: number, keys: readonly CommonPidKey[]): boolean { const commonPids = this.pmt_?.common_pids; - return !!commonPids && keys.some((key) => commonPids[key] === pid); + if (!commonPids) return false; + for (const key of keys) { + if (commonPids[key] === pid) return true; + } + return false; } private isVideoPid(pid: number): boolean { @@ -460,38 +459,36 @@ class TSDemuxer { offset += 4; } - const data = chunk.subarray(offset, offset + 188); - - const sync_byte = data[0]; + const sync_byte = chunk[offset]; if (sync_byte !== 0x47) { Log.e(this.TAG, `sync_byte = ${sync_byte}, not 0x47`); break; } - const payload_unit_start_indicator = (data[1] & 0x40) >>> 6; - const pid = ((data[1] & 0x1f) << 8) | data[2]; - const adaptation_field_control = (data[3] & 0x30) >>> 4; - const continuity_conunter = data[3] & 0x0f; + const payload_unit_start_indicator = (chunk[offset + 1] & 0x40) >>> 6; + const pid = ((chunk[offset + 1] & 0x1f) << 8) | chunk[offset + 2]; + const adaptation_field_control = (chunk[offset + 3] & 0x30) >>> 4; + const continuity_conunter = chunk[offset + 3] & 0x0f; const is_pcr_pid: boolean = !!(this.pmt_ && this.pmt_.pcr_pid === pid); - const adaptation_field_info: AdaptationFieldInfo = {}; + let discontinuityIndicator: number | undefined; + let randomAccessIndicator: number | undefined; let ts_payload_start_index = 4; if (adaptation_field_control === 0x02 || adaptation_field_control === 0x03) { // Adaptation field exists along with / without payload - const adaptation_field_length = data[4]; + const adaptation_field_length = chunk[offset + 4]; if (adaptation_field_length > 0 && (is_pcr_pid || adaptation_field_control === 0x03)) { // Parse adaptation field - adaptation_field_info.discontinuity_indicator = (data[5] & 0x80) >>> 7; - adaptation_field_info.random_access_indicator = (data[5] & 0x40) >>> 6; - adaptation_field_info.elementary_stream_priority_indicator = (data[5] & 0x20) >>> 5; + discontinuityIndicator = (chunk[offset + 5] & 0x80) >>> 7; + randomAccessIndicator = (chunk[offset + 5] & 0x40) >>> 6; - const PCR_flag = (data[5] & 0x10) >>> 4; + const PCR_flag = (chunk[offset + 5] & 0x10) >>> 4; if (PCR_flag) { // track PCR base for pts/dts wraparound detection - const pcrBase = this.getPcrBase(data); + const pcrBase = this.getPcrBase(chunk, offset); if (is_pcr_pid) { - this.onPcr?.(pcrBase, file_position, adaptation_field_info.discontinuity_indicator === 1); + this.onPcr?.(pcrBase, file_position, discontinuityIndicator === 1); } } } @@ -521,7 +518,7 @@ class TSDemuxer { file_position, payload_unit_start_indicator, continuity_conunter, - random_access_indicator: adaptation_field_info.random_access_indicator, + random_access_indicator: randomAccessIndicator, }); } else if (this.pmt_ !== undefined && this.pmt_.pid_stream_type[pid] !== undefined) { // PES @@ -530,7 +527,7 @@ class TSDemuxer { // process PES only for known common_pids if (this.isMediaPid(pid)) { - if (!this.shouldProcessPayload(pid, continuity_conunter, adaptation_field_info.discontinuity_indicator)) { + if (!this.shouldProcessPayload(pid, continuity_conunter, discontinuityIndicator)) { offset += 188; if (this.ts_packet_size_ === 204) { offset += 16; @@ -543,7 +540,7 @@ class TSDemuxer { file_position, payload_unit_start_indicator, continuity_conunter, - random_access_indicator: adaptation_field_info.random_access_indicator, + random_access_indicator: randomAccessIndicator, }); } } @@ -2150,13 +2147,13 @@ class TSDemuxer { this.video_metadata_changed_ = false; } - private getPcrBase(data: Uint8Array): number { + private getPcrBase(data: Uint8Array, offset: number): number { let pcr_base = - data[6] * 33554432 + // 1 << 25 - data[7] * 131072 + // 1 << 17 - data[8] * 512 + // 1 << 9 - data[9] * 2 + // 1 << 1 - (data[10] & 0x80) / 128 + // 1 >> 7 + data[offset + 6] * 33554432 + // 1 << 25 + data[offset + 7] * 131072 + // 1 << 17 + data[offset + 8] * 512 + // 1 << 9 + data[offset + 9] * 2 + // 1 << 1 + (data[offset + 10] & 0x80) / 128 + // 1 >> 7 this.timestamp_offset_; if (pcr_base + 0x100000000 < this.last_pcr_base_) { pcr_base += 0x200000000; // pcr_base wraparound diff --git a/web-ui/src/playback-engine/wasm/minimp3/Makefile b/web-ui/src/playback-engine/wasm/minimp3/Makefile index 6c89d90b..5406be86 100644 --- a/web-ui/src/playback-engine/wasm/minimp3/Makefile +++ b/web-ui/src/playback-engine/wasm/minimp3/Makefile @@ -3,9 +3,8 @@ EMCC = emcc OUTPUT = mp2_decoder -# -O2 preserves readable export names (-O3 would minify them). -# -o .wasm produces standalone WASM with no JS glue. -EMFLAGS = -O2 \ +# Standalone WASM has no JS glue and retains the public KEEPALIVE exports. +EMFLAGS = -O3 \ -s EXPORTED_FUNCTIONS='["_malloc", "_free"]' \ -s EXPORTED_RUNTIME_METHODS='[]' \ -s ALLOW_MEMORY_GROWTH=1 \ diff --git a/web-ui/src/playback-engine/wasm/minimp3/mp2_decoder.wasm b/web-ui/src/playback-engine/wasm/minimp3/mp2_decoder.wasm index ec367c8c..44ddf8be 100755 Binary files a/web-ui/src/playback-engine/wasm/minimp3/mp2_decoder.wasm and b/web-ui/src/playback-engine/wasm/minimp3/mp2_decoder.wasm differ diff --git a/web-ui/src/playback-engine/wasm/minimp3/wsola.c b/web-ui/src/playback-engine/wasm/minimp3/wsola.c index 547a2b8f..d01868c3 100644 --- a/web-ui/src/playback-engine/wasm/minimp3/wsola.c +++ b/web-ui/src/playback-engine/wasm/minimp3/wsola.c @@ -310,7 +310,35 @@ int wsola_process(Wsola* w, const float* input, int in_frames, float* output, in int best_q = p; float best_score = -1e30f; - for (int q = lo; q <= hi; q++) { + /* Score eight adjacent candidates together. Each dot product retains + * its original summation order, while independent accumulators let + * scalar WASM overlap arithmetic and reuse the reference loads. + * Keep the exhaustive search and tie ordering (audio is unchanged). */ + int q = lo; + for (; q + 7 <= hi; q += 8) { + const float* cand = w->mix + (q - lo); + float dot[8] = {0}; + for (int i = 0; i < w->overlap; i++) { + float ref = w->tail_mono[i]; + dot[0] += ref * cand[i]; + dot[1] += ref * cand[i + 1]; + dot[2] += ref * cand[i + 2]; + dot[3] += ref * cand[i + 3]; + dot[4] += ref * cand[i + 4]; + dot[5] += ref * cand[i + 5]; + dot[6] += ref * cand[i + 6]; + dot[7] += ref * cand[i + 7]; + } + for (int j = 0; j < 8; j++) { + float cand_energy = w->energy[q + j - lo + w->overlap] - w->energy[q + j - lo]; + float score = dot[j] / sqrtf(cand_energy + 1e-9f); + if (score > best_score) { + best_score = score; + best_q = q + j; + } + } + } + for (; q <= hi; q++) { const float* cand = w->mix + (q - lo); float dot = 0.0f; for (int i = 0; i < w->overlap; i++) { diff --git a/web-ui/src/playback-engine/wasm/minimp3/wsola.test.ts b/web-ui/src/playback-engine/wasm/minimp3/wsola.test.ts new file mode 100644 index 00000000..63e6eaa8 --- /dev/null +++ b/web-ui/src/playback-engine/wasm/minimp3/wsola.test.ts @@ -0,0 +1,86 @@ +import { readFileSync } from "node:fs"; +import { beforeAll, describe, expect, it } from "vitest"; + +interface Wsola { + memory: WebAssembly.Memory; + _initialize(): void; + malloc(bytes: number): number; + wsola_create(rate: number, channels: number): number; + wsola_set_ratio(handle: number, ratio: number): void; + wsola_process(handle: number, input: number, frames: number, output: number, capacity: number): number; + wsola_reset(handle: number): void; + wsola_position(handle: number): number; +} +let module: WebAssembly.Module; +beforeAll(async () => { + module = await WebAssembly.compile(readFileSync(new URL("./mp2_decoder.wasm", import.meta.url))); +}); + +async function stretch(input: Float32Array, rate: number, channels: number, ratio: number) { + const instance = await WebAssembly.instantiate(module, { env: { emscripten_notify_memory_growth() {} } }); + const x = instance.exports as unknown as Wsola; + x._initialize(); + const handle = x.wsola_create(rate, channels); + const src = x.malloc(4096 * channels * 4); + const dst = x.malloc(16384 * channels * 4); + x.wsola_set_ratio(handle, ratio); + const chunks: Float32Array[] = []; + let count = 0; + const sizes = [1, 97, 1152, 4096, 333]; + for (let pos = 0, index = 0; pos < input.length; index++) { + const frames = Math.min(sizes[index % sizes.length], (input.length - pos) / channels); + new Float32Array(x.memory.buffer, src, frames * channels).set(input.subarray(pos, pos + frames * channels)); + const n = x.wsola_process(handle, src, frames, dst, 16384); + const output = new Float32Array(x.memory.buffer, dst, n * channels).slice(); + chunks.push(output); + count += output.length; + pos += frames * channels; + } + const output = new Float32Array(count); + let pos = 0; + for (const chunk of chunks) { + output.set(chunk, pos); + pos += chunk.length; + } + x.wsola_reset(handle); + expect(x.wsola_position(handle)).toBe(0); + expect(x.wsola_process(handle, src, 0, dst, 16384)).toBe(0); + return output; +} + +function tone(rate: number, channels: number): Float32Array { + return Float32Array.from( + { length: rate * 3 * channels }, + (_, i) => 0.4 * Math.sin((2 * Math.PI * 440 * Math.floor(i / channels)) / rate), + ); +} + +describe("WSOLA PCM contract", () => { + it.each([8000, 11025, 22050, 32000, 44100, 48000])("preserves samples at 1x, %i Hz", async (rate) => { + for (const channels of [1, 2]) { + const input = tone(rate, channels); + expect(await stretch(input, rate, channels, 1)).toEqual(input); + } + }); + + it.each([0.5, 0.9, 1.01, 1.2, 2])("preserves pitch and requested duration at %f x", async (ratio) => { + for (const rate of [11025, 44100, 48000]) { + const output = await stretch(tone(rate, 2), rate, 2, ratio); + expect(Math.abs(output.length / (rate * 2) - 3 / ratio)).toBeLessThan(0.13); + let crossings = 0; + const start = Math.floor(rate / 5) * 2; + const end = output.length - start; + for (let i = start + 2; i < end; i += 2) { + if (output[i - 2] < 0 && output[i] >= 0) crossings++; + } + expect(output.every((sample, i) => Number.isFinite(sample) && sample === output[i - (i % 2)])).toBe(true); + expect(Math.abs((crossings * rate * 2) / (end - start) - 440)).toBeLessThan(6); + } + }); + + it("keeps silence silent while correcting drift", async () => { + const output = await stretch(new Float32Array(48000 * 2), 48000, 2, 1.01); + expect(output.length).toBeGreaterThan(48000); + expect(output.every((sample) => sample === 0)).toBe(true); + }); +});