Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion doc/lib/js/watch.md
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ in sync at the latency you ask for.
| `url`, `name` | Relay URL (with `?jwt=` if needed) and broadcast name. |
| `versions` | Protocol versions to offer, most preferred first, comma-separated like `moq-transport-18,moq-lite-06`. Every stable version by default. |
| `paused`, `muted`, `volume` | The usual player controls. |
| `delay` | How far playback trails the live edge: `"auto"` (the default, sized from how late frames arrive; see [audio jitter](/concept/audio-jitter)), a duration like `"300ms"`, or `"instant"` to paint video as it decodes, with no audio. |
| `delay` | How far playback trails the live edge: `"auto"` (the default, sized from how late frames arrive, at most 10 s; see [audio jitter](/concept/audio-jitter)), a duration like `"300ms"`, or `"instant"` to paint video as it decodes, with no audio. |
| `lookahead` | How far ahead of the live edge media may run; see [buffered playback](#buffered-playback). |
| `stretch` | How far audio may trail the clock before it skips ahead: a duration like `"2s"` (the default), or `"0"` to turn the time stretch off; see [catching up](#catching-up). |
| `captions` | The caption track to show. |
Expand Down
41 changes: 40 additions & 1 deletion js/watch/src/audio/decoder.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,14 @@ import * as Moq from "@moq/net";
import { Effect, Signal } from "@moq/signals";
import type { Broadcast } from "../broadcast";
import { supportCacheKey } from "../support-probe";
import { type Delay, STRETCH, Sync } from "../sync";
import { DELAY_MAX, type Delay, STRETCH, Sync } from "../sync";
import { Decoder as VideoDecoder } from "../video/decoder";
import { Source as VideoSource } from "../video/source";
import { decoderConfigKey } from "./config";
import type { Message, State, Terms } from "./render";
import { AudioRingBuffer } from "./ring-buffer";
import { Source } from "./source";
import { Terminal } from "./terminal";

// Bun cannot load the blob-URL worklet import.
mock.module("./render-worklet.ts?worklet", () => ({ default: async () => "blob:fake-render" }));
Expand Down Expand Up @@ -634,6 +635,44 @@ describe("Decoder across a broadcast disable", () => {
});

describe("Decoder across a delay change", () => {
it("reaches back and trims the join for a catalog warmup no further than the delay cap", async () => {
const trims = spyOn(Terminal.prototype, "clear");
// The max delay each subscribe and update asked the publisher for, in order.
const asked: (Duration | null | undefined)[] = [];
const subscribe = Moq.Track.Consumer.prototype.subscribe;
const update = Moq.Track.Subscriber.prototype.update;
const subscribes = spyOn(Moq.Track.Consumer.prototype, "subscribe").mockImplementation(function (
this: Moq.Track.Consumer,
...args: Parameters<typeof subscribe>
) {
asked.push(args[0]?.delay?.max);
return subscribe.apply(this, args);
});
const updates = spyOn(Moq.Track.Subscriber.prototype, "update").mockImplementation(function (
this: Moq.Track.Subscriber,
...args: Parameters<typeof update>
) {
asked.push(args[0]?.delay?.max);
return update.apply(this, args);
});
const playback = await play(Duration.millis(100));
try {
await playback.play();
const base = asked.at(-1);
if (!base) throw new Error("no subscription");
playback.restore({ warmup: 3_600_000 });
await microtasks();
await playback.play();
expect(asked.at(-1)).toEqual(base.add(DELAY_MAX));
expect(trims.mock.calls.at(-1)?.[1]).toBe(DELAY_MAX.asMicros());
} finally {
trims.mockRestore();
subscribes.mockRestore();
updates.mockRestore();
playback.close();
}
});

for (const initial of [Duration.millis(100), "auto"] as const) {
it(`keeps the subscription and ring when ${initial} becomes a number`, async () => {
const playback = await play(initial);
Expand Down
20 changes: 15 additions & 5 deletions js/watch/src/audio/decoder.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ import { hosted } from "../assets";
import { base64ToBytes } from "../base64";
import { arrivedAt, type Counter, type Key, type Received, Tally } from "../feedback";
import { type Failure, failureMoved, nextMedia, subscribeMedia } from "../media";
import type { Sync, Track } from "../sync";
import { DELAY_MAX, type Sync, type Track } from "../sync";
import { AudioBuffer } from "./buffer";
import { type DecoderConfig, frameDuration, type PlaybackIdentity, packetDuration, playbackIdentity } from "./config";
import { Handover } from "./handover";
Expand Down Expand Up @@ -333,7 +333,7 @@ export class Decoder {
const budget = this.#budget(effect);
const delay = budget.delay;
const max = maxDelay(budget);
const warmup = Duration.millis(effect.get(this.source.out.config)?.warmup ?? 0);
const warmup = Duration.millis(effect.get(this.source.out.config)?.warmup ?? 0).min(DELAY_MAX);
this.#delay.set({ min: delay, max });
this.#subscribeMaxDelay.set(max.add(warmup));
}
Expand Down Expand Up @@ -619,7 +619,12 @@ export class Decoder {
#runLegacyDecoder(effect: Effect, sub: Moq.Track.Subscriber, { config, failure, path, key }: Attempt): void {
const preSkip =
config.codec === "opus" && config.description ? Util.Opus.preSkip(Util.Hex.toBytes(config.description)) : 0;
this.#terminal.clear(preSkip, (config.warmup ?? 0) * 1000);
this.#terminal.clear(
preSkip,
Duration.millis(config.warmup ?? 0)
.min(DELAY_MAX)
.asMicros(),
);
const format =
config.container.kind === "loc" ? new Container.Loc.Format("audio") : new Container.Legacy.Format(config);
// Create consumer with slightly less latency than the render worklet to avoid underflowing.
Expand Down Expand Up @@ -705,7 +710,12 @@ export class Decoder {
const init = Cmaf.decodeInitSegment(initSegment);
const opusDescription = config.description ? Util.Hex.toBytes(config.description) : init.description;
const preSkip = config.codec === "opus" && opusDescription ? Util.Opus.preSkip(opusDescription) : 0;
this.#terminal.clear(preSkip, (config.warmup ?? 0) * 1000);
this.#terminal.clear(
preSkip,
Duration.millis(config.warmup ?? 0)
.min(DELAY_MAX)
.asMicros(),
);
// Opus in CMAF uses raw packets (not OGG-wrapped), so description must be omitted.
// The dOps box from the init segment is not a valid OGG Identification Header.
const description =
Expand Down Expand Up @@ -858,7 +868,7 @@ export class Decoder {
format,
delay: this.#delay,
wait: this.sync.out.maxDelay,
warmup: Duration.millis(config.warmup ?? 0),
warmup: Duration.millis(config.warmup ?? 0).min(DELAY_MAX),
Comment thread
kixelated marked this conversation as resolved.
Resolved
predecessor: continues ? outgoing.consumer : undefined,
});
effect.cleanup(() => consumer.close());
Expand Down
22 changes: 22 additions & 0 deletions js/watch/src/sync.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import { Duration, type Installed, Instant, Manual, now, yieldNow } from "@moq/c
const at = (ms: number) => Instant.ZERO.saturatingAdd(Duration.millis(ms));

import { Signal } from "@moq/signals";
import { target } from "./audio/latency";
import { type Delay, Sync } from "./sync";

// Effects in @moq/signals flush on a microtask, so let pending updates drain before asserting.
Expand Down Expand Up @@ -117,6 +118,27 @@ describe("auto delay", () => {
sync.close();
});

it("caps an auto delay that a catalog advertises past the ceiling", async () => {
// An hour of advertised jitter and catalog delay, through the audio target the decoder builds.
const hour = Duration.secs(3600);
const audio = new Signal<Duration | undefined>(
target({ measured: Duration.millis(0), advertised: hour, frame: Duration.millis(20), delay: hour }),
);
const sync = new Sync();
sync.track({ target: audio });
await flush();
expect(sync.out.delay.peek()).toEqual(Duration.secs(10));
expect(sync.out.maxDelay.peek()).toEqual(Duration.secs(10));
sync.close();
});

it("takes a fixed delay past the ceiling literally, since the viewer chose it", async () => {
const sync = new Sync({ delay: Duration.secs(20) });
await flush();
expect(sync.out.delay.peek()).toEqual(Duration.secs(20));
sync.close();
});

it("takes a fixed delay literally, ignoring the measured targets", async () => {
const media = new Signal<Duration | undefined>(Duration.millis(250));
const sync = new Sync({ delay: Duration.millis(100) });
Expand Down
16 changes: 13 additions & 3 deletions js/watch/src/sync.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@ import { Derived, Effect, type Getter, getter, type Inputs, type Readonlys, read
* How far playback trails the live edge.
*
* `"auto"` (the default) holds the largest playout target the registered decoders measure from how
* late their frames arrive (see doc/concept/audio-jitter.md). A `Duration` fixes the delay at
* exactly that value and disables adaptation.
* late their frames arrive (see doc/concept/audio-jitter.md), at most 10 s. A `Duration` fixes the
* delay at exactly that value and disables adaptation.
*
* `"instant"` drops the buffer and the pacing together: nothing is held, and the wait before
* presenting a frame never sleeps, so it presents as soon as it exists. It also overrides
Expand All @@ -18,6 +18,13 @@ export type Delay = "instant" | "auto" | Duration;
/** The default {@link SyncInput.stretch}: how far live audio may trail the clock before it skips. @internal */
export const STRETCH = Duration.secs(2);

/**
* The most an `"auto"` delay holds. The publisher advertises the jitter and delay it is measured
* from, so the clock bounds them here, whatever the catalog says. Matches
* `moq_audio::playback::Input::LATENCY_MAX`, which bounds `moq play`'s `--delay`. @internal
*/
export const DELAY_MAX = Duration.secs(10);

/** Signals the sync reads. */
export type SyncInput = {
/** How far playback trails the live edge. See {@link Delay}. */
Expand Down Expand Up @@ -66,7 +73,7 @@ export type SyncOutput = {
reference: Signal<number | undefined>;

// The resolved delay from the live edge to the playhead: the configured number, or in "auto" the
// largest registered playout target. Zero for "instant".
// largest registered playout target, at most DELAY_MAX. Zero for "instant".
delay: Signal<Duration>;

// The media timestamp of the most recently received frame.
Expand Down Expand Up @@ -488,6 +495,9 @@ export class Sync {
for (const { props } of effect.get(this.#clock.parts)) {
delay = delay.max(effect.get(props.target) ?? Duration.ZERO);
}
// Capped once, on the resolved delay, so audio and video stay on one clock. The
// subscription's max delay follows it.
delay = delay.min(DELAY_MAX);
}

this.#clock.out.delay.set(delay);
Expand Down
9 changes: 5 additions & 4 deletions js/watch/src/video/decoder.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import { Effect, Signal } from "@moq/signals";
import { ringSamples } from "../audio/latency";
import { AudioRingBuffer } from "../audio/ring-buffer";
import type { Broadcast } from "../broadcast";
import { type Delay, Sync, Track } from "../sync";
import { DELAY_MAX, type Delay, Sync, Track } from "../sync";
import { Decoder, insertRecoveryPoint } from "./decoder";
import { Source } from "./source";

Expand Down Expand Up @@ -1643,16 +1643,17 @@ describe("Decoder with a warmup rendition", () => {
[{}, 1],
[{ warmup: Catalog.u53(0) }, 1],
[{ warmup: WARMUP }, 2],
// An hour of warmup reaches back only as far as the delay cap.
[{ warmup: Catalog.u53(3_600_000) }, 2],
] as const) {
const subscribe = spyOn(Moq.Track.Consumer.prototype, "subscribe");
const run = await refresh(fields);
try {
expect(run.playback.codecs).toBe(decoders);
const budgets = subscribe.mock.calls.map(([options]) => options?.delay?.max);
// The max-delay decoder tolerates one sweep more; js/net merges the two upstream.
expect(budgets).toEqual(
decoders === 1 ? [Duration.millis(0)] : [Duration.millis(0), Duration.millis(WARMUP)],
);
const warmup = Duration.millis(fields.warmup ?? 0).min(DELAY_MAX);
expect(budgets).toEqual(decoders === 1 ? [Duration.millis(0)] : [Duration.millis(0), warmup]);
} finally {
subscribe.mockRestore();
await run.close();
Expand Down
5 changes: 3 additions & 2 deletions js/watch/src/video/decoder.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ import { rotateVideoDimensions } from "@moq/video";
import { base64ToBytes } from "../base64";
import { arrivedAt, type Counter, type Key, type Received, Tally } from "../feedback";
import { type Failure, failureMoved, nextMedia, subscribeMedia } from "../media";
import type { Sync, Track } from "../sync";
import { DELAY_MAX, type Sync, type Track } from "../sync";
import { type DecoderConfig, hardwareAcceleration, type PlaybackIdentity, playbackIdentity } from "./config";
import { caughtUp, renditionJitter, switchJitter } from "./playhead";
import { Recovery } from "./recovery";
Expand Down Expand Up @@ -1194,7 +1194,8 @@ class Rendition {

this.#signals.run((effect) => {
const config = effect.get(props.catalog)?.video?.renditions?.[props.track];
warmup.set(Duration.millis(config?.warmup ?? 0));
// Capped like the delay, so a catalog `warmup` cannot hold the viewer without bound.
warmup.set(Duration.millis(config?.warmup ?? 0).min(DELAY_MAX));
dual.set(!warmup.peek().isZero());
});
this.#signals.run((effect) => {
Expand Down
59 changes: 41 additions & 18 deletions quest/m1/audio-jitter-target/delay-cap.md
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
# [XS] A publisher's advertised delay cannot hold a browser viewer without bound
# [S] A publisher's advertised delay cannot hold a browser viewer without bound

## Goal

A publisher that advertises a huge `jitter` or catalog `delay` cannot hold a
`<moq-watch>` viewer's playback, or the media it buffers, past a fixed
ceiling. The browser caps the `"auto"` delay at the same 10 s `moq play`
uses, so a misbehaving publisher costs its viewers bounded latency and
memory, as AGENTS.md requires.
ceiling. The browser caps the resolved `"auto"` delay at 10 s, so a
misbehaving publisher costs its viewers bounded latency and memory, as
AGENTS.md requires.

## Plan

Expand All @@ -16,8 +16,8 @@ in m1 (rejected: m3 or later; not tracking it).

#8532 capped `moq play`'s clock at `moq_audio::playback::Input::LATENCY_MAX`
(10 s), which also bounds the catalog `delay` and `--delay` there. The
browser has no cap. In `"auto"`, `js/watch` `Sync#runDelay` holds the deepest
part's target, so an hour of advertised delay delays playback by an hour:
browser had no cap. In `"auto"`, `js/watch` `Sync#runDelay` holds the deepest
part's target, so an hour of advertised delay delayed playback by an hour:
- Audio's target is `max(measured, jitter) + frame + delay`
(`js/watch/src/audio/latency.ts` `target`).
- Video's target is `spread + delay`, floored by the rendition's `jitter`
Expand All @@ -26,15 +26,38 @@ part's target, so an hour of advertised delay delays playback by an hour:
The subscription's max delay (`delay + lookahead`) follows the same delay,
so the container consumer keeps that much encoded media.

Recommendations:

- Cap the resolved `"auto"` delay in `Sync#runDelay` at one exported
constant equal to `LATENCY_MAX`, rather than capping each term, so audio
and video stay on one clock.
- Leave a fixed delay alone: the viewer chose it, not the publisher.
- Test that a catalog advertising an hour of `jitter` and `delay` resolves
to the cap.
- Check that `moq_mux::sync` and `moq play` cap the same `"auto"` delay, and
say where the two constants are pinned together.

Public API: possibly one exported constant in `@moq/watch`. Wire: none.
Recommendations, as implemented:

- The resolved `"auto"` delay is capped once in `Sync#runDelay` at an internal
constant equal to `LATENCY_MAX`, rather than capping each term, so audio and
video stay on one clock.
- A fixed delay is left alone: the viewer chose it, not the publisher.
- A catalog advertising an hour of `jitter` and `delay` resolves to the cap.

Decided 2026-11-12 (coordinator, on the maintainer's behalf while away, from
the PR's recommendations):

- ✅ (a) cap `moq play`'s audio target sum, the held jitter target plus the
catalog delay, at `LATENCY_MAX` too, so one 10 s ceiling holds everywhere
(rejected: (b) raise the browser to 20 s; (c) keep the mismatch).
- ✅ (a) clamp catalog `warmup` at `LATENCY_MAX` where the browser reads it,
and in Rust for parity: `moq_mux::container::Playout`, `moq_video`'s
decode `Consumer`, and `moq play`'s video refresh, all through one
`moq_mux::container::WARMUP_MAX` (rejected: (b) reject the rendition; (c)
leave unbounded).

Findings:

- The constants are pinned together in one place only: the `play` feature's
test in `rs/moq-cli/src/play/args.rs` pins `DELAY_MAX` to
`moq_audio::playback::Input::LATENCY_MAX`. The browser's `DELAY_MAX` and
`moq-mux`'s `WARMUP_MAX` are pinned to it by comment alone, since no test
builds them together.
- The warmup cap also bounds each decoder's join trim (the browser's audio
terminal, `moq-audio`'s decoder, `moq-gst`'s Opus clip): the subscription
reaches back only the ceiling, so trimming further would hold playback
silent past it.

Public API: `moq_mux::container::WARMUP_MAX` (10 s) is exported so every
Rust reader of catalog `warmup` shares one ceiling; the browser's `DELAY_MAX`
stays internal. Wire: none.
50 changes: 49 additions & 1 deletion rs/moq-audio/src/decode/consumer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -219,7 +219,7 @@ impl Consumer {
}

let name = name.into();
let warmup = catalog.warmup.unwrap_or_default();
let warmup = catalog.warmup.unwrap_or_default().min(moq_mux::container::WARMUP_MAX);
let playout = moq_mux::container::Playout::new(options.delay, warmup);
let track = broadcast.track(hang::catalog::Reference::track(catalog, &name))?;
// The whole delay from the start, as the container keeps it: the transport enforces the
Expand Down Expand Up @@ -1365,6 +1365,54 @@ mod tests {
assert_eq!(stamps, expected, "the burst reached the decoder whole");
}

#[tokio::test(start_paused = true)]
async fn the_join_trim_stops_at_the_warmup_ceiling() {
let broadcast = moq_net::broadcast::Info::new().produce();
let track = broadcast
.create_track("audio", hang::container::track_info(hang::catalog::PRIORITY.audio))
.unwrap();
let mut catalog = hang::catalog::AudioConfig::new(hang::catalog::AudioCodec::Pcm, 48_000, 1);
catalog.warmup = Some(Duration::from_secs(3600));
let mut producer = moq_mux::container::Producer::new(
track,
moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Audio),
);
producer.seek(1).unwrap();
let ceiling = moq_mux::container::WARMUP_MAX.as_micros() as u64;
// Contiguous, so no gap re-arms the warmup.
for micros in (0..=ceiling).step_by(20_000) {
producer
.write(moq_mux::container::Frame {
timestamp: Timestamp::from_micros(micros).unwrap(),
duration: None,
payload: vec![0; 960 * size_of::<f32>()].into(),
keyframe: true,
})
.unwrap();
producer.cut(None).unwrap();
}
let _finished = producer.finish().unwrap();
let options = Options {
delay: Delay {
max: Some(moq_mux::container::WARMUP_MAX),
..Delay::default()
},
..Options::new()
};
let mut consumer = Consumer::new(&broadcast.consume(), &catalog, "audio", options)
.await
.unwrap();
let mut stamps = Vec::new();
while let Some(frame) = next_frame(&mut consumer).await.unwrap() {
stamps.push(frame.timestamp.as_micros());
}
assert_eq!(
stamps,
[ceiling as u128],
"an hour of warmup trims only up to the ceiling"
);
}

#[tokio::test(start_paused = true)]
async fn warmup_trims_the_join_without_trimming_groups_or_declared_returns() {
let (stamps, frames) = warmup_output(true).await;
Expand Down
Loading