From da60060e9cb43f59a66a284d4a8998b115bca521 Mon Sep 17 00:00:00 2001 From: Jebin Einstein E Date: Thu, 19 Nov 2026 20:22:19 +0530 Subject: [PATCH 1/5] fix(player): Play backwards starts on real hardware, and says why when it cannot Play backwards logged that it had started and then nothing moved. Reported in production; in headless Chromium, which decodes in software, it worked. The likely difference is the GPU: hardware decoders keep a small pool of frames and allow only a few decoders at once, and reverse playback was holding 24 of the decoder's frames per reader with up to three decoders open (the paused forward player's and two readers'). Either can leave a decoder waiting for ever, which is a first stretch that never arrives. - Readers now hold screen-sized copies: each frame is drawn (with the file's rotation) and the decoder's own frame released at once. Only the frames that will be kept are converted -- which ones comes from the packets' timestamps -- since converting every decoded frame cost more than decoding (0.3 picture changes a second instead of 27). - The forward player's decoder is released when reverse starts, and the two readers never decode at the same time. - Starting waits at most 8s for the first frames, then stops and says "This video could not be decoded backwards here." instead of hanging. - At the very start of a video -- where it lands after opening a file -- Play backwards now plays from the end, rather than doing nothing. - When it cannot play backwards (live stream, camera, YouTube), it says so. - Each stretch decoded is logged, start and finish, so a hang shows where. In headless: 27 picture changes a second at 1x, within 0.03s of the clock; stopping still hands the position to forward playback; a decoder that never delivers gives up after 8s with the message and the position kept; a click at 0:00 plays from the end. Real-GPU behaviour is untested here. Co-Authored-By: Claude Opus 5.5 --- assets/js/core/Player.js | 6 +- assets/js/core/playback/ReversePlayback.js | 127 ++++++++++++++---- assets/js/processing/shared/BackwardFrames.js | 52 ++++++- assets/js/ui/player/PlayerBindings.js | 5 +- assets/js/ui/player/PlayerKeyboard.js | 7 +- 5 files changed, 162 insertions(+), 35 deletions(-) diff --git a/assets/js/core/Player.js b/assets/js/core/Player.js index a55bb6b5..b2ac276c 100644 --- a/assets/js/core/Player.js +++ b/assets/js/core/Player.js @@ -463,7 +463,11 @@ export class CorePlayer { if (this.reverse?.active) { this.pause(); return; } togglePlayerPlay(this); } - /** Play backwards from here, or stop doing so. */ + /** + * Play backwards from here, or stop doing so. + * @returns {Promise<{started: boolean, reason?: string}|undefined>} when + * starting, whether it did, and if not, why, in words for the user + */ async toggleReverse() { if (this.reverse.active) return this.reverse.stop(); return this.reverse.start(); diff --git a/assets/js/core/playback/ReversePlayback.js b/assets/js/core/playback/ReversePlayback.js index da643be8..b07fbc37 100644 --- a/assets/js/core/playback/ReversePlayback.js +++ b/assets/js/core/playback/ReversePlayback.js @@ -16,6 +16,13 @@ import { BackwardFrameReader } from '../../processing/shared/BackwardFrames.js'; * stall at every key frame. Each holds fewer frames than the reader's default, * since together they hold twice as many. * + * What they hold are screen-sized copies, not the decoder's own frames: each + * decoded frame is drawn (with the file's rotation, as the forward path's + * CanvasSink applies it) and released straight away. A hardware decoder keeps + * only a few frames of its own and can stop producing more while they are + * held -- invisible in software decoding, where this was first measured, and + * exactly a reverse play that never starts on a real GPU. + * * Silent: reversed audio is not played. */ @@ -25,6 +32,10 @@ const FRAMES_PER_READER = 24; const PREFETCH_LEAD = 0.5; /** Two timestamps this close are the same instant. */ const EPSILON = 1e-6; +/** Closer than this to the start, there is nothing to play back from. */ +const AT_START = 0.05; +/** Give up on starting if the first frames take longer than this. */ +const START_TIMEOUT_MS = 8000; export class ReversePlayback { constructor(player) { @@ -32,29 +43,43 @@ export class ReversePlayback { this.active = false; this._raf = null; this._readers = null; - this._scratch = null; + this._canvas = null; } - /** Whether the loaded media can be played backwards. */ - get available() { + /** Why the loaded media cannot be played backwards, or null if it can. */ + get unavailableReason() { const p = this.player; - return !!p.videoTrack && !p.isLive && !p.isStreamMode && p.engine !== 'youtube'; + if (p.engine === 'youtube') return 'YouTube videos cannot be played backwards.'; + if (p.isLive || p.isStreamMode) return 'A live stream cannot be played backwards.'; + if (!p.videoTrack) return 'There is no video to play backwards.'; + return null; } /** - * Start playing backwards from the current position. - * @returns {Promise} whether it started + * Start playing backwards from the current position -- or from the end, + * when at the start, where there is nothing behind to play. + * @returns {Promise<{started: boolean, reason?: string}>} a reason, meant + * for the user, when it did not start */ async start() { const p = this.player; - if (this.active || !this.available) return false; + if (this.active) return { started: true }; + const unavailable = this.unavailableReason; + if (unavailable) return { started: false, reason: unavailable }; if (p.isPlaying) p.pause(false); + // The forward pipeline's decoder is let go: a GPU allows only a few + // hardware decoders at once, and a third one opening behind the paused + // player's can wait for ever. Stopping rebuilds it at the new position. + p.frames?.abandon?.(); this._start = Math.max(0, await p.videoTrack.getFirstTimestamp().catch(() => 0)); - const from = Math.min(p.currentTime || 0, p.duration || Infinity); - if (from <= this._start + EPSILON) return false; + const end = p.duration || 0; + let from = Math.min(p.currentTime || 0, end); + if (from <= this._start + AT_START) from = end; + if (!(from > this._start + AT_START)) return { started: false, reason: 'This video is too short to play backwards.' }; - this._readers = [0, 1].map(() => new BackwardFrameReader(p.videoTrack, { maxFrames: FRAMES_PER_READER })); + const toHeld = (sample) => this._copy(sample); + this._readers = [0, 1].map(() => new BackwardFrameReader(p.videoTrack, { maxFrames: FRAMES_PER_READER, toHeld })); this._shown = this._readers[0]; this._lastDrawn = null; this._origin = { position: from, wall: performance.now(), rate: p.playbackRate || 1 }; @@ -64,11 +89,28 @@ export class ReversePlayback { p.trigger?.('reversechange', { active: true }); Logger.log(`[Reverse] Playing backwards from ${from.toFixed(3)}s at ${this._origin.rate}x`); - // The first stretch is waited for, so the picture starts moving at once. - await this._shown.fill(from); - if (!this.active) return false; + // The first stretch is waited for, so the picture starts moving at + // once -- but not for ever: a decoder that never delivers would + // otherwise leave a button that seems to do nothing. + let timer; + const timedOut = new Promise(resolve => { timer = setTimeout(() => resolve('timeout'), START_TIMEOUT_MS); }); + let outcome; + try { + outcome = await Promise.race([this._fillLogged(this._shown, from).then(() => 'ok'), timedOut]); + } catch (error) { + Logger.warn('[Reverse] Could not decode the first stretch:', error); + outcome = 'error'; + } finally { + clearTimeout(timer); + } + if (!this.active) return { started: false }; + if (outcome !== 'ok' || this._shown.peek(from) === undefined) { + Logger.warn(`[Reverse] Gave up starting (${outcome})`); + await this.stop({ reposition: false }); + return { started: false, reason: 'This video could not be decoded backwards here.' }; + } this._raf = requestAnimationFrame(() => this._tick()); - return true; + return { started: true }; } /** @@ -133,10 +175,13 @@ export class ReversePlayback { this._shown = other; frame = this._shown.peek(t); } + // One decoder at a time, for the same reason the forward one was let + // go: never two fills at once. + const decoding = a.busy || b.busy; if (frame === undefined) { // Neither has it: fetched now, and the last frame stays up until // it arrives. Only happens if decoding falls behind. - if (!this._shown.busy) this._shown.fill(t); + if (!decoding) this._fillLogged(this._shown, t).catch(error => Logger.warn('[Reverse] Decoding failed:', error)); return; } if (frame && frame !== this._lastDrawn) { @@ -150,23 +195,49 @@ export class ReversePlayback { const oldest = this._shown.oldest; const lead = PREFETCH_LEAD * this._origin.rate; if (oldest !== null && oldest > this._start + EPSILON && t - oldest < lead - && !next.busy && next.peek(oldest - EPSILON) === undefined) { - next.fill(oldest - EPSILON).catch(error => Logger.warn('[Reverse] Prefetch failed:', error)); + && !decoding && next.peek(oldest - EPSILON) === undefined) { + this._fillLogged(next, oldest - EPSILON).catch(error => Logger.warn('[Reverse] Prefetch failed:', error)); } } - /** Through the player's own presentFrame, so filters and overlays apply. @private */ - _draw(frame) { + /** + * fill(), with a line before and after: where reverse playback is waiting, + * when it waits, is the first thing to know. + * @private + */ + async _fillLogged(reader, t) { + const began = performance.now(); + const decodedBefore = reader.decodedFrames; + Logger.log(`[Reverse] Decoding the stretch up to ${t.toFixed(3)}s`); + await reader.fill(t); + Logger.log(`[Reverse] Ready up to ${t.toFixed(3)}s: ${reader.held.length} frames held, ` + + `${reader.decodedFrames - decodedBefore} decoded, ${Math.round(performance.now() - began)}ms`); + } + + /** + * A decoded frame as a screen-sized image, so the decoder's frame can be + * released at once. Drawn with the file's rotation and flip, as the + * forward path's CanvasSink draws it. + * @private + */ + async _copy(sample) { const p = this.player; - if (!p.canvas) return; - if (!this._scratch || this._scratch.width !== p.canvas.width || this._scratch.height !== p.canvas.height) { - this._scratch = new OffscreenCanvas(p.canvas.width, p.canvas.height); - this._scratchCtx = this._scratch.getContext('2d'); + const width = p.canvas?.width || sample.displayWidth; + const height = p.canvas?.height || sample.displayHeight; + if (!this._canvas || this._canvas.width !== width || this._canvas.height !== height) { + this._canvas = new OffscreenCanvas(width, height); + this._canvasCtx = this._canvas.getContext('2d'); } - this._scratchCtx.clearRect(0, 0, this._scratch.width, this._scratch.height); - // Applies the file's rotation and flip, as the forward path's - // CanvasSink does. - frame.drawWithFit(this._scratchCtx, { fit: 'contain' }); - p.presentFrame(this._scratch, { clear: true }); + this._canvasCtx.clearRect(0, 0, width, height); + sample.drawWithFit(this._canvasCtx, { fit: 'contain' }); + // Synchronous, so two readers filling at once cannot interleave on + // the shared canvas. + const bitmap = this._canvas.transferToImageBitmap(); + return { timestamp: sample.timestamp, bitmap, close: () => bitmap.close() }; + } + + /** Through the player's own presentFrame, so filters and overlays apply. @private */ + _draw(frame) { + this.player.presentFrame(frame.bitmap, { clear: true }); } } diff --git a/assets/js/processing/shared/BackwardFrames.js b/assets/js/processing/shared/BackwardFrames.js index 408c7386..e700cc57 100644 --- a/assets/js/processing/shared/BackwardFrames.js +++ b/assets/js/processing/shared/BackwardFrames.js @@ -45,8 +45,15 @@ export class BackwardFrameReader { * @param {Object} [options.decoderOptions] - passed to VideoSampleSink * @param {number} [options.maxFrames] - hold fewer than the default, for * a caller that keeps two readers going at once + * @param {(sample: Object) => Promise<{timestamp: number, close: () => void}>} [options.toHeld] + * turn each decoded frame into what is held, so the decoder's own frame + * can be released at once. Hardware decoders keep only a small pool of + * frames and can stall while too many are held; a caller that only needs + * to draw a frame can hold a copy instead. sampleAt() needs the frames + * themselves, so leave this out to use it. */ - constructor(videoTrack, { decoderOptions, maxFrames = MAX_HELD_FRAMES } = {}) { + constructor(videoTrack, { decoderOptions, maxFrames = MAX_HELD_FRAMES, toHeld = null } = {}) { + this.toHeld = toHeld; this.track = videoTrack; this.sink = new MediaBunny.VideoSampleSink(videoTrack, decoderOptions); this.packets = new MediaBunny.EncodedPacketSink(videoTrack); @@ -156,7 +163,7 @@ export class BackwardFrameReader { * keeping the last `capacity` frames. * @private */ - async _fill(timestamp) { + async _fill(timestamp, { trustPackets = true } = {}) { const key = await this.packets.getKeyPacket(timestamp, { metadataOnly: true }) ?? await this.packets.getFirstPacket({ metadataOnly: true }); if (!key) { @@ -167,17 +174,39 @@ export class BackwardFrameReader { // Up to and including the frame at `timestamp`; a little past it, // since the range's end is exclusive. const end = timestamp + 2 * SAME_INSTANT; + // With a conversion, only the frames that will be kept are converted: + // which ones that is comes from the packets' timestamps, read without + // decoding. Converting every decoded frame and dropping most of them + // cost far more than the decoding did. + const keep = this.toHeld && trustPackets ? await this._lastTimestamps(key, timestamp) : null; + let decodedInRange = 0; const kept = []; for await (const sample of this.sink.samples(key.timestamp, end)) { this.decodedFrames++; if (sample.timestamp > timestamp + SAME_INSTANT) { sample.close(); continue; } - kept.push(sample); + decodedInRange++; + if (keep && !keep.has(Math.round(sample.timestamp * 1e6))) { sample.close(); continue; } + if (this.toHeld) { + try { + kept.push(await this.toHeld(sample)); + } finally { + sample.close(); + } + } else { + kept.push(sample); + } if (kept.length > this.capacity) kept.shift().close(); } if (this.closed) { for (const frame of kept) frame.close(); return; } + // Frames were decoded but none matched the packets' timestamps (a + // container whose timing the two disagree on): decode again, + // converting as it goes, rather than hold nothing and ask for ever. + if (keep && kept.length === 0 && decodedInRange > 0) { + return this._fill(timestamp, { trustPackets: false }); + } // Swapped in whole, so the old stretch stays drawable until the new // one is ready. const previous = this.held; @@ -185,4 +214,21 @@ export class BackwardFrameReader { this.heldUntil = timestamp; for (const frame of previous) frame.close(); } + + /** + * The timestamps of the last `capacity` frames from `key` up to + * `timestamp`, in microseconds, read from the packets alone. + * @private + */ + async _lastTimestamps(key, timestamp) { + const stamps = []; + for await (const packet of this.packets.packets(key, undefined, { metadataOnly: true })) { + // Decode order runs a little ahead of presentation order (B-frames), + // so read a couple of seconds past before stopping. + if (packet.timestamp > timestamp + 2) break; + if (packet.timestamp <= timestamp + SAME_INSTANT) stamps.push(packet.timestamp); + } + stamps.sort((a, b) => a - b); + return new Set(stamps.slice(-this.capacity).map(t => Math.round(t * 1e6))); + } } diff --git a/assets/js/ui/player/PlayerBindings.js b/assets/js/ui/player/PlayerBindings.js index 73730293..1a61a726 100644 --- a/assets/js/ui/player/PlayerBindings.js +++ b/assets/js/ui/player/PlayerBindings.js @@ -195,9 +195,10 @@ export function attachPlayerBindings(player) { player.ui.speedValue.textContent = `${player.playbackRate.toFixed(2)}x`; if (player.ui.reverseBtn) { - player.ui.reverseBtn.addEventListener('click', () => { - player.toggleReverse(); + player.ui.reverseBtn.addEventListener('click', async () => { player.ui.reverseBtn.blur(); + const result = await player.toggleReverse(); + if (result?.reason) Toast.show(result.reason, 3500); }); } } diff --git a/assets/js/ui/player/PlayerKeyboard.js b/assets/js/ui/player/PlayerKeyboard.js index 7fce507f..3f51bad0 100644 --- a/assets/js/ui/player/PlayerKeyboard.js +++ b/assets/js/ui/player/PlayerKeyboard.js @@ -1,3 +1,4 @@ +import { Toast } from '../../shared/utils/Toast.js'; export class PlayerKeyboard { constructor(player) { this.player = player; @@ -29,7 +30,11 @@ export class PlayerKeyboard { case 'j': e.preventDefault(); // Shift+J plays backwards (and stops doing so); J alone steps back. - if (e.shiftKey) p.toggleReverse(); + if (e.shiftKey) { + p.toggleReverse().then(result => { + if (result?.reason) Toast.show(result.reason, 3500); + }); + } else p._requestSeek(p.currentTime - 10); break; case 'l': From 2be85124f49798583ab175e16e8c28288312b52f Mon Sep 17 00:00:00 2001 From: Jebin Einstein E Date: Thu, 19 Nov 2026 20:31:01 +0530 Subject: [PATCH 2/5] feat(player): playing backwards plays the sound backwards too Reverse playback was silent. ReverseAudio (in ReversePlayback.js) decodes the selected audio track backwards in 2s windows: each decoded forwards with decodeTrackRange (split out of decodeAudioRange so it can take an open track), its samples reversed, and scheduled straight after the last through the player's own audio graph, so volume, mute and the equaliser apply and the speed changes the pitch, as going forwards. The next window is decoded once the current one has a second left. With sound, the picture follows the AudioContext clock, as forward playback does; without any, or if the audio cannot be decoded or gives out partway, the wall clock keeps time, taking over from wherever playback had got to. Measured on a clip with a click at each whole second, played backwards from 9.5s: the clicks were heard in descending order with the reverse clock reading 8.98, 7.98 ... 0.99s -- within 18ms -- and the picture stayed within 0.033s of the same clock. Co-Authored-By: Claude Opus 5.5 --- assets/js/core/playback/ReversePlayback.js | 154 +++++++++++++++++++- assets/js/processing/shared/AudioHelpers.js | 73 ++++++---- 2 files changed, 195 insertions(+), 32 deletions(-) diff --git a/assets/js/core/playback/ReversePlayback.js b/assets/js/core/playback/ReversePlayback.js index b07fbc37..72ec3aeb 100644 --- a/assets/js/core/playback/ReversePlayback.js +++ b/assets/js/core/playback/ReversePlayback.js @@ -1,5 +1,7 @@ import { Logger } from '../../shared/utils/Logger.js'; import { BackwardFrameReader } from '../../processing/shared/BackwardFrames.js'; +import { decodeTrackRange } from '../../processing/shared/AudioHelpers.js'; +import { initPlayerAudio } from '../audio/AudioEngine.js'; /** * ReversePlayback - plays the loaded video backwards, from wherever it is. @@ -23,7 +25,10 @@ import { BackwardFrameReader } from '../../processing/shared/BackwardFrames.js'; * held -- invisible in software decoding, where this was first measured, and * exactly a reverse play that never starts on a real GPU. * - * Silent: reversed audio is not played. + * The sound plays reversed too (ReverseAudio, below), and when it does, the + * picture follows its clock -- the AudioContext's -- as forward playback's + * does, so the two stay together. Without sound, or if the audio cannot be + * decoded, the picture keeps time by the wall clock. */ /** Frames each reader holds; two readers run at once. */ @@ -36,6 +41,118 @@ const EPSILON = 1e-6; const AT_START = 0.05; /** Give up on starting if the first frames take longer than this. */ const START_TIMEOUT_MS = 8000; +/** Reversed audio is decoded and played in windows this long, in seconds. */ +const AUDIO_WINDOW = 2; +/** Decode the next window once the current one has this little left to play. */ +const AUDIO_AHEAD = 1; + +/** + * The sound, backwards: decoded forwards a window at a time from the current + * position back, each window's samples reversed and played straight after + * the last, through the player's own audio graph -- so volume, mute and the + * equaliser apply, and the speed changes the pitch, as it does going forwards. + */ +class ReverseAudio { + constructor(player, track, from, start, rate) { + this.player = player; + this.track = track; + this.from = from; + this.start = start; + this.rate = rate; + this.ctx = null; + this.ctxStart = null; + this.sources = new Set(); + this.nextHigh = from; + this.decoding = false; + this.failed = false; + } + + /** Graph up, context running, first window decoded. @returns {Promise} */ + async prepare() { + const p = this.player; + if (!p.audioContext) await initPlayerAudio(p); + this.ctx = p.audioContext; + if (!this.ctx) return false; + await this.ctx.resume().catch(() => {}); + this.first = await this._decodeNext(); + return !!this.first; + } + + /** Start the sound now; the clock runs from here. */ + begin() { + this.ctxStart = this.ctx.currentTime + 0.05; + this._schedule(this.first); + this.first = null; + } + + /** Where playback is, by the audio clock. */ + position() { + if (this.ctxStart === null) return this.from; + return this.from - Math.max(0, this.ctx.currentTime - this.ctxStart) * this.rate; + } + + /** Keep the next window decoded ahead of the one playing. */ + pump() { + if (this.decoding || this.failed || this.nextHigh <= this.start + EPSILON) return; + const slot = this.ctxStart + (this.from - this.nextHigh) / this.rate; + if (slot - this.ctx.currentTime > AUDIO_AHEAD) return; + this._decodeNext() + .then(window => { if (window) this._schedule(window); }) + .catch(error => { + this.failed = true; + Logger.warn('[Reverse] Audio decoding failed; carrying on without sound:', error); + }); + } + + stop() { + for (const source of this.sources) { + try { source.stop(); } catch { /* not started */ } + source.disconnect(); + } + this.sources.clear(); + this.stopped = true; + } + + /** The next window back, reversed. @private */ + async _decodeNext() { + if (this.stopped) return null; + this.decoding = true; + try { + const high = this.nextHigh; + const low = Math.max(this.start, high - AUDIO_WINDOW); + this.nextHigh = low; + const buffer = await decodeTrackRange(this.track, low, high); + if (!buffer || this.stopped) return null; + for (let ch = 0; ch < buffer.numberOfChannels; ch++) buffer.getChannelData(ch).reverse(); + return { buffer, high }; + } finally { + this.decoding = false; + } + } + + /** Play a reversed window in its slot, or the rest of it if late. @private */ + _schedule({ buffer, high }) { + if (this.stopped) return; + const p = this.player; + const source = this.ctx.createBufferSource(); + source.buffer = buffer; + source.playbackRate.value = this.rate; + source.connect(p.audioEqualizer?.isInitialized ? p.audioEqualizer.getInputNode() : p.mixNode); + // Heard when the picture shows it: started early by the output + // latency, as forward playback starts its buffers. + const slot = this.ctxStart + (this.from - high) / this.rate - (this.ctx.outputLatency || 0); + const now = this.ctx.currentTime; + if (slot >= now) { + source.start(slot); + } else { + const offset = (now - slot) * this.rate; + if (offset >= buffer.duration) return; + source.start(now, offset); + } + this.sources.add(source); + source.onended = () => this.sources.delete(source); + } +} export class ReversePlayback { constructor(player) { @@ -89,6 +206,15 @@ export class ReversePlayback { p.trigger?.('reversechange', { active: true }); Logger.log(`[Reverse] Playing backwards from ${from.toFixed(3)}s at ${this._origin.rate}x`); + // The sound, if there is any the player can decode. Its first window + // is decoded alongside the first stretch of picture. + this._audio = p.audioTrack && p.audioSink + ? new ReverseAudio(p, p.audioTrack, from, this._start, this._origin.rate) + : null; + const audioReady = this._audio + ? this._audio.prepare().catch(error => { Logger.warn('[Reverse] No reversed sound:', error); return false; }) + : Promise.resolve(false); + // The first stretch is waited for, so the picture starts moving at // once -- but not for ever: a decoder that never delivers would // otherwise leave a button that seems to do nothing. @@ -109,6 +235,17 @@ export class ReversePlayback { await this.stop({ reposition: false }); return { started: false, reason: 'This video could not be decoded backwards here.' }; } + // The sound gets the same patience the picture got, and no more. + const withSound = await Promise.race([audioReady, new Promise(r => setTimeout(() => r(false), START_TIMEOUT_MS))]); + if (!this.active) return { started: false }; + if (withSound) { + this._audio.begin(); + Logger.log('[Reverse] With sound, timed by the audio clock'); + } else { + this._audio?.stop(); + this._audio = null; + } + this._origin.wall = performance.now(); this._raf = requestAnimationFrame(() => this._tick()); return { started: true }; } @@ -124,6 +261,8 @@ export class ReversePlayback { this.active = false; cancelAnimationFrame(this._raf); this._raf = null; + this._audio?.stop(); + this._audio = null; for (const reader of this._readers ?? []) reader.close(); this._readers = null; p._updatePlayPauseUI(); @@ -142,9 +281,20 @@ export class ReversePlayback { _tick() { if (!this.active) return; const p = this.player; + if (this._audio?.failed) { + // The sound gave out: the wall clock takes over from where it was. + this._origin = { position: this.position, wall: performance.now(), rate: this._origin.rate }; + this._audio.stop(); + this._audio = null; + } + // The audio clock when there is sound, as forward playback keeps + // time; otherwise the wall clock. const { position, wall, rate } = this._origin; - const t = Math.max(this._start, position - ((performance.now() - wall) / 1000) * rate); + const t = Math.max(this._start, this._audio + ? this._audio.position() + : position - ((performance.now() - wall) / 1000) * rate); this.position = t; + this._audio?.pump(); this._show(t); diff --git a/assets/js/processing/shared/AudioHelpers.js b/assets/js/processing/shared/AudioHelpers.js index d57f2b67..d93b5630 100644 --- a/assets/js/processing/shared/AudioHelpers.js +++ b/assets/js/processing/shared/AudioHelpers.js @@ -74,38 +74,51 @@ export async function decodeAudioRange(blob, start = 0, end = Infinity) { try { const from = firstTimestamp + Math.max(0, start); const to = firstTimestamp + Math.min(end, (await track.computeDuration()) - firstTimestamp); - if (!(to > from)) return null; + return await decodeTrackRange(track, from, to); + } finally { + input.dispose(); + } +} - // Sized from the first decoded buffer rather than the container's - // figures: for HE-AAC the container can state the core rate, half of - // what the decoder puts out, and a buffer sized from that would be cut - // short and play at the wrong pitch. - let result = null; - let sampleRate = 0; +/** + * Decode `from`..`to` of an audio track already open, on the track's own + * timeline, into one AudioBuffer at the rate the decoder puts out. Only that + * range is decoded and held. + * @param {Object} track - a Mediabunny InputAudioTrack + * @param {number} from - seconds + * @param {number} to - seconds + * @returns {Promise} null when the range holds no audio + */ +export async function decodeTrackRange(track, from, to) { + if (!(to > from)) return null; + + // Sized from the first decoded buffer rather than the container's + // figures: for HE-AAC the container can state the core rate, half of + // what the decoder puts out, and a buffer sized from that would be cut + // short and play at the wrong pitch. + let result = null; + let sampleRate = 0; - // Each decoded buffer is copied to where it belongs in the range, and - // whatever lies outside the range -- the ends of the first and last - // buffers -- is cut off. - for await (const { buffer, timestamp } of new MediaBunny.AudioBufferSink(track).buffers(from, to)) { - if (!result) { - sampleRate = buffer.sampleRate; - result = new AudioBuffer({ - length: Math.max(1, Math.round((to - from) * sampleRate)), - sampleRate, - numberOfChannels: buffer.numberOfChannels, - }); - } - const offset = Math.round((timestamp - from) * sampleRate); - const skip = Math.max(0, -offset); - const count = Math.min(buffer.length - skip, result.length - Math.max(0, offset)); - if (count <= 0) continue; - for (let ch = 0; ch < result.numberOfChannels; ch++) { - const source = buffer.getChannelData(Math.min(ch, buffer.numberOfChannels - 1)); - result.getChannelData(ch).set(source.subarray(skip, skip + count), Math.max(0, offset)); - } + // Each decoded buffer is copied to where it belongs in the range, and + // whatever lies outside the range -- the ends of the first and last + // buffers -- is cut off. + for await (const { buffer, timestamp } of new MediaBunny.AudioBufferSink(track).buffers(from, to)) { + if (!result) { + sampleRate = buffer.sampleRate; + result = new AudioBuffer({ + length: Math.max(1, Math.round((to - from) * sampleRate)), + sampleRate, + numberOfChannels: buffer.numberOfChannels, + }); + } + const offset = Math.round((timestamp - from) * sampleRate); + const skip = Math.max(0, -offset); + const count = Math.min(buffer.length - skip, result.length - Math.max(0, offset)); + if (count <= 0) continue; + for (let ch = 0; ch < result.numberOfChannels; ch++) { + const source = buffer.getChannelData(Math.min(ch, buffer.numberOfChannels - 1)); + result.getChannelData(ch).set(source.subarray(skip, skip + count), Math.max(0, offset)); } - return result; - } finally { - input.dispose(); } + return result; } From cf31ad5bd41785de104ca2a5252a3398344228fe Mon Sep 17 00:00:00 2001 From: Jebin Einstein E Date: Thu, 19 Nov 2026 20:47:37 +0530 Subject: [PATCH 3/5] feat(player): backwards is a playback direction, set from the control bar Playing backwards was a one-off action in the speed panel: pausing ended it, and the next Play went forwards again. It is now a direction, kept like speed: a button beside Play in the control bar (and Shift+J) switches which way Play goes, and Play, Pause, Space, seeks, scrubbing and speed changes all keep going that way. While backward is set the button is lit and the play arrow points back, so it is plain before pressing Play. - Switching while playing takes effect at once, from where playback is; while paused, the next Play goes the new way. - At 0:00, Play backwards plays from the end; reaching the start stops there, or with Loop on goes round again from the end. - Opening another file starts forward again. - When the media cannot be played backwards (live, camera, YouTube), the button says why instead of switching. The speed-panel button is gone. Checked with real clicks and keys: set while paused (button lit, arrow mirrored, nothing plays); Play from 0:00 ran back from 10.9s; Pause, then Play, carried on backwards; a click on the bar at 50% while reversing carried on back from 4.9s; 2x while reversing carried on at 2x; switching while reversing went straight to forward play; Space respected the direction; with Loop on the start went round from the end; a new file started forward. Co-Authored-By: Claude Opus 5.5 --- assets/css/player.css | 6 ++ assets/js/core/Player.js | 72 ++++++++++++++++--- assets/js/core/playback/PlaybackState.js | 5 +- assets/js/core/playback/ReversePlayback.js | 8 ++- assets/js/ui/player/PlayerBindings.js | 24 +++---- assets/js/ui/player/PlayerControlsView.js | 7 +- assets/js/ui/player/PlayerKeyboard.js | 2 +- assets/js/ui/player/PlayerUIUpdates.js | 12 ++++ public/assets/templates/player-templates.html | 10 ++- 9 files changed, 114 insertions(+), 32 deletions(-) diff --git a/assets/css/player.css b/assets/css/player.css index ee35b8c2..71601fef 100644 --- a/assets/css/player.css +++ b/assets/css/player.css @@ -487,6 +487,12 @@ button:focus-visible { color: var(--accent-primary); } +/* Backward is set: the play arrow points back, so it is plain which way Play + will go. The pause bars are symmetrical, so they read the same either way. */ +.jellyjump-btn.is-backward svg { + transform: scaleX(-1); +} + /* ----------------------------------------- Playlist Settings Menu ----------------------------------------- */ diff --git a/assets/js/core/Player.js b/assets/js/core/Player.js index b2ac276c..04cac1a4 100644 --- a/assets/js/core/Player.js +++ b/assets/js/core/Player.js @@ -278,6 +278,10 @@ export class CorePlayer { this.subtitles = new PlayerSubtitles(this); // Playing backwards runs beside the forward pipeline, not through it. this.reverse = new ReversePlayback(this); + // Which way Play goes. A setting, like speed: once backward, play, + // pause, seeks and speed changes all keep going backward until it is + // switched back, or another file is opened. + this.direction = 'forward'; this.loop = new PlayerLoopControl(this); this.thumbnails = new PlayerThumbnails(this); this.controlBar = new PlayerControlBar(this); @@ -440,8 +444,14 @@ export class CorePlayer { return resetPlayer(this); } async _cleanupForLoad() { - // Nothing to hand back to: the media is being replaced. + // Nothing to hand back to: the media is being replaced. A new file + // always starts forward, so it never surprises anyone by running + // backwards. await this.reverse?.stop({ reposition: false }); + if (this.direction !== 'forward') { + this.direction = 'forward'; + this._updatePlayPauseUI(); + } // Always, not only when leaving YouTube: this runs before every load, // and a playing iframe left visible would sit on top of the next video. // Suspended rather than destroyed — load() destroys it once it knows @@ -461,29 +471,53 @@ export class CorePlayer { async _handleVisibilityChange() { return handlePlayerVisibilityChange(this); } togglePlay() { if (this.reverse?.active) { this.pause(); return; } + if (this.direction === 'backward' && this.engine !== 'youtube') { this.play(); return; } togglePlayerPlay(this); } + /** - * Play backwards from here, or stop doing so. - * @returns {Promise<{started: boolean, reason?: string}|undefined>} when - * starting, whether it did, and if not, why, in words for the user + * Switch which way Play goes. While playing, the switch takes effect at + * once, from where playback is; while paused, the next Play goes the new + * way. + * @returns {Promise<{reason?: string}>} a reason, in words for the user, + * when playing backwards is not possible for this media */ - async toggleReverse() { - if (this.reverse.active) return this.reverse.stop(); - return this.reverse.start(); + async toggleDirection() { + if (this.direction === 'forward') { + const reason = this.reverse.unavailableReason; + if (reason) return { reason }; + const wasPlaying = this.isPlaying; + this.direction = 'backward'; + this._updatePlayPauseUI(); + if (wasPlaying) { + const result = await this._playBackward(); + return result.started ? {} : { reason: result.reason }; + } + return {}; + } + const wasReversing = this.reverse.active; + this.direction = 'forward'; + this._updatePlayPauseUI(); + if (wasReversing) { + await this.reverse.stop(); + await this.play(); + } + return {}; } _cycleSpeed(direction) { cyclePlayerSpeed(this, direction); } _stepFrame(direction) { stepPlayerFrame(this, direction); } async _seekTo(time) { - // A seek while playing backwards stops it; the seek decides the position. - if (this.reverse?.active) await this.reverse.stop({ reposition: false }); + // Playing backwards carries on from wherever the seek lands. + const wasReversing = !!this.reverse?.active; + if (wasReversing) await this.reverse.stop({ reposition: false }); if (this.engine === 'youtube') { this.youtube?.seek(time); this.currentTime = Math.max(0, time); this._updateProgress(); return; } - return seekPlayerTo(this, time); + await seekPlayerTo(this, time); + if (wasReversing) await this._playBackward(); } _requestSeek(time) { requestPlayerSeek(this, time); } _completeMedia() { completePlayerMedia(this); } @@ -503,11 +537,23 @@ export class CorePlayer { // ─── Play ──────────────────────────────────────────────────────────────────── async play() { if (this.engine === 'youtube') { this.youtube?.play(); return; } + if (this.direction === 'backward') return this._playBackward(); // Forwards from wherever playing backwards got to. if (this.reverse?.active) await this.reverse.stop(); return playPlayer(this); } + /** + * Play backwards, saying why if it cannot start. + * @private + */ + async _playBackward() { + if (this.reverse.active) return { started: true }; + const result = await this.reverse.start(); + if (!result.started && result.reason) this.trigger('reversefailed', { reason: result.reason }); + return result; + } + // ─── Pause ─────────────────────────────────────────────────────────────────── pause(showOverlay = true) { if (this.reverse?.active) { @@ -531,7 +577,11 @@ export class CorePlayer { this.youtube?.setRate(rate); return; } - return setPlayerPlaybackRate(this, rate); + // Playing backwards picks the new speed up from where it is. + const wasReversing = !!this.reverse?.active; + if (wasReversing) await this.reverse.stop({ reposition: false }); + await setPlayerPlaybackRate(this, rate); + if (wasReversing) await this._playBackward(); } // ─── Volume / Mute ─────────────────────────────────────────────────────────── diff --git a/assets/js/core/playback/PlaybackState.js b/assets/js/core/playback/PlaybackState.js index 5dfd7a6b..c945fe58 100644 --- a/assets/js/core/playback/PlaybackState.js +++ b/assets/js/core/playback/PlaybackState.js @@ -218,9 +218,10 @@ export function playerSeek(player, e) { export function playerScrubStart(player, e) { player.isScrubbing = true; - player.scrubWasPlaying = player.isPlaying; + // Playing backwards counts as playing: it resumes, backwards, on release. + player.scrubWasPlaying = player.isPlaying || !!player.reverse?.active; - if (player.isPlaying) player.pause(false); + if (player.scrubWasPlaying) player.pause(false); // Wake the context pause() just suspended while we are guaranteed to be // inside the input event (scrub start is always user input), so the diff --git a/assets/js/core/playback/ReversePlayback.js b/assets/js/core/playback/ReversePlayback.js index 72ec3aeb..d99920da 100644 --- a/assets/js/core/playback/ReversePlayback.js +++ b/assets/js/core/playback/ReversePlayback.js @@ -305,7 +305,13 @@ export class ReversePlayback { if (p.subtitles?.enabled) p._renderSubtitles(t); if (t <= this._start + EPSILON) { - this.stop(); + if (p.loopMode === 'one') { + // Looping, backwards: round again from the end, as forward + // looping goes round again from the start. + this.stop({ reposition: false }).then(() => this.start()); + } else { + this.stop(); + } return; } this._raf = requestAnimationFrame(() => this._tick()); diff --git a/assets/js/ui/player/PlayerBindings.js b/assets/js/ui/player/PlayerBindings.js index 1a61a726..b8454a5d 100644 --- a/assets/js/ui/player/PlayerBindings.js +++ b/assets/js/ui/player/PlayerBindings.js @@ -194,13 +194,15 @@ export function attachPlayerBindings(player) { player.ui.speedSlider.value = player.playbackRate; player.ui.speedValue.textContent = `${player.playbackRate.toFixed(2)}x`; - if (player.ui.reverseBtn) { - player.ui.reverseBtn.addEventListener('click', async () => { - player.ui.reverseBtn.blur(); - const result = await player.toggleReverse(); - if (result?.reason) Toast.show(result.reason, 3500); - }); - } + } + + if (player.ui.directionBtn) { + player.ui.directionBtn.addEventListener('click', async (e) => { + e.stopPropagation(); + player.ui.directionBtn.blur(); + const result = await player.toggleDirection(); + if (result?.reason) Toast.show(result.reason, 3500); + }); } if (player.config.controls.fullscreen) { @@ -508,10 +510,6 @@ export function attachPlayerBindings(player) { player._events = {}; // After the event list exists; registered any earlier, it would be wiped. - if (player.ui.reverseBtn) { - player.on('reversechange', ({ active }) => { - player.ui.reverseBtn.textContent = active ? '■ Stop playing backwards' : '◀ Play backwards'; - player.ui.reverseBtn.classList.toggle('active', active); - }); - } + // Play pressed while backward is set, and backwards could not start. + player.on('reversefailed', ({ reason }) => Toast.show(reason, 3500)); } diff --git a/assets/js/ui/player/PlayerControlsView.js b/assets/js/ui/player/PlayerControlsView.js index 1d1e44eb..245c19d8 100644 --- a/assets/js/ui/player/PlayerControlsView.js +++ b/assets/js/ui/player/PlayerControlsView.js @@ -26,7 +26,10 @@ export function createPlayerControls(player) { player.ui.playOverlay.style.display = 'none'; } - if (player.config.controls.playPause) player.ui.playBtn = q('#mb-play-btn'); + if (player.config.controls.playPause) { + player.ui.playBtn = q('#mb-play-btn'); + player.ui.directionBtn = q('#mb-direction-btn'); + } if (player.config.controls.navigation) { player.ui.prevBtn = q('#mb-prev-btn'); player.ui.nextBtn = q('#mb-next-btn'); @@ -77,7 +80,7 @@ function initSpeedPanel(player) { player.ui.speedSlider = q('#mb-speed-slider'); player.ui.speedValue = q('#mb-speed-value'); player.ui.resetSpeedBtn = q('#mb-reset-speed-btn'); - player.ui.reverseBtn = q('#mb-reverse-btn'); + player.ui.closeSpeedPanelBtn = q('.jellyjump-speed-panel .jellyjump-close-btn'); } diff --git a/assets/js/ui/player/PlayerKeyboard.js b/assets/js/ui/player/PlayerKeyboard.js index 3f51bad0..1db9493f 100644 --- a/assets/js/ui/player/PlayerKeyboard.js +++ b/assets/js/ui/player/PlayerKeyboard.js @@ -31,7 +31,7 @@ export class PlayerKeyboard { e.preventDefault(); // Shift+J plays backwards (and stops doing so); J alone steps back. if (e.shiftKey) { - p.toggleReverse().then(result => { + p.toggleDirection().then(result => { if (result?.reason) Toast.show(result.reason, 3500); }); } diff --git a/assets/js/ui/player/PlayerUIUpdates.js b/assets/js/ui/player/PlayerUIUpdates.js index 6454bf31..778a9689 100644 --- a/assets/js/ui/player/PlayerUIUpdates.js +++ b/assets/js/ui/player/PlayerUIUpdates.js @@ -4,6 +4,18 @@ import { Logger } from '../../shared/utils/Logger.js'; export function updatePlayerPlayPauseUI(player) { if (!player.ui.playBtn) return; + // Which way Play goes: the direction button lit, and the play arrow + // pointing back, so it is plain before pressing Play. + const backward = player.direction === 'backward'; + player.ui.playBtn.classList.toggle('is-backward', backward); + if (player.ui.directionBtn) { + player.ui.directionBtn.classList.toggle('active', backward); + player.ui.directionBtn.setAttribute('aria-pressed', String(backward)); + const label = backward ? 'Play forwards' : 'Play backwards'; + player.ui.directionBtn.setAttribute('aria-label', label); + player.ui.directionBtn.title = `${label} (Shift+J)`; + } + const use = player.ui.playBtn.querySelector('use'); // Playing backwards counts as playing: the button stops it. if (player.isPlaying || player.reverse?.active) { diff --git a/public/assets/templates/player-templates.html b/public/assets/templates/player-templates.html index fe771a75..8d2d0fb5 100644 --- a/public/assets/templates/player-templates.html +++ b/public/assets/templates/player-templates.html @@ -51,6 +51,13 @@

+ @@ -439,7 +445,7 @@

Playback

  • Space / K Play/Pause
  • J / L -/+ 10s
  • -
  • Shift+J Play backwards
  • +
  • Shift+J Play forwards / backwards
  • ← / → -/+ 5s
  • 0-9 Seek 0-90%
  • Home / End Start/End
  • From 52377a7199c4df70550855a4ec81204e5885b4e5 Mon Sep 17 00:00:00 2001 From: Jebin Einstein E Date: Thu, 19 Nov 2026 21:12:08 +0530 Subject: [PATCH 4/5] feat(player): hide the direction button where playing backwards cannot work A live stream keeps moving, so there is no fixed past to run back through; the camera, YouTube and media with no picture cannot be played backwards either. The button used to show and explain itself when pressed; it now hides for those, and shows again for anything that can, decided once a load has finished and it is known what was loaded. `hidden` needed its own rule to win over .jellyjump-btn's display: flex. What can, checked from another origin with CORS and byte ranges, as an internet URL is served: a WebM by URL and a VOD HLS stream (fMP4 segments) both play backwards with sound, at 30 pictures a second -- the same count as forward playback of the same URL -- and with 150ms latency at 4 Mbps the only difference was HLS taking 0.47s to start. A live HLS playlist hides the button. Co-Authored-By: Claude Opus 5.5 --- assets/css/player.css | 5 +++++ assets/js/core/playback/MediaLifecycle.js | 5 +++++ assets/js/ui/player/PlayerUIUpdates.js | 4 ++++ 3 files changed, 14 insertions(+) diff --git a/assets/css/player.css b/assets/css/player.css index 71601fef..9ee9b6fc 100644 --- a/assets/css/player.css +++ b/assets/css/player.css @@ -487,6 +487,11 @@ button:focus-visible { color: var(--accent-primary); } +/* `hidden` has to win over the display: flex above. */ +.jellyjump-btn[hidden] { + display: none; +} + /* Backward is set: the play arrow points back, so it is plain which way Play will go. The pause bars are symmetrical, so they read the same either way. */ .jellyjump-btn.is-backward svg { diff --git a/assets/js/core/playback/MediaLifecycle.js b/assets/js/core/playback/MediaLifecycle.js index 5795cc8e..4c226777 100644 --- a/assets/js/core/playback/MediaLifecycle.js +++ b/assets/js/core/playback/MediaLifecycle.js @@ -631,6 +631,8 @@ export async function loadPlayerMedia(player, url, autoplay = false, videoId = n player.currentVideoId = videoId || url; player._setLoading(true); await loadPlayerYouTube(player, youtube, autoplay); + // The direction button hides for YouTube. + player._updatePlayPauseUI(); Logger.log('Media loaded successfully (youtube)'); return; } @@ -655,6 +657,9 @@ export async function loadPlayerMedia(player, url, autoplay = false, videoId = n player._updateSubtitleMenu(); if (!player.isLive) player._setLoading(false); + // Now that it is known whether this is live and has a picture, the + // direction button shows or hides to match. + player._updatePlayPauseUI(); Logger.log('Media loaded successfully'); } catch (error) { diff --git a/assets/js/ui/player/PlayerUIUpdates.js b/assets/js/ui/player/PlayerUIUpdates.js index 778a9689..6ba3f96f 100644 --- a/assets/js/ui/player/PlayerUIUpdates.js +++ b/assets/js/ui/player/PlayerUIUpdates.js @@ -9,6 +9,10 @@ export function updatePlayerPlayPauseUI(player) { const backward = player.direction === 'backward'; player.ui.playBtn.classList.toggle('is-backward', backward); if (player.ui.directionBtn) { + // Hidden where playing backwards cannot work: a live stream (it + // keeps moving, so there is no fixed past to run back through), the + // camera, YouTube, or something with no picture. + player.ui.directionBtn.hidden = !!player.reverse?.unavailableReason; player.ui.directionBtn.classList.toggle('active', backward); player.ui.directionBtn.setAttribute('aria-pressed', String(backward)); const label = backward ? 'Play forwards' : 'Play backwards'; From 3b19568875aa856ba27a27b5db15251d759fcaaa Mon Sep 17 00:00:00 2001 From: Jebin Einstein E Date: Thu, 19 Nov 2026 21:21:55 +0530 Subject: [PATCH 5/5] perf(reverse): the reverse export decodes ahead, and holds copies, not decoder frames The reverse export used the same backward reader as playback, but not what playback learned since. It held up to 48 of the decoder's own frames -- what left reverse playback waiting for ever on a real GPU -- and decoded then encoded each stretch in turn, the decoder idle while the encoder worked. backwardSamples (new, in BackwardFrames.js) gives the export two readers taking turns, as playback has: the stretch before the current one is decoded while the current one is encoded, one decoder at a time. What they hold are copies in ordinary memory (copyOutOfDecoder), with pixel format, size, timing, rotation and colour space carried over; a frame whose format cannot be read directly is copied as RGBX. The output is unchanged: in 9 cases (key frames every 2, 5 and 10s; speeds 0.5, 1 and 2) every frame matched samplesAtTimestamps by timestamp and pixels, rotation survived the copy, and it runs in the media worker. In headless, where decoding and encoding share the CPU, it is about 20% slower than before (720p 1.5s -> 1.8s): the copies cost ~0.5s and decoding ahead wins back about half. Checked on real hardware, where decode and encode run on separate units, before merging. Co-Authored-By: Claude Opus 5.5 --- assets/js/processing/shared/BackwardFrames.js | 53 ++++++++++++++- assets/js/processing/speed/SpeedService.js | 64 ++++++++++++------- 2 files changed, 93 insertions(+), 24 deletions(-) diff --git a/assets/js/processing/shared/BackwardFrames.js b/assets/js/processing/shared/BackwardFrames.js index e700cc57..c21777a0 100644 --- a/assets/js/processing/shared/BackwardFrames.js +++ b/assets/js/processing/shared/BackwardFrames.js @@ -49,8 +49,9 @@ export class BackwardFrameReader { * turn each decoded frame into what is held, so the decoder's own frame * can be released at once. Hardware decoders keep only a small pool of * frames and can stall while too many are held; a caller that only needs - * to draw a frame can hold a copy instead. sampleAt() needs the frames - * themselves, so leave this out to use it. + * to draw a frame can hold a copy instead. sampleAt() and + * backwardSamples() clone what is held, so for them the copy must itself + * be a VideoSample. */ constructor(videoTrack, { decoderOptions, maxFrames = MAX_HELD_FRAMES, toHeld = null } = {}) { this.toHeld = toHeld; @@ -232,3 +233,51 @@ export class BackwardFrameReader { return new Set(stamps.slice(-this.capacity).map(t => Math.round(t * 1e6))); } } + +/** + * Frames for a falling sequence of timestamps, with the stretch before the + * current one decoded while the current one is being used -- for an export, + * where the caller is busy encoding each frame and the decoder would + * otherwise sit idle until it finished. Two readers take turns, as in reverse + * playback; only one ever decodes at a time. + * + * Yields a VideoSample (or null before the first frame) per timestamp, + * owned by the caller, exactly as samplesAtTimestamps does. + * + * @param {Object} videoTrack - a Mediabunny InputVideoTrack + * @param {Iterable|AsyncIterable} timestamps - falling + * @param {Object} [options] - passed to each BackwardFrameReader; `toHeld` + * must give VideoSamples (it is what gets cloned and handed out) + */ +export async function* backwardSamples(videoTrack, timestamps, options = {}) { + const readers = [new BackwardFrameReader(videoTrack, options), new BackwardFrameReader(videoTrack, options)]; + let current = readers[0]; + const failed = []; + // Nothing comes before the first frame, so nothing to decode ahead there. + const first = await videoTrack.getFirstTimestamp().catch(() => 0); + try { + for await (const t of timestamps) { + let other = current === readers[0] ? readers[1] : readers[0]; + if (current.peek(t) === undefined) { + // The stretch decoded ahead is waited for, never decoded over. + if (other.busy) await other.fill(t).catch(() => {}); + if (failed.length) throw failed[0]; + if (other.peek(t) !== undefined) { + [current, other] = [other, current]; + } else { + await current.fill(t); + } + } + // The stretch before this one, decoded while this one is used. + const oldest = current.oldest; + if (oldest !== null && oldest > first + SAME_INSTANT + && !other.busy && other.peek(oldest - SAME_INSTANT) === undefined) { + other.fill(oldest - SAME_INSTANT).catch(error => failed.push(error)); + } + const frame = current.peek(t); + yield frame ? frame.clone() : await current.sink.getSample(t); + } + } finally { + for (const reader of readers) reader.close(); + } +} diff --git a/assets/js/processing/speed/SpeedService.js b/assets/js/processing/speed/SpeedService.js index 184afc73..6f1ccd3a 100644 --- a/assets/js/processing/speed/SpeedService.js +++ b/assets/js/processing/speed/SpeedService.js @@ -4,7 +4,28 @@ import { AUDIO_BITRATE_BPS, createMediaBunnyInput } from '../shared/InputFactory import { addAudioSamples, stretchAudioBuffer } from '../shared/AudioHelpers.js'; import { cancellableProgress, cancelUnfinished, throwIfCancelled, isCancellation } from '../shared/Cancellation.js'; import { copyMetadataTags } from '../shared/MetadataTags.js'; -import { BackwardFrameReader } from '../shared/BackwardFrames.js'; +import { backwardSamples } from '../shared/BackwardFrames.js'; + +/** + * A decoded frame copied into ordinary memory, so the decoder's own frame can + * be released at once. Pixel format, size, timing, rotation and colour space + * carry over; a frame whose format cannot be read directly (some GPU frames) + * is copied as RGBX. + */ +async function copyOutOfDecoder(sample) { + const options = sample.format ? undefined : { format: 'RGBX' }; + const data = new Uint8Array(sample.allocationSize(options)); + await sample.copyTo(data, options); + return new MediaBunny.VideoSample(data, { + format: sample.format ?? 'RGBX', + codedWidth: sample.codedWidth, + codedHeight: sample.codedHeight, + timestamp: sample.timestamp, + duration: sample.duration, + rotation: sample.rotation, + colorSpace: sample.colorSpace, + }); +} async function processAudioWithSpeed(audioTrack, audioSource, speed = 1, signal) { const audioSink = new MediaBunny.AudioSampleSink(audioTrack); @@ -158,10 +179,13 @@ export async function reverseVideo({ source, includeAudio = false, speed = 1, on await copyMetadataTags(input, output); await output.start(); - // Frames come from BackwardFrameReader rather than samplesAtTimestamps: + // Frames come from backwardSamples rather than samplesAtTimestamps: // the latter re-decodes from the key frame for every frame when the - // timestamps fall, the reader decodes each stretch once. - const reader = new BackwardFrameReader(videoTrack); + // timestamps fall; this decodes each stretch once, and the next one + // back while the current one is being encoded. What it holds are + // copies in ordinary memory, not the decoder's own frames: a hardware + // decoder keeps only a few, and holding them is what left reverse + // playback waiting for ever on a real GPU. const reverseTimestampGenerator = (async function* () { for (let outputFrame = 0; outputFrame < totalOutputFrames; outputFrame++) { const outputProgress = outputFrame / totalOutputFrames; @@ -173,28 +197,24 @@ export async function reverseVideo({ source, includeAudio = false, speed = 1, on let outputTimestamp = 0; let frameCount = 0; - try { - for await (const sample of reader.samplesAt(reverseTimestampGenerator)) { - if (!sample) continue; - try { - sample.setTimestamp(outputTimestamp); - sample.setDuration(frameDuration); - await canvasSource.add(sample); - } finally { - sample.close(); - } - outputTimestamp += frameDuration; - frameCount++; + for await (const sample of backwardSamples(videoTrack, reverseTimestampGenerator, { maxFrames: 24, toHeld: copyOutOfDecoder })) { + if (!sample) continue; + try { + sample.setTimestamp(outputTimestamp); + sample.setDuration(frameDuration); + await canvasSource.add(sample); + } finally { + sample.close(); + } + outputTimestamp += frameDuration; + frameCount++; - if (onProgress) { - onProgress(frameCount / totalOutputFrames * 0.8); - } + if (onProgress) { + onProgress(frameCount / totalOutputFrames * 0.8); } - } finally { - reader.close(); } - Logger.log(`[MediaProcessor] Encoded ${frameCount} frames (${reader.decodedFrames} decoded)`); + Logger.log(`[MediaProcessor] Encoded ${frameCount} frames`); if (includeAudio && audioSource) { Logger.log('[MediaProcessor] Reversing audio...');