diff --git a/assets/css/player.css b/assets/css/player.css index ee35b8c2..9ee9b6fc 100644 --- a/assets/css/player.css +++ b/assets/css/player.css @@ -487,6 +487,17 @@ button:focus-visible { color: var(--accent-primary); } +/* `hidden` has to win over the display: flex above. */ +.jellyjump-btn[hidden] { + display: none; +} + +/* Backward is set: the play arrow points back, so it is plain which way Play + will go. The pause bars are symmetrical, so they read the same either way. */ +.jellyjump-btn.is-backward svg { + transform: scaleX(-1); +} + /* ----------------------------------------- Playlist Settings Menu ----------------------------------------- */ diff --git a/assets/js/core/Player.js b/assets/js/core/Player.js index a55bb6b5..04cac1a4 100644 --- a/assets/js/core/Player.js +++ b/assets/js/core/Player.js @@ -278,6 +278,10 @@ export class CorePlayer { this.subtitles = new PlayerSubtitles(this); // Playing backwards runs beside the forward pipeline, not through it. this.reverse = new ReversePlayback(this); + // Which way Play goes. A setting, like speed: once backward, play, + // pause, seeks and speed changes all keep going backward until it is + // switched back, or another file is opened. + this.direction = 'forward'; this.loop = new PlayerLoopControl(this); this.thumbnails = new PlayerThumbnails(this); this.controlBar = new PlayerControlBar(this); @@ -440,8 +444,14 @@ export class CorePlayer { return resetPlayer(this); } async _cleanupForLoad() { - // Nothing to hand back to: the media is being replaced. + // Nothing to hand back to: the media is being replaced. A new file + // always starts forward, so it never surprises anyone by running + // backwards. await this.reverse?.stop({ reposition: false }); + if (this.direction !== 'forward') { + this.direction = 'forward'; + this._updatePlayPauseUI(); + } // Always, not only when leaving YouTube: this runs before every load, // and a playing iframe left visible would sit on top of the next video. // Suspended rather than destroyed — load() destroys it once it knows @@ -461,25 +471,53 @@ export class CorePlayer { async _handleVisibilityChange() { return handlePlayerVisibilityChange(this); } togglePlay() { if (this.reverse?.active) { this.pause(); return; } + if (this.direction === 'backward' && this.engine !== 'youtube') { this.play(); return; } togglePlayerPlay(this); } - /** Play backwards from here, or stop doing so. */ - async toggleReverse() { - if (this.reverse.active) return this.reverse.stop(); - return this.reverse.start(); + + /** + * Switch which way Play goes. While playing, the switch takes effect at + * once, from where playback is; while paused, the next Play goes the new + * way. + * @returns {Promise<{reason?: string}>} a reason, in words for the user, + * when playing backwards is not possible for this media + */ + async toggleDirection() { + if (this.direction === 'forward') { + const reason = this.reverse.unavailableReason; + if (reason) return { reason }; + const wasPlaying = this.isPlaying; + this.direction = 'backward'; + this._updatePlayPauseUI(); + if (wasPlaying) { + const result = await this._playBackward(); + return result.started ? {} : { reason: result.reason }; + } + return {}; + } + const wasReversing = this.reverse.active; + this.direction = 'forward'; + this._updatePlayPauseUI(); + if (wasReversing) { + await this.reverse.stop(); + await this.play(); + } + return {}; } _cycleSpeed(direction) { cyclePlayerSpeed(this, direction); } _stepFrame(direction) { stepPlayerFrame(this, direction); } async _seekTo(time) { - // A seek while playing backwards stops it; the seek decides the position. - if (this.reverse?.active) await this.reverse.stop({ reposition: false }); + // Playing backwards carries on from wherever the seek lands. + const wasReversing = !!this.reverse?.active; + if (wasReversing) await this.reverse.stop({ reposition: false }); if (this.engine === 'youtube') { this.youtube?.seek(time); this.currentTime = Math.max(0, time); this._updateProgress(); return; } - return seekPlayerTo(this, time); + await seekPlayerTo(this, time); + if (wasReversing) await this._playBackward(); } _requestSeek(time) { requestPlayerSeek(this, time); } _completeMedia() { completePlayerMedia(this); } @@ -499,11 +537,23 @@ export class CorePlayer { // ─── Play ──────────────────────────────────────────────────────────────────── async play() { if (this.engine === 'youtube') { this.youtube?.play(); return; } + if (this.direction === 'backward') return this._playBackward(); // Forwards from wherever playing backwards got to. if (this.reverse?.active) await this.reverse.stop(); return playPlayer(this); } + /** + * Play backwards, saying why if it cannot start. + * @private + */ + async _playBackward() { + if (this.reverse.active) return { started: true }; + const result = await this.reverse.start(); + if (!result.started && result.reason) this.trigger('reversefailed', { reason: result.reason }); + return result; + } + // ─── Pause ─────────────────────────────────────────────────────────────────── pause(showOverlay = true) { if (this.reverse?.active) { @@ -527,7 +577,11 @@ export class CorePlayer { this.youtube?.setRate(rate); return; } - return setPlayerPlaybackRate(this, rate); + // Playing backwards picks the new speed up from where it is. + const wasReversing = !!this.reverse?.active; + if (wasReversing) await this.reverse.stop({ reposition: false }); + await setPlayerPlaybackRate(this, rate); + if (wasReversing) await this._playBackward(); } // ─── Volume / Mute ─────────────────────────────────────────────────────────── diff --git a/assets/js/core/playback/MediaLifecycle.js b/assets/js/core/playback/MediaLifecycle.js index 5795cc8e..4c226777 100644 --- a/assets/js/core/playback/MediaLifecycle.js +++ b/assets/js/core/playback/MediaLifecycle.js @@ -631,6 +631,8 @@ export async function loadPlayerMedia(player, url, autoplay = false, videoId = n player.currentVideoId = videoId || url; player._setLoading(true); await loadPlayerYouTube(player, youtube, autoplay); + // The direction button hides for YouTube. + player._updatePlayPauseUI(); Logger.log('Media loaded successfully (youtube)'); return; } @@ -655,6 +657,9 @@ export async function loadPlayerMedia(player, url, autoplay = false, videoId = n player._updateSubtitleMenu(); if (!player.isLive) player._setLoading(false); + // Now that it is known whether this is live and has a picture, the + // direction button shows or hides to match. + player._updatePlayPauseUI(); Logger.log('Media loaded successfully'); } catch (error) { diff --git a/assets/js/core/playback/PlaybackState.js b/assets/js/core/playback/PlaybackState.js index 5dfd7a6b..c945fe58 100644 --- a/assets/js/core/playback/PlaybackState.js +++ b/assets/js/core/playback/PlaybackState.js @@ -218,9 +218,10 @@ export function playerSeek(player, e) { export function playerScrubStart(player, e) { player.isScrubbing = true; - player.scrubWasPlaying = player.isPlaying; + // Playing backwards counts as playing: it resumes, backwards, on release. + player.scrubWasPlaying = player.isPlaying || !!player.reverse?.active; - if (player.isPlaying) player.pause(false); + if (player.scrubWasPlaying) player.pause(false); // Wake the context pause() just suspended while we are guaranteed to be // inside the input event (scrub start is always user input), so the diff --git a/assets/js/core/playback/ReversePlayback.js b/assets/js/core/playback/ReversePlayback.js index da643be8..d99920da 100644 --- a/assets/js/core/playback/ReversePlayback.js +++ b/assets/js/core/playback/ReversePlayback.js @@ -1,5 +1,7 @@ import { Logger } from '../../shared/utils/Logger.js'; import { BackwardFrameReader } from '../../processing/shared/BackwardFrames.js'; +import { decodeTrackRange } from '../../processing/shared/AudioHelpers.js'; +import { initPlayerAudio } from '../audio/AudioEngine.js'; /** * ReversePlayback - plays the loaded video backwards, from wherever it is. @@ -16,7 +18,17 @@ import { BackwardFrameReader } from '../../processing/shared/BackwardFrames.js'; * stall at every key frame. Each holds fewer frames than the reader's default, * since together they hold twice as many. * - * Silent: reversed audio is not played. + * What they hold are screen-sized copies, not the decoder's own frames: each + * decoded frame is drawn (with the file's rotation, as the forward path's + * CanvasSink applies it) and released straight away. A hardware decoder keeps + * only a few frames of its own and can stop producing more while they are + * held -- invisible in software decoding, where this was first measured, and + * exactly a reverse play that never starts on a real GPU. + * + * The sound plays reversed too (ReverseAudio, below), and when it does, the + * picture follows its clock -- the AudioContext's -- as forward playback's + * does, so the two stay together. Without sound, or if the audio cannot be + * decoded, the picture keeps time by the wall clock. */ /** Frames each reader holds; two readers run at once. */ @@ -25,6 +37,122 @@ const FRAMES_PER_READER = 24; const PREFETCH_LEAD = 0.5; /** Two timestamps this close are the same instant. */ const EPSILON = 1e-6; +/** Closer than this to the start, there is nothing to play back from. */ +const AT_START = 0.05; +/** Give up on starting if the first frames take longer than this. */ +const START_TIMEOUT_MS = 8000; +/** Reversed audio is decoded and played in windows this long, in seconds. */ +const AUDIO_WINDOW = 2; +/** Decode the next window once the current one has this little left to play. */ +const AUDIO_AHEAD = 1; + +/** + * The sound, backwards: decoded forwards a window at a time from the current + * position back, each window's samples reversed and played straight after + * the last, through the player's own audio graph -- so volume, mute and the + * equaliser apply, and the speed changes the pitch, as it does going forwards. + */ +class ReverseAudio { + constructor(player, track, from, start, rate) { + this.player = player; + this.track = track; + this.from = from; + this.start = start; + this.rate = rate; + this.ctx = null; + this.ctxStart = null; + this.sources = new Set(); + this.nextHigh = from; + this.decoding = false; + this.failed = false; + } + + /** Graph up, context running, first window decoded. @returns {Promise} */ + async prepare() { + const p = this.player; + if (!p.audioContext) await initPlayerAudio(p); + this.ctx = p.audioContext; + if (!this.ctx) return false; + await this.ctx.resume().catch(() => {}); + this.first = await this._decodeNext(); + return !!this.first; + } + + /** Start the sound now; the clock runs from here. */ + begin() { + this.ctxStart = this.ctx.currentTime + 0.05; + this._schedule(this.first); + this.first = null; + } + + /** Where playback is, by the audio clock. */ + position() { + if (this.ctxStart === null) return this.from; + return this.from - Math.max(0, this.ctx.currentTime - this.ctxStart) * this.rate; + } + + /** Keep the next window decoded ahead of the one playing. */ + pump() { + if (this.decoding || this.failed || this.nextHigh <= this.start + EPSILON) return; + const slot = this.ctxStart + (this.from - this.nextHigh) / this.rate; + if (slot - this.ctx.currentTime > AUDIO_AHEAD) return; + this._decodeNext() + .then(window => { if (window) this._schedule(window); }) + .catch(error => { + this.failed = true; + Logger.warn('[Reverse] Audio decoding failed; carrying on without sound:', error); + }); + } + + stop() { + for (const source of this.sources) { + try { source.stop(); } catch { /* not started */ } + source.disconnect(); + } + this.sources.clear(); + this.stopped = true; + } + + /** The next window back, reversed. @private */ + async _decodeNext() { + if (this.stopped) return null; + this.decoding = true; + try { + const high = this.nextHigh; + const low = Math.max(this.start, high - AUDIO_WINDOW); + this.nextHigh = low; + const buffer = await decodeTrackRange(this.track, low, high); + if (!buffer || this.stopped) return null; + for (let ch = 0; ch < buffer.numberOfChannels; ch++) buffer.getChannelData(ch).reverse(); + return { buffer, high }; + } finally { + this.decoding = false; + } + } + + /** Play a reversed window in its slot, or the rest of it if late. @private */ + _schedule({ buffer, high }) { + if (this.stopped) return; + const p = this.player; + const source = this.ctx.createBufferSource(); + source.buffer = buffer; + source.playbackRate.value = this.rate; + source.connect(p.audioEqualizer?.isInitialized ? p.audioEqualizer.getInputNode() : p.mixNode); + // Heard when the picture shows it: started early by the output + // latency, as forward playback starts its buffers. + const slot = this.ctxStart + (this.from - high) / this.rate - (this.ctx.outputLatency || 0); + const now = this.ctx.currentTime; + if (slot >= now) { + source.start(slot); + } else { + const offset = (now - slot) * this.rate; + if (offset >= buffer.duration) return; + source.start(now, offset); + } + this.sources.add(source); + source.onended = () => this.sources.delete(source); + } +} export class ReversePlayback { constructor(player) { @@ -32,29 +160,43 @@ export class ReversePlayback { this.active = false; this._raf = null; this._readers = null; - this._scratch = null; + this._canvas = null; } - /** Whether the loaded media can be played backwards. */ - get available() { + /** Why the loaded media cannot be played backwards, or null if it can. */ + get unavailableReason() { const p = this.player; - return !!p.videoTrack && !p.isLive && !p.isStreamMode && p.engine !== 'youtube'; + if (p.engine === 'youtube') return 'YouTube videos cannot be played backwards.'; + if (p.isLive || p.isStreamMode) return 'A live stream cannot be played backwards.'; + if (!p.videoTrack) return 'There is no video to play backwards.'; + return null; } /** - * Start playing backwards from the current position. - * @returns {Promise} whether it started + * Start playing backwards from the current position -- or from the end, + * when at the start, where there is nothing behind to play. + * @returns {Promise<{started: boolean, reason?: string}>} a reason, meant + * for the user, when it did not start */ async start() { const p = this.player; - if (this.active || !this.available) return false; + if (this.active) return { started: true }; + const unavailable = this.unavailableReason; + if (unavailable) return { started: false, reason: unavailable }; if (p.isPlaying) p.pause(false); + // The forward pipeline's decoder is let go: a GPU allows only a few + // hardware decoders at once, and a third one opening behind the paused + // player's can wait for ever. Stopping rebuilds it at the new position. + p.frames?.abandon?.(); this._start = Math.max(0, await p.videoTrack.getFirstTimestamp().catch(() => 0)); - const from = Math.min(p.currentTime || 0, p.duration || Infinity); - if (from <= this._start + EPSILON) return false; + const end = p.duration || 0; + let from = Math.min(p.currentTime || 0, end); + if (from <= this._start + AT_START) from = end; + if (!(from > this._start + AT_START)) return { started: false, reason: 'This video is too short to play backwards.' }; - this._readers = [0, 1].map(() => new BackwardFrameReader(p.videoTrack, { maxFrames: FRAMES_PER_READER })); + const toHeld = (sample) => this._copy(sample); + this._readers = [0, 1].map(() => new BackwardFrameReader(p.videoTrack, { maxFrames: FRAMES_PER_READER, toHeld })); this._shown = this._readers[0]; this._lastDrawn = null; this._origin = { position: from, wall: performance.now(), rate: p.playbackRate || 1 }; @@ -64,11 +206,48 @@ export class ReversePlayback { p.trigger?.('reversechange', { active: true }); Logger.log(`[Reverse] Playing backwards from ${from.toFixed(3)}s at ${this._origin.rate}x`); - // The first stretch is waited for, so the picture starts moving at once. - await this._shown.fill(from); - if (!this.active) return false; + // The sound, if there is any the player can decode. Its first window + // is decoded alongside the first stretch of picture. + this._audio = p.audioTrack && p.audioSink + ? new ReverseAudio(p, p.audioTrack, from, this._start, this._origin.rate) + : null; + const audioReady = this._audio + ? this._audio.prepare().catch(error => { Logger.warn('[Reverse] No reversed sound:', error); return false; }) + : Promise.resolve(false); + + // The first stretch is waited for, so the picture starts moving at + // once -- but not for ever: a decoder that never delivers would + // otherwise leave a button that seems to do nothing. + let timer; + const timedOut = new Promise(resolve => { timer = setTimeout(() => resolve('timeout'), START_TIMEOUT_MS); }); + let outcome; + try { + outcome = await Promise.race([this._fillLogged(this._shown, from).then(() => 'ok'), timedOut]); + } catch (error) { + Logger.warn('[Reverse] Could not decode the first stretch:', error); + outcome = 'error'; + } finally { + clearTimeout(timer); + } + if (!this.active) return { started: false }; + if (outcome !== 'ok' || this._shown.peek(from) === undefined) { + Logger.warn(`[Reverse] Gave up starting (${outcome})`); + await this.stop({ reposition: false }); + return { started: false, reason: 'This video could not be decoded backwards here.' }; + } + // The sound gets the same patience the picture got, and no more. + const withSound = await Promise.race([audioReady, new Promise(r => setTimeout(() => r(false), START_TIMEOUT_MS))]); + if (!this.active) return { started: false }; + if (withSound) { + this._audio.begin(); + Logger.log('[Reverse] With sound, timed by the audio clock'); + } else { + this._audio?.stop(); + this._audio = null; + } + this._origin.wall = performance.now(); this._raf = requestAnimationFrame(() => this._tick()); - return true; + return { started: true }; } /** @@ -82,6 +261,8 @@ export class ReversePlayback { this.active = false; cancelAnimationFrame(this._raf); this._raf = null; + this._audio?.stop(); + this._audio = null; for (const reader of this._readers ?? []) reader.close(); this._readers = null; p._updatePlayPauseUI(); @@ -100,9 +281,20 @@ export class ReversePlayback { _tick() { if (!this.active) return; const p = this.player; + if (this._audio?.failed) { + // The sound gave out: the wall clock takes over from where it was. + this._origin = { position: this.position, wall: performance.now(), rate: this._origin.rate }; + this._audio.stop(); + this._audio = null; + } + // The audio clock when there is sound, as forward playback keeps + // time; otherwise the wall clock. const { position, wall, rate } = this._origin; - const t = Math.max(this._start, position - ((performance.now() - wall) / 1000) * rate); + const t = Math.max(this._start, this._audio + ? this._audio.position() + : position - ((performance.now() - wall) / 1000) * rate); this.position = t; + this._audio?.pump(); this._show(t); @@ -113,7 +305,13 @@ export class ReversePlayback { if (p.subtitles?.enabled) p._renderSubtitles(t); if (t <= this._start + EPSILON) { - this.stop(); + if (p.loopMode === 'one') { + // Looping, backwards: round again from the end, as forward + // looping goes round again from the start. + this.stop({ reposition: false }).then(() => this.start()); + } else { + this.stop(); + } return; } this._raf = requestAnimationFrame(() => this._tick()); @@ -133,10 +331,13 @@ export class ReversePlayback { this._shown = other; frame = this._shown.peek(t); } + // One decoder at a time, for the same reason the forward one was let + // go: never two fills at once. + const decoding = a.busy || b.busy; if (frame === undefined) { // Neither has it: fetched now, and the last frame stays up until // it arrives. Only happens if decoding falls behind. - if (!this._shown.busy) this._shown.fill(t); + if (!decoding) this._fillLogged(this._shown, t).catch(error => Logger.warn('[Reverse] Decoding failed:', error)); return; } if (frame && frame !== this._lastDrawn) { @@ -150,23 +351,49 @@ export class ReversePlayback { const oldest = this._shown.oldest; const lead = PREFETCH_LEAD * this._origin.rate; if (oldest !== null && oldest > this._start + EPSILON && t - oldest < lead - && !next.busy && next.peek(oldest - EPSILON) === undefined) { - next.fill(oldest - EPSILON).catch(error => Logger.warn('[Reverse] Prefetch failed:', error)); + && !decoding && next.peek(oldest - EPSILON) === undefined) { + this._fillLogged(next, oldest - EPSILON).catch(error => Logger.warn('[Reverse] Prefetch failed:', error)); } } - /** Through the player's own presentFrame, so filters and overlays apply. @private */ - _draw(frame) { + /** + * fill(), with a line before and after: where reverse playback is waiting, + * when it waits, is the first thing to know. + * @private + */ + async _fillLogged(reader, t) { + const began = performance.now(); + const decodedBefore = reader.decodedFrames; + Logger.log(`[Reverse] Decoding the stretch up to ${t.toFixed(3)}s`); + await reader.fill(t); + Logger.log(`[Reverse] Ready up to ${t.toFixed(3)}s: ${reader.held.length} frames held, ` + + `${reader.decodedFrames - decodedBefore} decoded, ${Math.round(performance.now() - began)}ms`); + } + + /** + * A decoded frame as a screen-sized image, so the decoder's frame can be + * released at once. Drawn with the file's rotation and flip, as the + * forward path's CanvasSink draws it. + * @private + */ + async _copy(sample) { const p = this.player; - if (!p.canvas) return; - if (!this._scratch || this._scratch.width !== p.canvas.width || this._scratch.height !== p.canvas.height) { - this._scratch = new OffscreenCanvas(p.canvas.width, p.canvas.height); - this._scratchCtx = this._scratch.getContext('2d'); + const width = p.canvas?.width || sample.displayWidth; + const height = p.canvas?.height || sample.displayHeight; + if (!this._canvas || this._canvas.width !== width || this._canvas.height !== height) { + this._canvas = new OffscreenCanvas(width, height); + this._canvasCtx = this._canvas.getContext('2d'); } - this._scratchCtx.clearRect(0, 0, this._scratch.width, this._scratch.height); - // Applies the file's rotation and flip, as the forward path's - // CanvasSink does. - frame.drawWithFit(this._scratchCtx, { fit: 'contain' }); - p.presentFrame(this._scratch, { clear: true }); + this._canvasCtx.clearRect(0, 0, width, height); + sample.drawWithFit(this._canvasCtx, { fit: 'contain' }); + // Synchronous, so two readers filling at once cannot interleave on + // the shared canvas. + const bitmap = this._canvas.transferToImageBitmap(); + return { timestamp: sample.timestamp, bitmap, close: () => bitmap.close() }; + } + + /** Through the player's own presentFrame, so filters and overlays apply. @private */ + _draw(frame) { + this.player.presentFrame(frame.bitmap, { clear: true }); } } diff --git a/assets/js/processing/shared/AudioHelpers.js b/assets/js/processing/shared/AudioHelpers.js index d57f2b67..d93b5630 100644 --- a/assets/js/processing/shared/AudioHelpers.js +++ b/assets/js/processing/shared/AudioHelpers.js @@ -74,38 +74,51 @@ export async function decodeAudioRange(blob, start = 0, end = Infinity) { try { const from = firstTimestamp + Math.max(0, start); const to = firstTimestamp + Math.min(end, (await track.computeDuration()) - firstTimestamp); - if (!(to > from)) return null; + return await decodeTrackRange(track, from, to); + } finally { + input.dispose(); + } +} - // Sized from the first decoded buffer rather than the container's - // figures: for HE-AAC the container can state the core rate, half of - // what the decoder puts out, and a buffer sized from that would be cut - // short and play at the wrong pitch. - let result = null; - let sampleRate = 0; +/** + * Decode `from`..`to` of an audio track already open, on the track's own + * timeline, into one AudioBuffer at the rate the decoder puts out. Only that + * range is decoded and held. + * @param {Object} track - a Mediabunny InputAudioTrack + * @param {number} from - seconds + * @param {number} to - seconds + * @returns {Promise} null when the range holds no audio + */ +export async function decodeTrackRange(track, from, to) { + if (!(to > from)) return null; + + // Sized from the first decoded buffer rather than the container's + // figures: for HE-AAC the container can state the core rate, half of + // what the decoder puts out, and a buffer sized from that would be cut + // short and play at the wrong pitch. + let result = null; + let sampleRate = 0; - // Each decoded buffer is copied to where it belongs in the range, and - // whatever lies outside the range -- the ends of the first and last - // buffers -- is cut off. - for await (const { buffer, timestamp } of new MediaBunny.AudioBufferSink(track).buffers(from, to)) { - if (!result) { - sampleRate = buffer.sampleRate; - result = new AudioBuffer({ - length: Math.max(1, Math.round((to - from) * sampleRate)), - sampleRate, - numberOfChannels: buffer.numberOfChannels, - }); - } - const offset = Math.round((timestamp - from) * sampleRate); - const skip = Math.max(0, -offset); - const count = Math.min(buffer.length - skip, result.length - Math.max(0, offset)); - if (count <= 0) continue; - for (let ch = 0; ch < result.numberOfChannels; ch++) { - const source = buffer.getChannelData(Math.min(ch, buffer.numberOfChannels - 1)); - result.getChannelData(ch).set(source.subarray(skip, skip + count), Math.max(0, offset)); - } + // Each decoded buffer is copied to where it belongs in the range, and + // whatever lies outside the range -- the ends of the first and last + // buffers -- is cut off. + for await (const { buffer, timestamp } of new MediaBunny.AudioBufferSink(track).buffers(from, to)) { + if (!result) { + sampleRate = buffer.sampleRate; + result = new AudioBuffer({ + length: Math.max(1, Math.round((to - from) * sampleRate)), + sampleRate, + numberOfChannels: buffer.numberOfChannels, + }); + } + const offset = Math.round((timestamp - from) * sampleRate); + const skip = Math.max(0, -offset); + const count = Math.min(buffer.length - skip, result.length - Math.max(0, offset)); + if (count <= 0) continue; + for (let ch = 0; ch < result.numberOfChannels; ch++) { + const source = buffer.getChannelData(Math.min(ch, buffer.numberOfChannels - 1)); + result.getChannelData(ch).set(source.subarray(skip, skip + count), Math.max(0, offset)); } - return result; - } finally { - input.dispose(); } + return result; } diff --git a/assets/js/processing/shared/BackwardFrames.js b/assets/js/processing/shared/BackwardFrames.js index 408c7386..c21777a0 100644 --- a/assets/js/processing/shared/BackwardFrames.js +++ b/assets/js/processing/shared/BackwardFrames.js @@ -45,8 +45,16 @@ export class BackwardFrameReader { * @param {Object} [options.decoderOptions] - passed to VideoSampleSink * @param {number} [options.maxFrames] - hold fewer than the default, for * a caller that keeps two readers going at once + * @param {(sample: Object) => Promise<{timestamp: number, close: () => void}>} [options.toHeld] + * turn each decoded frame into what is held, so the decoder's own frame + * can be released at once. Hardware decoders keep only a small pool of + * frames and can stall while too many are held; a caller that only needs + * to draw a frame can hold a copy instead. sampleAt() and + * backwardSamples() clone what is held, so for them the copy must itself + * be a VideoSample. */ - constructor(videoTrack, { decoderOptions, maxFrames = MAX_HELD_FRAMES } = {}) { + constructor(videoTrack, { decoderOptions, maxFrames = MAX_HELD_FRAMES, toHeld = null } = {}) { + this.toHeld = toHeld; this.track = videoTrack; this.sink = new MediaBunny.VideoSampleSink(videoTrack, decoderOptions); this.packets = new MediaBunny.EncodedPacketSink(videoTrack); @@ -156,7 +164,7 @@ export class BackwardFrameReader { * keeping the last `capacity` frames. * @private */ - async _fill(timestamp) { + async _fill(timestamp, { trustPackets = true } = {}) { const key = await this.packets.getKeyPacket(timestamp, { metadataOnly: true }) ?? await this.packets.getFirstPacket({ metadataOnly: true }); if (!key) { @@ -167,17 +175,39 @@ export class BackwardFrameReader { // Up to and including the frame at `timestamp`; a little past it, // since the range's end is exclusive. const end = timestamp + 2 * SAME_INSTANT; + // With a conversion, only the frames that will be kept are converted: + // which ones that is comes from the packets' timestamps, read without + // decoding. Converting every decoded frame and dropping most of them + // cost far more than the decoding did. + const keep = this.toHeld && trustPackets ? await this._lastTimestamps(key, timestamp) : null; + let decodedInRange = 0; const kept = []; for await (const sample of this.sink.samples(key.timestamp, end)) { this.decodedFrames++; if (sample.timestamp > timestamp + SAME_INSTANT) { sample.close(); continue; } - kept.push(sample); + decodedInRange++; + if (keep && !keep.has(Math.round(sample.timestamp * 1e6))) { sample.close(); continue; } + if (this.toHeld) { + try { + kept.push(await this.toHeld(sample)); + } finally { + sample.close(); + } + } else { + kept.push(sample); + } if (kept.length > this.capacity) kept.shift().close(); } if (this.closed) { for (const frame of kept) frame.close(); return; } + // Frames were decoded but none matched the packets' timestamps (a + // container whose timing the two disagree on): decode again, + // converting as it goes, rather than hold nothing and ask for ever. + if (keep && kept.length === 0 && decodedInRange > 0) { + return this._fill(timestamp, { trustPackets: false }); + } // Swapped in whole, so the old stretch stays drawable until the new // one is ready. const previous = this.held; @@ -185,4 +215,69 @@ export class BackwardFrameReader { this.heldUntil = timestamp; for (const frame of previous) frame.close(); } + + /** + * The timestamps of the last `capacity` frames from `key` up to + * `timestamp`, in microseconds, read from the packets alone. + * @private + */ + async _lastTimestamps(key, timestamp) { + const stamps = []; + for await (const packet of this.packets.packets(key, undefined, { metadataOnly: true })) { + // Decode order runs a little ahead of presentation order (B-frames), + // so read a couple of seconds past before stopping. + if (packet.timestamp > timestamp + 2) break; + if (packet.timestamp <= timestamp + SAME_INSTANT) stamps.push(packet.timestamp); + } + stamps.sort((a, b) => a - b); + return new Set(stamps.slice(-this.capacity).map(t => Math.round(t * 1e6))); + } +} + +/** + * Frames for a falling sequence of timestamps, with the stretch before the + * current one decoded while the current one is being used -- for an export, + * where the caller is busy encoding each frame and the decoder would + * otherwise sit idle until it finished. Two readers take turns, as in reverse + * playback; only one ever decodes at a time. + * + * Yields a VideoSample (or null before the first frame) per timestamp, + * owned by the caller, exactly as samplesAtTimestamps does. + * + * @param {Object} videoTrack - a Mediabunny InputVideoTrack + * @param {Iterable|AsyncIterable} timestamps - falling + * @param {Object} [options] - passed to each BackwardFrameReader; `toHeld` + * must give VideoSamples (it is what gets cloned and handed out) + */ +export async function* backwardSamples(videoTrack, timestamps, options = {}) { + const readers = [new BackwardFrameReader(videoTrack, options), new BackwardFrameReader(videoTrack, options)]; + let current = readers[0]; + const failed = []; + // Nothing comes before the first frame, so nothing to decode ahead there. + const first = await videoTrack.getFirstTimestamp().catch(() => 0); + try { + for await (const t of timestamps) { + let other = current === readers[0] ? readers[1] : readers[0]; + if (current.peek(t) === undefined) { + // The stretch decoded ahead is waited for, never decoded over. + if (other.busy) await other.fill(t).catch(() => {}); + if (failed.length) throw failed[0]; + if (other.peek(t) !== undefined) { + [current, other] = [other, current]; + } else { + await current.fill(t); + } + } + // The stretch before this one, decoded while this one is used. + const oldest = current.oldest; + if (oldest !== null && oldest > first + SAME_INSTANT + && !other.busy && other.peek(oldest - SAME_INSTANT) === undefined) { + other.fill(oldest - SAME_INSTANT).catch(error => failed.push(error)); + } + const frame = current.peek(t); + yield frame ? frame.clone() : await current.sink.getSample(t); + } + } finally { + for (const reader of readers) reader.close(); + } } diff --git a/assets/js/processing/speed/SpeedService.js b/assets/js/processing/speed/SpeedService.js index 184afc73..6f1ccd3a 100644 --- a/assets/js/processing/speed/SpeedService.js +++ b/assets/js/processing/speed/SpeedService.js @@ -4,7 +4,28 @@ import { AUDIO_BITRATE_BPS, createMediaBunnyInput } from '../shared/InputFactory import { addAudioSamples, stretchAudioBuffer } from '../shared/AudioHelpers.js'; import { cancellableProgress, cancelUnfinished, throwIfCancelled, isCancellation } from '../shared/Cancellation.js'; import { copyMetadataTags } from '../shared/MetadataTags.js'; -import { BackwardFrameReader } from '../shared/BackwardFrames.js'; +import { backwardSamples } from '../shared/BackwardFrames.js'; + +/** + * A decoded frame copied into ordinary memory, so the decoder's own frame can + * be released at once. Pixel format, size, timing, rotation and colour space + * carry over; a frame whose format cannot be read directly (some GPU frames) + * is copied as RGBX. + */ +async function copyOutOfDecoder(sample) { + const options = sample.format ? undefined : { format: 'RGBX' }; + const data = new Uint8Array(sample.allocationSize(options)); + await sample.copyTo(data, options); + return new MediaBunny.VideoSample(data, { + format: sample.format ?? 'RGBX', + codedWidth: sample.codedWidth, + codedHeight: sample.codedHeight, + timestamp: sample.timestamp, + duration: sample.duration, + rotation: sample.rotation, + colorSpace: sample.colorSpace, + }); +} async function processAudioWithSpeed(audioTrack, audioSource, speed = 1, signal) { const audioSink = new MediaBunny.AudioSampleSink(audioTrack); @@ -158,10 +179,13 @@ export async function reverseVideo({ source, includeAudio = false, speed = 1, on await copyMetadataTags(input, output); await output.start(); - // Frames come from BackwardFrameReader rather than samplesAtTimestamps: + // Frames come from backwardSamples rather than samplesAtTimestamps: // the latter re-decodes from the key frame for every frame when the - // timestamps fall, the reader decodes each stretch once. - const reader = new BackwardFrameReader(videoTrack); + // timestamps fall; this decodes each stretch once, and the next one + // back while the current one is being encoded. What it holds are + // copies in ordinary memory, not the decoder's own frames: a hardware + // decoder keeps only a few, and holding them is what left reverse + // playback waiting for ever on a real GPU. const reverseTimestampGenerator = (async function* () { for (let outputFrame = 0; outputFrame < totalOutputFrames; outputFrame++) { const outputProgress = outputFrame / totalOutputFrames; @@ -173,28 +197,24 @@ export async function reverseVideo({ source, includeAudio = false, speed = 1, on let outputTimestamp = 0; let frameCount = 0; - try { - for await (const sample of reader.samplesAt(reverseTimestampGenerator)) { - if (!sample) continue; - try { - sample.setTimestamp(outputTimestamp); - sample.setDuration(frameDuration); - await canvasSource.add(sample); - } finally { - sample.close(); - } - outputTimestamp += frameDuration; - frameCount++; + for await (const sample of backwardSamples(videoTrack, reverseTimestampGenerator, { maxFrames: 24, toHeld: copyOutOfDecoder })) { + if (!sample) continue; + try { + sample.setTimestamp(outputTimestamp); + sample.setDuration(frameDuration); + await canvasSource.add(sample); + } finally { + sample.close(); + } + outputTimestamp += frameDuration; + frameCount++; - if (onProgress) { - onProgress(frameCount / totalOutputFrames * 0.8); - } + if (onProgress) { + onProgress(frameCount / totalOutputFrames * 0.8); } - } finally { - reader.close(); } - Logger.log(`[MediaProcessor] Encoded ${frameCount} frames (${reader.decodedFrames} decoded)`); + Logger.log(`[MediaProcessor] Encoded ${frameCount} frames`); if (includeAudio && audioSource) { Logger.log('[MediaProcessor] Reversing audio...'); diff --git a/assets/js/ui/player/PlayerBindings.js b/assets/js/ui/player/PlayerBindings.js index 73730293..b8454a5d 100644 --- a/assets/js/ui/player/PlayerBindings.js +++ b/assets/js/ui/player/PlayerBindings.js @@ -194,12 +194,15 @@ export function attachPlayerBindings(player) { player.ui.speedSlider.value = player.playbackRate; player.ui.speedValue.textContent = `${player.playbackRate.toFixed(2)}x`; - if (player.ui.reverseBtn) { - player.ui.reverseBtn.addEventListener('click', () => { - player.toggleReverse(); - player.ui.reverseBtn.blur(); - }); - } + } + + if (player.ui.directionBtn) { + player.ui.directionBtn.addEventListener('click', async (e) => { + e.stopPropagation(); + player.ui.directionBtn.blur(); + const result = await player.toggleDirection(); + if (result?.reason) Toast.show(result.reason, 3500); + }); } if (player.config.controls.fullscreen) { @@ -507,10 +510,6 @@ export function attachPlayerBindings(player) { player._events = {}; // After the event list exists; registered any earlier, it would be wiped. - if (player.ui.reverseBtn) { - player.on('reversechange', ({ active }) => { - player.ui.reverseBtn.textContent = active ? '■ Stop playing backwards' : '◀ Play backwards'; - player.ui.reverseBtn.classList.toggle('active', active); - }); - } + // Play pressed while backward is set, and backwards could not start. + player.on('reversefailed', ({ reason }) => Toast.show(reason, 3500)); } diff --git a/assets/js/ui/player/PlayerControlsView.js b/assets/js/ui/player/PlayerControlsView.js index 1d1e44eb..245c19d8 100644 --- a/assets/js/ui/player/PlayerControlsView.js +++ b/assets/js/ui/player/PlayerControlsView.js @@ -26,7 +26,10 @@ export function createPlayerControls(player) { player.ui.playOverlay.style.display = 'none'; } - if (player.config.controls.playPause) player.ui.playBtn = q('#mb-play-btn'); + if (player.config.controls.playPause) { + player.ui.playBtn = q('#mb-play-btn'); + player.ui.directionBtn = q('#mb-direction-btn'); + } if (player.config.controls.navigation) { player.ui.prevBtn = q('#mb-prev-btn'); player.ui.nextBtn = q('#mb-next-btn'); @@ -77,7 +80,7 @@ function initSpeedPanel(player) { player.ui.speedSlider = q('#mb-speed-slider'); player.ui.speedValue = q('#mb-speed-value'); player.ui.resetSpeedBtn = q('#mb-reset-speed-btn'); - player.ui.reverseBtn = q('#mb-reverse-btn'); + player.ui.closeSpeedPanelBtn = q('.jellyjump-speed-panel .jellyjump-close-btn'); } diff --git a/assets/js/ui/player/PlayerKeyboard.js b/assets/js/ui/player/PlayerKeyboard.js index 7fce507f..1db9493f 100644 --- a/assets/js/ui/player/PlayerKeyboard.js +++ b/assets/js/ui/player/PlayerKeyboard.js @@ -1,3 +1,4 @@ +import { Toast } from '../../shared/utils/Toast.js'; export class PlayerKeyboard { constructor(player) { this.player = player; @@ -29,7 +30,11 @@ export class PlayerKeyboard { case 'j': e.preventDefault(); // Shift+J plays backwards (and stops doing so); J alone steps back. - if (e.shiftKey) p.toggleReverse(); + if (e.shiftKey) { + p.toggleDirection().then(result => { + if (result?.reason) Toast.show(result.reason, 3500); + }); + } else p._requestSeek(p.currentTime - 10); break; case 'l': diff --git a/assets/js/ui/player/PlayerUIUpdates.js b/assets/js/ui/player/PlayerUIUpdates.js index 6454bf31..6ba3f96f 100644 --- a/assets/js/ui/player/PlayerUIUpdates.js +++ b/assets/js/ui/player/PlayerUIUpdates.js @@ -4,6 +4,22 @@ import { Logger } from '../../shared/utils/Logger.js'; export function updatePlayerPlayPauseUI(player) { if (!player.ui.playBtn) return; + // Which way Play goes: the direction button lit, and the play arrow + // pointing back, so it is plain before pressing Play. + const backward = player.direction === 'backward'; + player.ui.playBtn.classList.toggle('is-backward', backward); + if (player.ui.directionBtn) { + // Hidden where playing backwards cannot work: a live stream (it + // keeps moving, so there is no fixed past to run back through), the + // camera, YouTube, or something with no picture. + player.ui.directionBtn.hidden = !!player.reverse?.unavailableReason; + player.ui.directionBtn.classList.toggle('active', backward); + player.ui.directionBtn.setAttribute('aria-pressed', String(backward)); + const label = backward ? 'Play forwards' : 'Play backwards'; + player.ui.directionBtn.setAttribute('aria-label', label); + player.ui.directionBtn.title = `${label} (Shift+J)`; + } + const use = player.ui.playBtn.querySelector('use'); // Playing backwards counts as playing: the button stops it. if (player.isPlaying || player.reverse?.active) { diff --git a/public/assets/templates/player-templates.html b/public/assets/templates/player-templates.html index fe771a75..8d2d0fb5 100644 --- a/public/assets/templates/player-templates.html +++ b/public/assets/templates/player-templates.html @@ -51,6 +51,13 @@

+ @@ -439,7 +445,7 @@

Playback

  • Space / K Play/Pause
  • J / L -/+ 10s
  • -
  • Shift+J Play backwards
  • +
  • Shift+J Play forwards / backwards
  • ← / → -/+ 5s
  • 0-9 Seek 0-90%
  • Home / End Start/End