From 3b3faf168d3bde2167c2a0a24b64efd2f6300d7e Mon Sep 17 00:00:00 2001 From: Stackie Jia Date: Tue, 8 Sep 2026 07:31:48 +0800 Subject: [PATCH 1/6] feat(player): improve temporal denoising and upscale fidelity --- docs/en/guide/web-player.md | 9 + docs/guide/web-player.md | 9 + tools/README.md | 1 + tools/video-quality/README.md | 169 +++++ tools/video-quality/index.html | 27 + tools/video-quality/invariants.ts | 156 +++++ tools/video-quality/lifecycle.ts | 227 ++++++ tools/video-quality/main.ts | 493 +++++++++++++ tools/video-quality/measurements.json | 653 ++++++++++++++++++ tools/video-quality/prepare-baseline.mjs | 24 + tools/video-quality/vite.config.ts | 60 ++ web-ui/src/playback-engine/render/denoise.ts | 237 +++++++ .../render/filters/gl-utils.ts | 30 +- .../render/filters/mosquito-nr.ts | 205 ------ web-ui/src/playback-engine/render/fsr.ts | 184 +++-- web-ui/src/playback-engine/render/index.ts | 1 - web-ui/src/playback-engine/render/renderer.ts | 225 +++--- 17 files changed, 2268 insertions(+), 442 deletions(-) create mode 100644 tools/video-quality/README.md create mode 100644 tools/video-quality/index.html create mode 100644 tools/video-quality/invariants.ts create mode 100644 tools/video-quality/lifecycle.ts create mode 100644 tools/video-quality/main.ts create mode 100644 tools/video-quality/measurements.json create mode 100644 tools/video-quality/prepare-baseline.mjs create mode 100644 tools/video-quality/vite.config.ts create mode 100644 web-ui/src/playback-engine/render/denoise.ts delete mode 100644 web-ui/src/playback-engine/render/filters/mosquito-nr.ts diff --git a/docs/en/guide/web-player.md b/docs/en/guide/web-player.md index 8c7f717c..942d2c21 100644 --- a/docs/en/guide/web-player.md +++ b/docs/en/guide/web-player.md @@ -9,6 +9,7 @@ rtp2httpd includes a modern web-based player that allows you to watch configured - **Time-Shifted Playback**: Supports EPG (Electronic Program Guide) and time-shifted playback (requires catchup source) - **Fast Startup**: Achieves millisecond-level channel switching with FCC - **Seamless Channel Switching**: Preloads the new stream when switching channels to reduce black screens +- **Video Enhancement**: Motion-adaptive denoising, edge-adaptive upscaling, and luminance sharpening - **Responsive Design**: UI adapts to both desktop and mobile devices - **PWA Support**: Can be added to the home screen on phones, tablets, desktops, or LG webOS TVs for app-like quick access - **Zero Overhead**: Pure web frontend implementation with virtually no resource overhead on rtp2httpd (no decoding/transcoding overhead) @@ -122,6 +123,14 @@ The built-in web player enables **seamless channel switching** by default. When This feature uses dual-slot preloading. Because two video streams are pulled simultaneously for a brief period during channel switching, you may experience stuttering when switching channels if your bandwidth is limited (for example, when accessing rtp2httpd over the public internet), or if the upstream only supports a single multicast stream. In that case, you can manually disable this option in the **Settings** menu (gear icon) at the top-right of the player page by turning off **Seamless switch**. When disabled, the player stops the current stream before loading the new one, using less bandwidth but may show a brief black screen during switching. +## Video Enhancement + +The player enables **Video enhancement** by default. You can toggle it in **Settings** (the gear icon) at the top-right. It is intended for devices with a capable GPU, such as PCs and iPads, and requires browser support for WebGL2 and video frame callbacks. Processing runs on the viewing device, adding no decoding or transcoding overhead to the server. + +The enhancement pipeline combines the current image with previous images to reduce noise, upscales it to the display area, and applies moderate edge sharpening. Motion, scene cuts, and channel changes suppress or clear the influence of previous images. Sharpening preserves the original colors and tonal relationships, limits edge overshoot, and avoids amplifying noise again. + +This feature processes **1080p and lower-resolution sources**, with an output limit of **3840 × 2160**. The actual size depends on the display area and device pixel ratio. Native 4K video plays directly through the browser. Video enhancement does not generate intermediate frames. For interlaced video with **Auto deinterlace** enabled, the player reconstructs each field first (for example, 50i → 50p), then enhances each reconstructed image. + ## Time Placeholders The built-in web player supports the following time placeholder formats, which can be used in M3U `catchup-source` URLs: diff --git a/docs/guide/web-player.md b/docs/guide/web-player.md index 77a1ec5e..38322854 100644 --- a/docs/guide/web-player.md +++ b/docs/guide/web-player.md @@ -9,6 +9,7 @@ rtp2httpd 内置了一个基于 Web 的现代化播放器，可以直接在浏 - **时移回看**：支持 EPG 电子节目单和时移回看（需要有回看源） - **快速起播**：搭配 FCC 可实现毫秒级换台速度 - **无缝换台**：换台时预加载新流，减少黑屏 +- **画质增强**：运动自适应降噪、边缘自适应放大与亮度锐化 - **响应式设计**：桌面和移动端 UI 自适应 - **PWA 支持**：可添加到手机、平板、桌面或 LG webOS 电视主屏幕，像原生应用一样快捷打开 - **零开销**：纯 Web 前端实现，对 rtp2httpd 运行几乎没有资源占用（无解码转码开销） @@ -122,6 +123,14 @@ http://192.168.1.1:5140/rtp/239.253.64.200:5140$高清 该功能通过双槽位预加载实现。由于在换台过程中短暂存在两路视频同时拉流的情况，如果你的带宽比较小（公网访问时），或是上游只支持单路组播，导致换台出现卡顿，可以手动关闭这个选项——在播放器页面右上角的 **设置**（齿轮图标）中关闭 **无缝换台** 即可。关闭后换台会先断开当前流再加载新流，占用带宽更低，但切换时可能出现短暂黑屏。 +## 画质增强 + +播放器默认开启 **画质增强**，可在右上角的 **设置**（齿轮图标）中切换。适合具有较强 GPU 的 PC、iPad 等设备，需要浏览器支持 WebGL2 和视频帧回调。处理在观看设备上完成，不增加服务器的解码或转码开销。 + +增强链路先结合当前画面与历史画面降低噪声，再按显示区域放大，并适度增强边缘清晰度。运动、场景切换和换台时会抑制或清除历史画面的影响；锐化保持原有色彩与明暗关系，限制边缘过冲，避免将噪声重新放大。 + +该功能仅处理 **1080p 及以下的视频源**，放大输出最高为 **3840 × 2160**，实际尺寸取决于显示区域和设备像素比。原生 4K 视频由浏览器直接播放。画质增强不会生成中间帧；隔行视频开启 **自动去隔行** 后，先按场重建（例如 50i → 50p），再对每一场进行增强。 + ## 时间占位符 内置 Web 播放器支持以下时间占位符格式，可在 M3U `catchup-source` URL 中使用： diff --git a/tools/README.md b/tools/README.md index 8ac9f491..befbeeef 100644 --- a/tools/README.md +++ b/tools/README.md @@ -5,5 +5,6 @@ Utility tools are organized by tool directory: - [udp-replay](./udp-replay/README.md): IGMP-aware multicast UDP replay from pcapng captures. - [stress-test](./stress-test/README.md): automated performance tests for rtp2httpd, msd_lite, udpxy, and tvgate. - [devlab](./devlab/README.md): local mock IPTV upstreams for web-player development. +- [video-quality](./video-quality/README.md): real WebGL video enhancement checks, frame comparisons, and GPU timings. Shared test captures and playlists live in [fixtures](./fixtures/). diff --git a/tools/video-quality/README.md b/tools/video-quality/README.md new file mode 100644 index 00000000..4b47b63b --- /dev/null +++ b/tools/video-quality/README.md @@ -0,0 +1,169 @@ +# Video quality lab + +Exercises the production WebGL shaders and renderer in a browser, alongside the +previous enhancement implementation and a bilinear reference. No additional +JavaScript dependencies are needed. Recordings and generated PNG/JSON results +stay in the gitignored `build-video-quality/` directory. + +## Run + +From the repository root: + +```sh +source ~/.nvm/nvm.sh +nvm use +node tools/video-quality/prepare-baseline.mjs +pnpm exec vite tools/video-quality --config tools/video-quality/vite.config.ts +``` + +Open `http://127.0.0.1:5186/`. The baseline defaults to commit +`3bf4ca8c5521eebcd8be76c387c3ecbe904d55ac`; pass another compatible revision to +`prepare-baseline.mjs` to select it explicitly. Its source is read with `git show`, +without changing the checkout. The extracted revision is recorded in +`build-video-quality/baseline/revision.json`. + +Use the buttons or the developer console: + +```js +await qualityLab.synthetic(); +await qualityLab.invariants(); +await qualityLab.timing(1920, 1080, 1920, 1080); +await qualityLab.timing(1920, 1080, 3840, 2160); +await qualityLab.timing(720, 576, 1920, 1080); +await qualityLab.compare("sport", 3); +await qualityLab.compare("film", 3); +await qualityLab.restoration("sport", 0); +await qualityLab.restoration("sport", 6); +await qualityLab.restoration("film", 0); +await qualityLab.restoration("film", 6); +await qualityLab.lifecycle(); +``` + +Run these sequentially, with other video playback paused. Keep the page visible. +Use `?label=safari` (or another label) to keep a browser's results separate. +The lab listens on loopback; its result endpoint only writes into the artifacts +directory. It serves recordings with byte ranges so seeking decodes the requested +frames correctly. + +## Recording fixtures + +Synthetic, invariant, and GPU timing checks need no recordings. For programme +comparisons, put your own `sport.ts` and `film.ts` captures under +`build-video-quality/fixtures/`. Both should be 1080i, top field first. Prepare +50 fps progressive recordings so all three enhancement modes receive exactly the +same deinterlaced pixels: + +```sh +ffmpeg -i build-video-quality/fixtures/sport.ts -an \ + -vf bwdif=mode=send_field:parity=tff:deint=all -c:v libx264 -crf 12 \ + -pix_fmt yuv420p -movflags +faststart build-video-quality/fixtures/sport.mp4 +ffmpeg -i build-video-quality/fixtures/film.ts -an \ + -vf bwdif=mode=send_field:parity=tff:deint=all -c:v libx264 -crf 12 \ + -pix_fmt yuv420p -movflags +faststart build-video-quality/fixtures/film.mp4 +ffmpeg -i build-video-quality/fixtures/sport.ts -an -c:v copy \ + -movflags +faststart build-video-quality/fixtures/sport-interlaced.mp4 +ffmpeg -i build-video-quality/fixtures/film.mp4 -an -t 8 \ + -vf scale=720:576 -c:v libx264 -crf 16 -movflags +faststart \ + build-video-quality/fixtures/sd.mp4 +ffmpeg -ss 3 -i build-video-quality/fixtures/sport.mp4 -frames:v 1 \ + build-video-quality/fixtures/sport-reference.png +ffmpeg -ss 3 -i build-video-quality/fixtures/film.mp4 -frames:v 1 \ + build-video-quality/fixtures/film-reference.png +``` + +The SD fixture is derived from HD. Its 16:9 sample aspect ratio means the browser +reports a display width of 1024, although the encoded raster is 720 × 576. +Do not commit recordings, private playlists, or programme images. + +## What the checks measure + +- **Synthetic:** clean gradients, thin detail, static grain, moving edges, and + moving edges with grain. Noise uses deterministic independent seeds for 16 + frames, approximately 6 code values of luma standard deviation plus chroma + noise. PSNR/MAE compare RGB with known clean pixels, excluding alpha. Exact + equality is represented as PSNR 100. Flat colors check transfer-curve drift. +- **Invariants:** odd dimensions, history orientation, scene cuts, slow + low-contrast movement, and isoluminant movement. Both float history and a + forced RGBA8 fallback execute on real WebGL objects. The motion checks compare + the entire image, including vacated areas and corners, with known pixels. +- **Restoration:** exact CPU 2 × 2 area reduction of a known 1080p frame to 540p, + followed by reconstruction to 1080p. The noisy variant adds independent grain + to 16 observations of that reference. This tests reconstruction fidelity; it + does not establish a noise-free reference for the original broadcast. +- **Recorded comparisons:** independently seeks twelve successive 50 fps frames, + checks each decoded media timestamp within 1 ms, and feeds them in order to + each mode. Saved PNGs use the same last frame and output size. Inspect faces, + subtitles, dark textures, and motion boundaries at 1:1 pixel scale. +- **Timing:** asynchronous `EXT_disjoint_timer_query_webgl2` queries around + denoising and presentation. Each sample averages eight draws; 45 samples per + mode are collected across three rounds with rotating mode order and twelve + warm-up draws. Reported p95 is the p95 of these batch averages. Upload, decode, + deinterlacing, and browser composition are excluded. Without the extension, + results are labeled CPU+GPU completion time, not GPU time. Disjoint samples + are discarded and insufficient samples fail the run. +- **Lifecycle:** the actual video render pipeline processes an interlaced + recording, checks two presentations per source frame, repeated paused toggles, + seeking, source/resolution changes, raw-video fallback, context restoration, + GL errors, and resource release. It uses the production context attributes. + +## Reference measurements + +Recorded on 2026-09-08 with an Apple M3 Max (40 GPU cores), Chromium 152 and ANGLE +Metal. Full numerical results and source/fixture hashes are in +[`measurements.json`](./measurements.json). These are measurements on one device; +Safari/WebKit and physical iPad performance have not been verified. + +PSNR in dB (higher is better): + +| Case | Bilinear/raw | Previous enhancement | Current enhancement | +| --- | ---: | ---: | ---: | +| Clean synthetic detail | Exact | 38.06 | 46.94 | +| Static synthetic grain | 32.20 | 28.48 | 34.52 | +| Moving detail with grain | 32.20 | 28.40 | 34.35 | +| Sport reference, 540p → 1080p | 34.81 | 34.33 | 36.85 | +| Film reference, 540p → 1080p | 33.51 | 33.51 | 35.19 | +| Noisy sport reference, 540p → 1080p | 32.48 | 30.37 | 33.66 | +| Noisy film reference, 540p → 1080p | 31.78 | 30.74 | 32.85 | + +Flat colors had zero code-value error in the current enhancement, versus a +maximum of six in the previous implementation. Clean denoiser-only output had +a maximum error of one. The low-contrast and isoluminant motion invariants had +zero error in both history formats. + +Enhancement GPU milliseconds per draw, measured as described above: + +| Source → output | Previous median | Current median | Previous p95 | Current p95 | +| --- | ---: | ---: | ---: | ---: | +| 1920 × 1080 → 1920 × 1080 | 0.516 | 0.720 | 1.086 | 0.962 | +| 1920 × 1080 → 3840 × 2160 | 1.925 | 2.066 | 3.454 | 4.104 | +| 720 × 576 → 1920 × 1080 | 1.062 | 1.161 | 1.360 | 2.074 | + +Full MSE playback through a separate NAS preview was also checked: a 15-second +1080i broadcast interval decoded 375 frames with zero dropped/corrupted frames +and zero GL errors, both at native output and at 3840 × 2160. The renderer +lifecycle test presented 50 fields for 25 source callbacks and retained no +textures, framebuffers, programs, or shaders after destruction. + +## Implementation boundaries + +The denoiser uses one source-resolution pass with nine current-frame samples and +five history samples. RGB stores the filtered image; alpha stores original luma +for motion/noise analysis. Two RGBA16F history textures cost 31.6 MiB at 1080p; +RGBA8 fallback halves that. EASU's RGB10_A2 intermediate costs 31.6 MiB at 4K, +the same storage as RGBA8. These figures exclude upload/deinterlacing targets and +the browser's canvas buffers. + +FSR presentation expects the denoiser's original-luma alpha. Its intermediate +alpha carries quantized noise information for luma-only sharpening. The source +gate remains 1920 × 1088, with output capped at 3840 × 2160 and the device's +texture limit. History resets on seeks, source/stage changes, and discontinuous +timestamps. No future frames or motion-compensated frame interpolation are added; +interlaced playback retains the existing two-field presentation. + +The RGB video upload ring uses immutable storage before its first +`texSubImage2D(video)` call. This avoids reusing decoder-backed storage imported +by `texImage2D(video)`, which can reject subsequent frame updates on ANGLE Metal. + +For a static preview alongside a running daemon, proxy its application-prefix +routes to that daemon as well as serving the new `web-ui/dist` assets. The M3U +parser converts absolute programme/EPG URLs to site-root-relative paths. diff --git a/tools/video-quality/index.html b/tools/video-quality/index.html new file mode 100644 index 00000000..eaaf15cf --- /dev/null +++ b/tools/video-quality/index.html @@ -0,0 +1,27 @@ + + + + + + Video quality lab + + + +

Video quality lab

+

Tests the actual WebGL shaders. Recordings and results stay in build-video-quality.

+ + + + + + + 
Loading…
+ + + + diff --git a/tools/video-quality/invariants.ts b/tools/video-quality/invariants.ts new file mode 100644 index 00000000..7bd01fe1 --- /dev/null +++ b/tools/video-quality/invariants.ts @@ -0,0 +1,156 @@ +import { TemporalDenoiser } from "../../web-ui/src/playback-engine/render/denoise"; +import { FsrPresenter } from "../../web-ui/src/playback-engine/render/fsr"; +import { PassthroughPresenter } from "../../web-ui/src/playback-engine/render/presenters"; + +type Pixels = Uint8Array; + +function difference(actual: Pixels, expected: Pixels) { + let maximum = 0; + let sum = 0; + for (let i = 0; i < actual.length; i++) { + if (i % 4 === 3) continue; + const delta = Math.abs(actual[i] - expected[i]); + maximum = Math.max(maximum, delta); + sum += delta; + } + return { maximum, mae: sum / ((actual.length / 4) * 3) }; +} + +/** Ground-truth motion and resource checks on real GPU objects, in both history formats. */ +export function invariants() { + const results: Record = {}; + for (const forceRgba8 of [false, true]) { + const width = 193; + const height = 109; + const canvas = document.createElement("canvas"); + canvas.width = width; + canvas.height = height; + const real = canvas.getContext("webgl2", { alpha: false, antialias: false, preserveDrawingBuffer: true }); + if (!real) throw Error("WebGL2 unavailable"); + const formats: number[] = []; + const gl = new Proxy(real, { + get(target, key) { + if (key === "getExtension") + return (name: string) => (forceRgba8 && name === "EXT_color_buffer_float" ? null : target.getExtension(name)); + if (key === "texStorage2D") + return (...args: Parameters) => { + formats.push(args[2]); + target.texStorage2D(...args); + }; + const value = Reflect.get(target, key, target); + return typeof value === "function" ? value.bind(target) : value; + }, + }); + const input = gl.createTexture(); + if (!input) throw Error("Input texture unavailable"); + gl.bindTexture(gl.TEXTURE_2D, input); + gl.texStorage2D(gl.TEXTURE_2D, 1, gl.RGBA8, width, height); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); + const denoiser = new TemporalDenoiser(); + const raw = new PassthroughPresenter(); + const fsr = new FsrPresenter(); + denoiser.init(gl); + raw.init(gl); + fsr.init(gl); + const checks: Record = {}; + const draw = (data: Pixels, flipY = false, sharpen = false) => { + gl.activeTexture(gl.TEXTURE0); + gl.bindTexture(gl.TEXTURE_2D, input); + gl.texSubImage2D(gl.TEXTURE_2D, 0, 0, 0, width, height, gl.RGBA, gl.UNSIGNED_BYTE, data); + const texture = denoiser.render(gl, input, width, height, flipY); + gl.bindFramebuffer(gl.FRAMEBUFFER, null); + gl.viewport(0, 0, width, height); + (sharpen ? fsr : raw).present(gl, texture, width, height, width, height, false); + const pixels = new Uint8Array(data.length); + gl.readPixels(0, 0, width, height, gl.RGBA, gl.UNSIGNED_BYTE, pixels); + return pixels; + }; + const solid = (color: number[]) => { + const data = new Uint8Array(width * height * 4); + for (let i = 0; i < data.length; i += 4) data.set([...color, 255], i); + return data; + }; + try { + let flatError = 0; + for (const color of [ + [0, 0, 0], + [255, 255, 255], + [2, 4, 6], + [17, 17, 17], + [201, 41, 97], + ]) { + denoiser.reset(); + const data = solid(color); + let actual = data; + for (let frame = 0; frame < 24; frame++) actual = draw(data, false, true); + flatError = Math.max(flatError, difference(actual, data).maximum); + } + checks.flatColorMaximum = flatError; + if (flatError > 1) throw Error(`Flat color drifted by ${flatError} codes`); + + // An asymmetric gradient makes an upside-down history visible immediately. + const gradient = solid([0, 0, 0]); + for (let y = 0; y < height; y++) + for (let x = 0; x < width; x++) { + const i = (y * width + x) * 4; + gradient.set([20 + y, 30 + Math.floor(x / 2), 200 - y, 255], i); + } + const reversed = new Uint8Array(gradient.length); + for (let y = 0; y < height; y++) + reversed.set(gradient.subarray(y * width * 4, (y + 1) * width * 4), (height - y - 1) * width * 4); + denoiser.reset(); + let flipped = gradient; + for (let frame = 0; frame < 20; frame++) flipped = draw(gradient, true); + checks.flippedHistory = difference(flipped, reversed); + if (difference(flipped, reversed).maximum > 1) throw Error("History orientation changed the gradient"); + + const before = solid([30, 100, 180]); + denoiser.reset(); + for (let frame = 0; frame < 20; frame++) draw(before); + const cut = draw(gradient); + denoiser.reset(); + const fresh = draw(gradient); + checks.sceneCut = difference(cut, fresh); + if (difference(cut, fresh).maximum > 2) throw Error("A scene cut retained the previous image"); + + for (const [name, foreground] of [ + ["low-contrast", [87, 87, 87]], + // Almost equal BT.709 luma; only chroma can identify the moving patch. + ["isoluminant", [120, 68, 80]], + ] as const) { + for (const speed of [1, 7]) { + denoiser.reset(); + let maxError = 0; + let maxMae = 0; + for (let frame = 0; frame < 20; frame++) { + const data = solid([80, 80, 80]); + const left = 4 + frame * speed; + for (let y = 20; y < 80; y++) + for (let x = left; x < left + 29; x++) data.set([...foreground, 255], (y * width + x) * 4); + const delta = difference(draw(data), data); + maxError = Math.max(maxError, delta.maximum); + maxMae = Math.max(maxMae, delta.mae); + } + checks[`${name}-${speed}px`] = { maximum: maxError, mae: maxMae }; + if (maxError > 4 || maxMae > 0.15) + throw Error(`Visible trails in ${name} at ${speed}px/frame: maximum=${maxError}, mae=${maxMae}`); + } + } + const code = gl.getError(); + if (code !== gl.NO_ERROR) throw Error(`WebGL invariant error ${code}`); + checks.historyFormat = formats[1] === gl.RGBA16F ? "RGBA16F" : "RGBA8"; + checks.source = [width, height]; + results[forceRgba8 ? "fallback" : "preferred"] = checks; + } finally { + denoiser.destroy(gl); + raw.destroy(gl); + fsr.destroy(gl); + gl.deleteTexture(input); + real.getExtension("WEBGL_lose_context")?.loseContext(); + } + } + return results; +} diff --git a/tools/video-quality/lifecycle.ts b/tools/video-quality/lifecycle.ts new file mode 100644 index 00000000..9671a292 --- /dev/null +++ b/tools/video-quality/lifecycle.ts @@ -0,0 +1,227 @@ +import { createVideoRenderPipeline } from "../../web-ui/src/playback-engine/render"; +import type { PlayerRenderState } from "../../web-ui/src/playback-engine/types"; + +function assert(condition: unknown, message: string): asserts condition { + if (!condition) throw new Error(message); +} + +function event(target: EventTarget, name: string, timeoutMs = 10000): Promise { + return new Promise((resolve, reject) => { + const done = () => { + clearTimeout(timeout); + resolve(); + }; + const timeout = setTimeout(() => { + target.removeEventListener(name, done); + reject(new Error(`Timed out waiting for ${name}`)); + }, timeoutMs); + target.addEventListener(name, done, { once: true }); + }); +} + +/** Exercises the real renderer and real WebGL objects, including context loss. */ +export async function lifecycle() { + const container = document.createElement("div"); + container.style.cssText = "position:relative;width:1280px;height:720px;max-width:none"; + const video = document.createElement("video"); + video.muted = true; + video.playsInline = true; + video.style.cssText = "position:absolute;width:100%;height:100%;opacity:0"; + const canvas = document.createElement("canvas"); + canvas.style.cssText = "position:absolute;width:100%;height:100%"; + container.append(video, canvas); + document.body.append(container); + + const live = new Map(["Texture", "Framebuffer", "Program", "Shader"].map((name) => [name, new Set()])); + const createContext = canvas.getContext.bind(canvas); + let context: WebGL2RenderingContext | null = null; + let draws = 0; + let presentations = 0; + const errors: { method: string; code: number; stack?: string }[] = []; + const states: PlayerRenderState[] = []; + Object.defineProperty(canvas, "getContext", { + value(type: string, options: WebGLContextAttributes) { + assert(type === "webgl2", "Unexpected context type"); + if (context) return context; + const real = createContext("webgl2", options); + assert(real, "WebGL2 unavailable"); + const functions = new Map(); + context = new Proxy(real, { + get(target, key) { + const value = Reflect.get(target, key, target); + if (typeof value !== "function") return value; + if (!functions.has(key)) + functions.set(key, (...args: unknown[]) => { + const result = Reflect.apply(value, target, args); + const name = String(key); + if (name.startsWith("create") && result) live.get(name.slice(6))?.add(result); + if (name.startsWith("delete")) live.get(name.slice(6))?.delete(args[0]); + if (name === "drawArrays") { + draws++; + if (real.getParameter(real.FRAMEBUFFER_BINDING) === null) presentations++; + } + if (name !== "getError") { + const code = real.getError(); + if (code && code !== real.CONTEXT_LOST_WEBGL && errors.length < 3) + errors.push({ method: name, code, stack: new Error().stack }); + } + return result; + }); + return functions.get(key); + }, + }); + return context; + }, + }); + // Context loss invalidates all of its objects without explicit delete calls. + canvas.addEventListener("webglcontextlost", () => { + for (const resources of live.values()) resources.clear(); + }); + const pipeline = createVideoRenderPipeline(video, canvas, (state) => states.push(state)); + const gl = () => { + assert(context, "Renderer did not create a context"); + return context; + }; + const frames = (count: number) => + new Promise((resolve, reject) => { + let handle = 0; + const timeout = setTimeout(() => { + video.cancelVideoFrameCallback(handle); + reject(Error("Video frames stalled")); + }, 10000); + const next = (_now: number, metadata: VideoFrameCallbackMetadata) => { + if (--count === 0) { + clearTimeout(timeout); + resolve(metadata); + } else handle = video.requestVideoFrameCallback(next); + }; + handle = video.requestVideoFrameCallback(next); + }); + const sample = () => { + const ctx = gl(); + ctx.bindFramebuffer(ctx.FRAMEBUFFER, null); + const data = new Uint8Array(16 * 16 * 4); + ctx.readPixels( + Math.floor(canvas.width / 2), + Math.floor(canvas.height / 2), + 16, + 16, + ctx.RGBA, + ctx.UNSIGNED_BYTE, + data, + ); + const code = ctx.getError(); + assert( + code === ctx.NO_ERROR && errors.length === 0, + `WebGL error while sampling output: ${code} ${JSON.stringify(errors)}`, + ); + return data; + }; + const snapshot = () => Object.fromEntries([...live].map(([name, objects]) => [name, objects.size])); + const playbackQuality = () => { + const quality = video.getVideoPlaybackQuality(); + return { + total: quality.totalVideoFrames, + dropped: quality.droppedVideoFrames, + corrupted: quality.corruptedVideoFrames, + }; + }; + const results: Record = {}; + try { + pipeline.setScanType("interlaced"); + const loaded = event(video, "loadeddata"); + video.src = "/fixtures/sport-interlaced.mp4"; + await loaded; + await video.play(); + await frames(6); + assert(pipeline.active, "Interlaced enhancement did not activate"); + const start = presentations; + const first = playbackQuality(); + const metadata = await frames(25); + const presented = presentations - start; + assert(presented >= 42 && presented <= 54, `Expected two fields per frame, saw ${presented} presentations`); + results.interlaced = { + callbacks: 25, + presentations: presented, + mediaTime: metadata.mediaTime, + qualityBefore: first, + qualityAfter: playbackQuality(), + }; + + video.pause(); + pipeline.setPictureEnhancementEnabled(false); + pipeline.setPictureEnhancementEnabled(true); + const expected = sample(); + assert( + expected.some((value, index) => index % 4 !== 3 && value > 0), + "Enhanced frame is black", + ); + const initial = snapshot(); + for (let i = 0; i < 8; i++) { + pipeline.setPictureEnhancementEnabled(false); + pipeline.setPictureEnhancementEnabled(true); + const current = sample(); + assert( + current.every((value, index) => Math.abs(value - expected[index]) <= 1), + "Paused frame changed after repeated toggles", + ); + } + assert(JSON.stringify(snapshot()) === JSON.stringify(initial), "GPU resources grew across toggles"); + results.pausedToggles = { repeats: 8, resources: snapshot() }; + + const seeked = event(video, "seeked"); + video.currentTime = 4; + await seeked; + assert(gl().getError() === gl().NO_ERROR, "WebGL error after seeking"); + results.seek = video.currentTime; + + pipeline.setAutoDeinterlaceEnabled(false); + pipeline.setPictureEnhancementEnabled(false); + assert(!pipeline.active, "Disabled processing did not return to raw video"); + assert(live.get("Texture")?.size === 0, "Disabling processing retained frame textures"); + pipeline.setPictureEnhancementEnabled(true); + assert(pipeline.active, "Enhancement did not restart"); + + const lose = gl().getExtension("WEBGL_lose_context"); + assert(lose, "Context-loss test unavailable"); + const lost = event(canvas, "webglcontextlost"); + lose.loseContext(); + await lost; + assert(!pipeline.active, "Lost context did not return to raw video"); + // Restoration is only allowed after the lost event has finished dispatching. + await new Promise((resolve) => requestAnimationFrame(() => resolve())); + const restored = event(canvas, "webglcontextrestored"); + lose.restoreContext(); + await restored; + assert(pipeline.active, "Context restoration did not restart enhancement"); + assert(gl().getError() === gl().NO_ERROR, "WebGL error after context restoration"); + results.contextRestored = true; + + pipeline.reset(); + const resized = event(video, "loadeddata"); + video.src = "/fixtures/sd.mp4"; + await resized; + await video.play(); + await frames(4); + video.pause(); + pipeline.setPictureEnhancementEnabled(false); + pipeline.setPictureEnhancementEnabled(true); + assert(pipeline.active, "Enhancement did not survive source/resolution change"); + assert(gl().getError() === gl().NO_ERROR, "WebGL error after resolution change"); + results.resized = { source: [video.videoWidth, video.videoHeight], output: [canvas.width, canvas.height] }; + pipeline.destroy(); + for (const [kind, objects] of live) assert(objects.size === 0, `Leaked ${objects.size} ${kind} objects`); + results.resourcesAfterDestroy = snapshot(); + results.states = states; + results.draws = draws; + assert(errors.length === 0, `WebGL errors: ${JSON.stringify(errors)}`); + results.glErrors = errors; + return results; + } finally { + pipeline.destroy(); + video.pause(); + video.removeAttribute("src"); + video.load(); + container.remove(); + } +} diff --git a/tools/video-quality/main.ts b/tools/video-quality/main.ts new file mode 100644 index 00000000..6702c5bf --- /dev/null +++ b/tools/video-quality/main.ts @@ -0,0 +1,493 @@ +import { TemporalDenoiser } from "../../web-ui/src/playback-engine/render/denoise"; +import type { VideoFilter } from "../../web-ui/src/playback-engine/render/filters/types"; +import { FsrPresenter } from "../../web-ui/src/playback-engine/render/fsr"; +import { PassthroughPresenter, type Presenter } from "../../web-ui/src/playback-engine/render/presenters"; +import { invariants } from "./invariants"; +import { lifecycle } from "./lifecycle"; + +declare const __QUALITY_BASELINE__: string; + +type Mode = "raw" | "baseline" | "candidate" | "denoise"; +type Pixels = Uint8Array; +function required(value: T | null, description: string): T { + if (value === null) throw new Error(description); + return value; +} +const output = required(document.querySelector("#output"), "Output canvas missing"); +const status = required(document.querySelector("#status"), "Status element missing"); +const label = new URLSearchParams(location.search).get("label")?.replace(/[^\w-]/g, ""); +const resultPrefix = label ? `${label}-` : ""; +const gl = required( + output.getContext("webgl2", { alpha: false, antialias: false, preserveDrawingBuffer: true }), + "WebGL2 unavailable", +); +const denoiser = new TemporalDenoiser(); +denoiser.init(gl); +const candidate = new FsrPresenter(); +candidate.init(gl); +const raw = new PassthroughPresenter(); +raw.init(gl); +let baseline: Presenter; +let baselineNr: VideoFilter; +let baselineTarget: { texture: WebGLTexture; fbo: WebGLFramebuffer; width: number; height: number } | undefined; +const input = required(gl.createTexture(), "Texture allocation failed"); +gl.bindTexture(gl.TEXTURE_2D, input); +gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); +gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR); +gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); +gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); + +async function loadBaseline() { + if (baseline) return; + const fsrUrl = `${__QUALITY_BASELINE__}/fsr.ts`; + const nrUrl = `${__QUALITY_BASELINE__}/filters/mosquito-nr.ts`; + const module = await import(/* @vite-ignore */ fsrUrl); + const nrModule = await import(/* @vite-ignore */ nrUrl); + baseline = new module.FsrPresenter(); + baselineNr = new nrModule.MosquitoNrFilter(); + baseline.init(gl); + baselineNr.init(gl); +} + +function resize(width: number, height: number) { + if (output.width !== width) output.width = width; + if (output.height !== height) output.height = height; +} + +function upload(data: Pixels | TexImageSource, width: number, height: number) { + gl.activeTexture(gl.TEXTURE0); + gl.bindTexture(gl.TEXTURE_2D, input); + if (data instanceof Uint8Array) { + gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, width, height, 0, gl.RGBA, gl.UNSIGNED_BYTE, data); + } else gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, data); +} + +function draw(mode: Mode, width: number, height: number, outWidth: number, outHeight: number, flipY = false) { + let source = input; + if (mode === "baseline") { + if (!baselineTarget || baselineTarget.width !== width || baselineTarget.height !== height) { + if (baselineTarget) { + gl.deleteTexture(baselineTarget.texture); + gl.deleteFramebuffer(baselineTarget.fbo); + } + const texture = required(gl.createTexture(), "Texture allocation failed"); + const fbo = required(gl.createFramebuffer(), "Framebuffer allocation failed"); + gl.bindTexture(gl.TEXTURE_2D, texture); + gl.texStorage2D(gl.TEXTURE_2D, 1, gl.RGBA8, width, height); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); + gl.bindFramebuffer(gl.FRAMEBUFFER, fbo); + gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, texture, 0); + baselineTarget = { texture, fbo, width, height }; + } + gl.bindFramebuffer(gl.FRAMEBUFFER, baselineTarget.fbo); + gl.viewport(0, 0, width, height); + baselineNr.render(gl, [input], { width, height, flipY, keepField: 0, isSecondField: false, spatialOnly: true }); + source = baselineTarget.texture; + flipY = false; + } else if (mode === "candidate" || mode === "denoise") { + source = denoiser.render(gl, input, width, height, flipY); + flipY = false; + } + gl.bindFramebuffer(gl.FRAMEBUFFER, null); + gl.viewport(0, 0, outWidth, outHeight); + (mode === "raw" || mode === "denoise" ? raw : mode === "baseline" ? baseline : candidate).present( + gl, + source, + width, + height, + outWidth, + outHeight, + flipY, + ); +} + +function pixels(width: number, height: number, flipY = false): Pixels { + const data = new Uint8Array(width * height * 4); + gl.readPixels(0, 0, width, height, gl.RGBA, gl.UNSIGNED_BYTE, data); + if (flipY) { + const row = new Uint8Array(width * 4); + for (let y = 0; y < Math.floor(height / 2); y++) { + const top = y * row.length; + const bottom = (height - y - 1) * row.length; + row.set(data.subarray(top, top + row.length)); + data.copyWithin(top, bottom, bottom + row.length); + data.set(row, bottom); + } + } + return data; +} + +function scene(width: number, height: number, frame = 0, moving = false): Pixels { + const data = new Uint8Array(width * height * 4); + for (let y = 0; y < height; y++) + for (let x = 0; x < width; x++) { + const index = (y * width + x) * 4; + let color: number[]; + if (y < height / 3) color = [25 + (x / width) * 205, 25 + (x / width) * 205, 25 + (x / width) * 205]; + else if (x < width / 3) { + const v = x > (y - height / 3) * 0.42 ? 195 : 50; + color = [v, v * 0.9, v * 0.7]; + } else if (x < (width * 2) / 3) { + const v = 110 + Math.sin(x * 1.2) * Math.cos(y * 0.7) * 22; + color = [v * 0.7, v, v * 0.8]; + } else color = [110 + 18 * Math.sin(x * 0.04), 95 + 15 * Math.cos(y * 0.05), 160]; + const rectX = moving ? 12 + ((frame * 13) % Math.max(1, width - 60)) : -100; + if (x >= rectX && x < rectX + 35 && y > height * 0.2 && y < height * 0.8) color = [220, 70, 55]; + for (let c = 0; c < 3; c++) data[index + c] = Math.round(color[c]); + data[index + 3] = 255; + } + return data; +} + +function noisy(clean: Pixels, seed: number, amplitude = 6): Pixels { + const data = clean.slice(); + let state = seed | 0; + const random = () => { + state ^= state << 13; + state ^= state >>> 17; + state ^= state << 5; + return (state >>> 0) / 4294967296; + }; + for (let i = 0; i < data.length; i += 4) { + const noise = (random() + random() + random() + random() - 2) * amplitude * Math.sqrt(3); + for (let c = 0; c < 3; c++) + data[i + c] = Math.round(Math.max(0, Math.min(255, clean[i + c] + noise + (random() - 0.5) * amplitude))); + } + return data; +} + +function error(actual: Pixels, expected: Pixels) { + let squared = 0, + absolute = 0, + maximum = 0; + for (let i = 0; i < actual.length; i++) + if (i % 4 !== 3) { + const delta = Math.abs(actual[i] - expected[i]); + squared += delta * delta; + absolute += delta; + maximum = Math.max(maximum, delta); + } + const count = (actual.length / 4) * 3; + return { psnr: squared ? 10 * Math.log10((255 * 255) / (squared / count)) : 100, mae: absolute / count, maximum }; +} + +async function save(name: string, result: unknown) { + const text = JSON.stringify(result, null, 2); + const response = await fetch(`/results/${resultPrefix}${name}.json`, { method: "POST", body: text }); + if (!response.ok) throw new Error(`Saving ${name} failed: ${response.status}`); + status.textContent = text; +} + +async function savePng(name: string) { + const blob = await new Promise((resolve, reject) => + output.toBlob((b) => (b ? resolve(b) : reject(Error("PNG failed")))), + ); + const response = await fetch(`/results/${resultPrefix}${name}.png`, { method: "POST", body: blob }); + if (!response.ok) throw new Error(`Saving ${name} failed`); +} + +async function synthetic() { + await loadBaseline(); + const width = 384, + height = 216; + resize(width, height); + const checks: Record = {}; + for (const mode of ["raw", "baseline", "candidate", "denoise"] as const) { + const modeResult: Record = {}; + for (const [name, moving, amplitude] of [ + ["clean", false, 0], + ["noise", false, 6], + ["motion", true, 0], + ["noisy-motion", true, 6], + ] as const) { + denoiser.reset(); + let expected = scene(width, height); + for (let frame = 0; frame < 16; frame++) { + expected = scene(width, height, frame, moving); + const data = amplitude ? noisy(expected, 1009 + frame * 719, amplitude) : expected; + upload(data, width, height); + draw(mode, width, height, width, height); + } + modeResult[name] = error(pixels(width, height), expected); + await savePng(`synthetic-${name}-${mode}`); + } + let maximum = 0; + for (const color of [ + [0, 0, 0], + [255, 255, 255], + [4, 4, 4], + [128, 128, 128], + [251, 251, 251], + [200, 70, 30], + [40, 140, 190], + ]) { + denoiser.reset(); + const data = new Uint8Array(width * height * 4); + for (let i = 0; i < data.length; i += 4) data.set([...color, 255], i); + for (let frame = 0; frame < 8; frame++) { + upload(data, width, height); + draw(mode, width, height, width, height); + } + maximum = Math.max(maximum, error(pixels(width, height), data).maximum); + } + modeResult.flatColorMaximumError = maximum; + checks[mode] = modeResult; + } + checks.glError = gl.getError(); + await save("synthetic", { gpu: gpuInfo(), checks }); + return checks; +} + +function gpuInfo() { + const debug = gl.getExtension("WEBGL_debug_renderer_info"); + return { + renderer: debug ? gl.getParameter(debug.UNMASKED_RENDERER_WEBGL) : gl.getParameter(gl.RENDERER), + vendor: debug ? gl.getParameter(debug.UNMASKED_VENDOR_WEBGL) : gl.getParameter(gl.VENDOR), + timer: !!gl.getExtension("EXT_disjoint_timer_query_webgl2"), + floatHistory: !!gl.getExtension("EXT_color_buffer_float"), + maxTextureSize: gl.getParameter(gl.MAX_TEXTURE_SIZE), + userAgent: navigator.userAgent, + }; +} + +function distribution(values: number[]) { + const sorted = [...values].sort((a, b) => a - b); + return { + samples: values.length, + mean: values.reduce((a, b) => a + b, 0) / values.length, + median: sorted[Math.floor(sorted.length * 0.5)], + p95: sorted[Math.floor(sorted.length * 0.95)], + }; +} + +async function timing(width = 1920, height = 1080, outWidth = 3840, outHeight = 2160) { + await loadBaseline(); + resize(outWidth, outHeight); + upload(scene(width, height), width, height); + const ext = gl.getExtension("EXT_disjoint_timer_query_webgl2"); + const drawsPerSample = 8; + const order = ["raw", "baseline", "candidate"] as const; + const readings = Object.fromEntries( + order.map((mode) => [mode, { samples: [] as number[], submissions: [] as number[] }]), + ); + // Rotate the order across three rounds to distribute GPU clock/thermal drift. + for (let round = 0; round < 3; round++) + for (let position = 0; position < order.length; position++) { + const mode = order[(round + position) % order.length]; + const { samples, submissions } = readings[mode]; + denoiser.reset(); + for (let i = 0; i < 12; i++) draw(mode, width, height, outWidth, outHeight); + gl.finish(); + for (let i = 0; i < 15; i++) { + const query = ext ? gl.createQuery() : null; + if (query) gl.beginQuery(ext.TIME_ELAPSED_EXT, query); + const start = performance.now(); + for (let frame = 0; frame < drawsPerSample; frame++) draw(mode, width, height, outWidth, outHeight); + submissions.push((performance.now() - start) / drawsPerSample); + if (query) { + gl.endQuery(ext.TIME_ELAPSED_EXT); + gl.flush(); + const deadline = performance.now() + 5000; + while (!gl.getQueryParameter(query, gl.QUERY_RESULT_AVAILABLE)) { + if (performance.now() > deadline || gl.isContextLost()) { + gl.deleteQuery(query); + throw Error("GPU timing query did not complete"); + } + await new Promise((r) => setTimeout(r, 0)); + } + if (!gl.getParameter(ext.GPU_DISJOINT_EXT)) + samples.push(gl.getQueryParameter(query, gl.QUERY_RESULT) / 1e6 / drawsPerSample); + gl.deleteQuery(query); + } else { + gl.finish(); + samples.push((performance.now() - start) / drawsPerSample); + } + if (i % 5 === 0) await new Promise((r) => setTimeout(r, 0)); + } + } + const modes: Record = {}; + for (const mode of order) { + const { samples, submissions } = readings[mode]; + if (samples.length < 30) throw Error(`Only ${samples.length} valid ${mode} timing samples`); + modes[mode] = { + measurement: ext ? "GPU timer ms" : "synchronous CPU+GPU completion ms", + render: distribution(samples), + submission: distribution(submissions), + samples, + }; + } + const result = { + gpu: gpuInfo(), + source: [width, height], + output: [outWidth, outHeight], + drawsPerSample, + modes, + glError: gl.getError(), + }; + await save(`timing-${width}x${height}-${outWidth}x${outHeight}`, result); + return result; +} + +async function compare(clip = "sport", time = 3, outWidth = 2560, outHeight = 1440) { + if (time < 11 / 50) throw Error("Comparison needs eleven preceding 50 fps frames"); + await loadBaseline(); + const video = document.createElement("video"); + video.muted = true; + video.playsInline = true; + video.style.cssText = "position:fixed;width:16px;height:9px;opacity:0;pointer-events:none"; + document.body.append(video); + try { + video.src = `/fixtures/${clip}.mp4`; + await new Promise((resolve, reject) => { + const timeout = setTimeout(() => reject(Error("Recording load timed out")), 10000); + video.onloadeddata = () => { + clearTimeout(timeout); + resolve(); + }; + video.onerror = () => { + clearTimeout(timeout); + reject(Error("Cannot load recording")); + }; + }); + const width = video.videoWidth, + height = video.videoHeight; + resize(outWidth, outHeight); + const timestamps: number[] = []; + const seek = async (t: number) => { + await new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + video.cancelVideoFrameCallback(handle); + reject(Error(`No decoded frame for ${t}s`)); + }, 5000); + const handle = video.requestVideoFrameCallback((_now, metadata) => { + clearTimeout(timeout); + timestamps.push(metadata.mediaTime); + if (Math.abs(metadata.mediaTime - t) > 0.001) + reject(Error(`Requested ${t}s, decoded ${metadata.mediaTime}s`)); + else resolve(); + }); + video.currentTime = t; + }); + }; + for (const mode of ["raw", "baseline", "candidate"] as const) { + denoiser.reset(); + // Independent prior frames, not repeated observations of the same image. + for (let frame = 11; frame >= 0; frame--) { + await seek(time - frame / 50); + upload(video, width, height); + draw(mode, width, height, outWidth, outHeight, true); + } + await savePng(`${clip}-${time}-${mode}`); + } + const result = { + clip, + time, + timestamps, + source: [width, height], + output: [outWidth, outHeight], + glError: gl.getError(), + }; + await save(`${clip}-${time}`, result); + return result; + } finally { + video.removeAttribute("src"); + video.load(); + video.remove(); + } +} + +async function restoration(clip = "sport", amplitude = 0) { + await loadBaseline(); + const response = await fetch(`/fixtures/${clip}-reference.png`); + if (!response.ok) throw Error("Reference PNG unavailable"); + const reference = await createImageBitmap(await response.blob()); + const width = reference.width / 2, + height = reference.height / 2; + const canvas = document.createElement("canvas"); + canvas.width = reference.width; + canvas.height = reference.height; + const ctx = canvas.getContext("2d"); + if (!ctx) throw Error("2D canvas unavailable"); + ctx.drawImage(reference, 0, 0); + const expected = new Uint8Array(ctx.getImageData(0, 0, canvas.width, canvas.height).data); + reference.close(); + const downsampled = new Uint8Array(width * height * 4); + // Exact 2x2 area reduction, identical for every algorithm and browser. + for (let y = 0; y < height; y++) + for (let x = 0; x < width; x++) { + const i = (y * width + x) * 4; + const p = (y * 2 * canvas.width + x * 2) * 4; + for (let c = 0; c < 3; c++) + downsampled[i + c] = Math.round( + (expected[p + c] + + expected[p + 4 + c] + + expected[p + canvas.width * 4 + c] + + expected[p + canvas.width * 4 + 4 + c]) / + 4, + ); + downsampled[i + 3] = 255; + } + resize(canvas.width, canvas.height); + const checks: Record = {}; + for (const mode of ["raw", "baseline", "candidate"] as const) { + denoiser.reset(); + for (let frame = 0; frame < 16; frame++) { + upload(amplitude ? noisy(downsampled, 1009 + frame * 719, amplitude) : downsampled, width, height); + draw(mode, width, height, canvas.width, canvas.height, true); + } + checks[mode] = error(pixels(canvas.width, canvas.height, true), expected); + await savePng(`restoration-${clip}-${amplitude}-${mode}`); + } + await save(`restoration-${clip}-${amplitude}`, { + clip, + amplitude, + source: [width, height], + output: [canvas.width, canvas.height], + checks, + glError: gl.getError(), + }); + return checks; +} + +const lab = { + synthetic, + timing, + compare, + restoration, + gpuInfo, + invariants: async () => { + const result = invariants(); + await save("invariants", result); + return result; + }, + lifecycle: async () => { + const result = await lifecycle(); + await save("lifecycle", result); + return result; + }, +}; +Object.assign(window, { qualityLab: lab }); +function run(task: () => Promise) { + status.textContent = "Running…"; + void task().catch((error) => { + status.textContent = String(error.stack ?? error); + console.error(error); + }); +} +required(document.querySelector("#synthetic"), "Test button missing").addEventListener("click", () => run(synthetic)); +required(document.querySelector("#timing"), "Timing button missing").addEventListener("click", () => + run(() => timing()), +); +required(document.querySelector("#compare"), "Compare button missing").addEventListener("click", () => + run(() => compare(required(document.querySelector("#clip"), "Clip selector missing").value)), +); +required(document.querySelector("#invariants"), "Invariant button missing").addEventListener("click", () => + run(lab.invariants), +); +required(document.querySelector("#lifecycle"), "Lifecycle button missing").addEventListener("click", () => + run(lab.lifecycle), +); +status.textContent = JSON.stringify(gpuInfo(), null, 2); diff --git a/tools/video-quality/measurements.json b/tools/video-quality/measurements.json new file mode 100644 index 00000000..101ab6a0 --- /dev/null +++ b/tools/video-quality/measurements.json @@ -0,0 +1,653 @@ +{ + "date": "2026-09-08", + "hardware": "Apple M3 Max, 40 GPU cores, 128 GB unified memory", + "baseline": "3bf4ca8c5521eebcd8be76c387c3ecbe904d55ac", + "candidateSource": { + "web-ui/src/playback-engine/render/denoise.ts": "ba4f30b8c606be5546b9866b5fcf94bfdedae8b3166fc6afd4b60391ae4553d9", + "web-ui/src/playback-engine/render/fsr.ts": "94d1aae020b798e3721a6e65bb5d4f8b8a3412c0b2d150ab660faf4e57117c59", + "web-ui/src/playback-engine/render/renderer.ts": "bcbb4ae3dcdba34489b7e6b99ba4b5e98674535f16fc81ed0c5b3163ebd47127", + "web-ui/src/playback-engine/render/filters/gl-utils.ts": "7a18e9f87acbcec95274990a7da5459c93ef06f35680e0fde43959016a6b12d0", + "web-ui/src/playback-engine/render/index.ts": "0b2546efeecf632e7954ba74319ecf3b6cffbcf6733a7f586210b2d61d967ab6" + }, + "fixtureSha256": { + "sport.mp4": "4857d456739e47d27a23817e45a56770386475a67da2ed261788ebf077e59085", + "film.mp4": "79670586664326d3b437a7eb2564640c68ce22f93e4fb33d4e4047b8388122e3", + "sport-interlaced.mp4": "817f3eba07c88c7d2b7a7d1be418d2c3519bed5256abb50af817dd1846257d94", + "sd.mp4": "a5e11114710250aa3a807ca20afe5178f7c94b6b8e556071a908c1cd9e60989f", + "sport-reference.png": "fbe10419dc048cffb65a75def2623a443272d400763ff4a089c1832115767c22", + "film-reference.png": "9cd86d702de1553ab4319b0e9e426ebf3a8c2696fa2505cdd194870fe39a72d1" + }, + "synthetic": { + "gpu": { + "renderer": "ANGLE (Apple, ANGLE Metal Renderer: Apple M3 Max, Unspecified Version)", + "vendor": "Google Inc. (Apple)", + "timer": true, + "floatHistory": true, + "maxTextureSize": 16384, + "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/152.0.0.0 Safari/537.36" + }, + "checks": { + "raw": { + "clean": { + "psnr": 100, + "mae": 0, + "maximum": 0 + }, + "noise": { + "psnr": 32.203139923584125, + "mae": 5.038487815072016, + "maximum": 22 + }, + "motion": { + "psnr": 100, + "mae": 0, + "maximum": 0 + }, + "noisy-motion": { + "psnr": 32.203139923584125, + "mae": 5.038487815072016, + "maximum": 22 + }, + "flatColorMaximumError": 0 + }, + "baseline": { + "clean": { + "psnr": 38.06231511646425, + "mae": 2.5045814043209877, + "maximum": 49 + }, + "noise": { + "psnr": 28.480429866217975, + "mae": 8.02073286394033, + "maximum": 54 + }, + "motion": { + "psnr": 37.34026542757714, + "mae": 2.650688014403292, + "maximum": 49 + }, + "noisy-motion": { + "psnr": 28.397426539390963, + "mae": 8.06494743441358, + "maximum": 54 + }, + "flatColorMaximumError": 6 + }, + "candidate": { + "clean": { + "psnr": 46.938336386590684, + "mae": 0.4383278677983539, + "maximum": 8 + }, + "noise": { + "psnr": 34.52463292381823, + "mae": 3.6404602301954734, + "maximum": 24 + }, + "motion": { + "psnr": 48.05744209920781, + "mae": 0.3478531700102881, + "maximum": 8 + }, + "noisy-motion": { + "psnr": 34.35470092112995, + "mae": 3.7281418788580245, + "maximum": 23 + }, + "flatColorMaximumError": 0 + }, + "denoise": { + "clean": { + "psnr": 82.45198763760479, + "mae": 0.00036972736625514403, + "maximum": 1 + }, + "noise": { + "psnr": 35.2309005634998, + "mae": 3.3373480902777777, + "maximum": 21 + }, + "motion": { + "psnr": 72.84707305044152, + "mae": 0.0033757716049382714, + "maximum": 1 + }, + "noisy-motion": { + "psnr": 35.0518931818319, + "mae": 3.421609760802469, + "maximum": 21 + }, + "flatColorMaximumError": 0 + }, + "glError": 0 + } + }, + "invariants": { + "preferred": { + "flatColorMaximum": 0, + "flippedHistory": { + "maximum": 0, + "mae": 0 + }, + "sceneCut": { + "maximum": 2, + "mae": 0.014323968880226901 + }, + "low-contrast-1px": { + "maximum": 0, + "mae": 0 + }, + "low-contrast-7px": { + "maximum": 0, + "mae": 0 + }, + "isoluminant-1px": { + "maximum": 0, + "mae": 0 + }, + "isoluminant-7px": { + "maximum": 0, + "mae": 0 + }, + "historyFormat": "RGBA16F", + "source": [193, 109] + }, + "fallback": { + "flatColorMaximum": 0, + "flippedHistory": { + "maximum": 0, + "mae": 0 + }, + "sceneCut": { + "maximum": 2, + "mae": 0.014450729666777582 + }, + "low-contrast-1px": { + "maximum": 0, + "mae": 0 + }, + "low-contrast-7px": { + "maximum": 0, + "mae": 0 + }, + "isoluminant-1px": { + "maximum": 0, + "mae": 0 + }, + "isoluminant-7px": { + "maximum": 0, + "mae": 0 + }, + "historyFormat": "RGBA8", + "source": [193, 109] + } + }, + "restoration": { + "sportClean": { + "clip": "sport", + "amplitude": 0, + "source": [960, 540], + "output": [1920, 1080], + "checks": { + "raw": { + "psnr": 34.805624607002024, + "mae": 1.2843205054012345, + "maximum": 100 + }, + "baseline": { + "psnr": 34.33075123014509, + "mae": 3.491092785493827, + "maximum": 110 + }, + "candidate": { + "psnr": 36.84825968489091, + "mae": 1.0314226466049383, + "maximum": 110 + } + }, + "glError": 0 + }, + "sportNoisy": { + "clip": "sport", + "amplitude": 6, + "source": [960, 540], + "output": [1920, 1080], + "checks": { + "raw": { + "psnr": 32.48447106294431, + "mae": 3.854054944701646, + "maximum": 101 + }, + "baseline": { + "psnr": 30.36909461543647, + "mae": 5.948925057870371, + "maximum": 106 + }, + "candidate": { + "psnr": 33.65924349128982, + "mae": 3.3803742283950617, + "maximum": 109 + } + }, + "glError": 0 + }, + "filmClean": { + "clip": "film", + "amplitude": 0, + "source": [960, 540], + "output": [1920, 1080], + "checks": { + "raw": { + "psnr": 33.51179913453376, + "mae": 2.265140335648148, + "maximum": 135 + }, + "baseline": { + "psnr": 33.5124439361545, + "mae": 3.5712315136316874, + "maximum": 141 + }, + "candidate": { + "psnr": 35.190036193647536, + "mae": 1.8991718106995885, + "maximum": 134 + } + }, + "glError": 0 + }, + "filmNoisy": { + "clip": "film", + "amplitude": 6, + "source": [960, 540], + "output": [1920, 1080], + "checks": { + "raw": { + "psnr": 31.777167095062715, + "mae": 4.195530156893004, + "maximum": 139 + }, + "baseline": { + "psnr": 30.739252806047283, + "mae": 5.199043852880658, + "maximum": 140 + }, + "candidate": { + "psnr": 32.85489923904128, + "mae": 3.7249528999485597, + "maximum": 145 + } + }, + "glError": 0 + } + }, + "timing": { + "native": { + "gpu": { + "renderer": "ANGLE (Apple, ANGLE Metal Renderer: Apple M3 Max, Unspecified Version)", + "vendor": "Google Inc. (Apple)", + "timer": true, + "floatHistory": true, + "maxTextureSize": 16384, + "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/152.0.0.0 Safari/537.36" + }, + "source": [1920, 1080], + "output": [1920, 1080], + "drawsPerSample": 8, + "modes": { + "raw": { + "measurement": "GPU timer ms", + "render": { + "samples": 45, + "mean": 0.07545317500000001, + "median": 0.063125, + "p95": 0.12784375 + }, + "submission": { + "samples": 45, + "mean": 0.0025000002649095323, + "median": 0, + "p95": 0.012500002980232239 + }, + "samples": [ + 0.1222655, 0.09779675, 0.059625, 0.059203, 0.058812375, 0.059177, 0.09249475, 0.05902075, 0.0595, 0.0589635, + 0.05920825, 0.113880125, 0.12784375, 0.058572875, 0.058906125, 0.107171875, 0.03598425, 0.036124875, + 0.11496875, 0.066901, 0.13952075, 0.036354125, 0.036442625, 0.070359375, 0.121328125, 0.058385375, + 0.03603125, 0.063125, 0.0663125, 0.122156125, 0.063828, 0.0403385, 0.14136975, 0.11223425, 0.042156125, + 0.040031125, 0.105140625, 0.094802, 0.06961975, 0.05211975, 0.039828, 0.064499875, 0.11867175, 0.11264575, + 0.041671875 + ] + }, + "baseline": { + "measurement": "GPU timer ms", + "render": { + "samples": 45, + "mean": 0.5519924333333334, + "median": 0.515776, + "p95": 1.086135375 + }, + "submission": { + "samples": 45, + "mean": 0.007500000463591681, + "median": 0.012499988079071045, + "p95": 0.024999991059303284 + }, + "samples": [ + 1.175166625, 0.760046875, 0.756510375, 0.809609375, 0.494192625, 0.533854125, 0.374640625, 0.371078125, + 0.515776, 0.518578125, 0.436161375, 0.447135375, 0.5246405, 0.31075, 0.4398125, 1.086135375, 0.646328125, + 0.584078, 0.459802, 0.62403125, 0.569776, 0.578640625, 0.651104125, 0.478776, 0.61348425, 0.437828125, + 0.474052, 0.667453125, 0.662348875, 0.66995825, 1.10773425, 0.341171875, 0.445203125, 0.3675885, 0.3823905, + 0.411052, 0.341046875, 0.321541625, 0.4995, 0.338536375, 0.520479125, 0.5297395, 0.505541625, 0.379760375, + 0.676625 + ] + }, + "candidate": { + "measurement": "GPU timer ms", + "render": { + "samples": 45, + "mean": 0.7438852416666667, + "median": 0.720124875, + "p95": 0.9624895 + }, + "submission": { + "samples": 45, + "mean": 0.0058333333995607166, + "median": 0, + "p95": 0.012500002980232239 + }, + "samples": [ + 0.845437375, 0.493942625, 0.53952075, 0.4960885, 0.372979125, 0.418447875, 0.498536375, 0.473874875, + 0.84460925, 0.85990625, 0.650677, 0.676901, 0.838781125, 0.8735, 0.9624895, 1.778, 0.9285885, 0.877156125, + 0.773031125, 0.8802395, 0.732947875, 0.676911375, 0.7093905, 0.833552, 0.720947875, 0.788187375, + 0.673609375, 0.712130125, 0.737567625, 0.82939575, 2.032609375, 0.88336975, 0.738296875, 0.57825, + 0.720124875, 0.63284375, 0.584531125, 0.78659375, 0.748651, 0.527010375, 0.4807395, 0.552328125, 0.63290625, + 0.48090625, 0.598328125 + ] + } + }, + "glError": 0 + }, + "uhd": { + "gpu": { + "renderer": "ANGLE (Apple, ANGLE Metal Renderer: Apple M3 Max, Unspecified Version)", + "vendor": "Google Inc. (Apple)", + "timer": true, + "floatHistory": true, + "maxTextureSize": 16384, + "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/152.0.0.0 Safari/537.36" + }, + "source": [1920, 1080], + "output": [3840, 2160], + "drawsPerSample": 8, + "modes": { + "raw": { + "measurement": "GPU timer ms", + "render": { + "samples": 45, + "mean": 0.1718023527777778, + "median": 0.14425, + "p95": 0.312609375 + }, + "submission": { + "samples": 45, + "mean": 0.0016666667328940497, + "median": 0, + "p95": 0.012500002980232239 + }, + "samples": [ + 0.3828905, 0.2223905, 0.345078, 0.288473875, 0.216328, 0.1631405, 0.210656125, 0.25489575, 0.292781125, + 0.242671875, 0.183942625, 0.218302, 0.25484375, 0.27186975, 0.312609375, 0.15024475, 0.096156125, + 0.081624875, 0.079749875, 0.0798385, 0.085041625, 0.108052, 0.085312375, 0.09602075, 0.106343625, + 0.137161375, 0.110312375, 0.108151, 0.111765625, 0.130104125, 0.254703125, 0.14425, 0.1467395, 0.132421875, + 0.1324375, 0.13223425, 0.1297655, 0.161468625, 0.140687375, 0.169578, 0.1325, 0.131968625, 0.157104125, + 0.13417175, 0.204322875 + ] + }, + "baseline": { + "measurement": "GPU timer ms", + "render": { + "samples": 45, + "mean": 2.350057577777778, + "median": 1.924604125, + "p95": 3.454473875 + }, + "submission": { + "samples": 45, + "mean": 0.003333333796925015, + "median": 0, + "p95": 0.012500002980232239 + }, + "samples": [ + 3.693651, 1.77683325, 1.790697875, 1.758156125, 1.796161375, 1.820135375, 1.878276, 1.8271875, 1.90253125, + 1.919161375, 1.8978645, 1.76898425, 2.018718625, 1.942510375, 1.61574475, 3.454473875, 1.661, 1.789437375, + 1.67289575, 1.69730725, 1.693536375, 1.799125, 1.7893125, 1.862052, 1.979541625, 1.46833325, 1.924604125, + 2.004848875, 1.62883325, 2.03534375, 6.12679675, 2.268374875, 3.25427075, 3.2524635, 3.10892175, + 3.247354125, 3.04949475, 3.051697875, 3.24898425, 2.848046875, 3.066859375, 2.99389575, 2.537416625, + 2.845692625, 2.9850625 + ] + }, + "candidate": { + "measurement": "GPU timer ms", + "render": { + "samples": 45, + "mean": 2.608560241666667, + "median": 2.0657395, + "p95": 4.104484375 + }, + "submission": { + "samples": 45, + "mean": 0.006111110581292046, + "median": 0, + "p95": 0.024999991059303284 + }, + "samples": [ + 4.43070825, 1.678625, 1.96009375, 2.01899475, 1.612197875, 2.036005125, 2.132953, 1.66525, 2.2785625, + 1.897276, 1.911453, 1.85521875, 1.93510925, 1.956661375, 1.96246875, 3.38727075, 1.966078125, 1.96123425, + 1.723104125, 2.08346875, 1.99727075, 1.9066875, 2.021249875, 2.31770825, 2.0657395, 1.990609375, + 1.951010375, 2.24617175, 1.85092175, 1.944135375, 4.62777075, 4.093526, 4.104484375, 3.65836975, + 2.852661375, 3.369124875, 3.90278125, 2.75818225, 3.5477395, 3.44790625, 3.538999875, 3.705760375, + 3.69629675, 3.5682135, 3.769156125 + ] + } + }, + "glError": 0 + }, + "sd": { + "gpu": { + "renderer": "ANGLE (Apple, ANGLE Metal Renderer: Apple M3 Max, Unspecified Version)", + "vendor": "Google Inc. (Apple)", + "timer": true, + "floatHistory": true, + "maxTextureSize": 16384, + "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/152.0.0.0 Safari/537.36" + }, + "source": [720, 576], + "output": [1920, 1080], + "drawsPerSample": 8, + "modes": { + "raw": { + "measurement": "GPU timer ms", + "render": { + "samples": 45, + "mean": 0.057101088888888894, + "median": 0.048317625, + "p95": 0.104109375 + }, + "submission": { + "samples": 45, + "mean": 0.001388888888888889, + "median": 0, + "p95": 0.012500002980232239 + }, + "samples": [ + 0.054625, 0.027921875, 0.052953, 0.03530725, 0.029359375, 0.034453125, 0.060755125, 0.0430155, 0.035927, + 0.035526, 0.035609375, 0.0455885, 0.05211975, 0.035505125, 0.035723875, 0.079578, 0.07604675, 0.045927, + 0.048317625, 0.046625, 0.04343225, 0.073874875, 0.096802, 0.06204675, 0.04652075, 0.036005125, 0.061692625, + 0.093229125, 0.046656125, 0.0659635, 0.08477075, 0.052781125, 0.0865, 0.048666625, 0.11602075, 0.066234375, + 0.035776, 0.039374875, 0.0950155, 0.064078, 0.104109375, 0.037625, 0.037036375, 0.0412655, 0.123187375 + ] + }, + "baseline": { + "measurement": "GPU timer ms", + "render": { + "samples": 45, + "mean": 1.112544961111111, + "median": 1.06196875, + "p95": 1.360026 + }, + "submission": { + "samples": 45, + "mean": 0.005555555886692471, + "median": 0, + "p95": 0.012500002980232239 + }, + "samples": [ + 1.4054375, 0.919151, 0.976052, 0.801947875, 0.78511975, 0.8527395, 0.894921875, 1.038562375, 0.756828, + 1.122796875, 0.941348875, 0.967203, 1.166203125, 0.85998425, 1.01484375, 1.28564575, 0.88679675, + 1.103687375, 1.249218625, 1.06196875, 1.159859375, 1.0757395, 0.885385375, 1.20924475, 1.096390625, + 1.01746875, 0.9155, 0.955177, 1.212479125, 1.013281125, 2.97745825, 1.287526, 1.093130125, 1.0389635, + 1.2565155, 1.1247135, 1.1692395, 1.3071145, 1.047906125, 1.360026, 1.284729125, 0.9956405, 1.161302, + 1.3351875, 0.9940885 + ] + }, + "candidate": { + "measurement": "GPU timer ms", + "render": { + "samples": 45, + "mean": 1.2410100055555553, + "median": 1.16075, + "p95": 2.07361975 + }, + "submission": { + "samples": 45, + "mean": 0.0063888887564341225, + "median": 0, + "p95": 0.012500002980232239 + }, + "samples": [ + 2.07361975, 1.14489575, 1.191416625, 1.006578, 0.921104125, 1.158192625, 0.876156125, 1.350999875, + 1.08386975, 1.112260375, 0.9554375, 1.115796875, 1.193749875, 1.252401, 0.869677, 2.531260375, 1.0895, + 0.98220825, 1.293776, 1.0742655, 1.28795825, 1.129812375, 0.856921875, 1.194854125, 1.191598875, 1.02753125, + 1.232942625, 1.231749875, 1.173067625, 1.3535, 3.23375, 1.132578, 1.493322875, 1.38318225, 1.09402075, + 1.1995155, 0.933421875, 1.069723875, 1.305541625, 1.16075, 1.125374875, 1.18368225, 1.27389575, 1.000041625, + 1.29954675 + ] + } + }, + "glError": 0 + } + }, + "lifecycle": { + "interlaced": { + "callbacks": 25, + "presentations": 50, + "mediaTime": 1.2, + "qualityBefore": { + "total": 9, + "dropped": 0, + "corrupted": 0 + }, + "qualityAfter": { + "total": 34, + "dropped": 0, + "corrupted": 0 + } + }, + "pausedToggles": { + "repeats": 8, + "resources": { + "Texture": 8, + "Framebuffer": 5, + "Program": 5, + "Shader": 0 + } + }, + "seek": 4, + "contextRestored": true, + "resized": { + "source": [1024, 576], + "output": [2560, 1440] + }, + "resourcesAfterDestroy": { + "Texture": 0, + "Framebuffer": 0, + "Program": 0, + "Shader": 0 + }, + "states": [ + { + "active": false, + "deinterlacing": false + }, + { + "active": true, + "deinterlacing": true + }, + { + "active": true, + "deinterlacing": false + }, + { + "active": false, + "deinterlacing": false + }, + { + "active": true, + "deinterlacing": false + }, + { + "active": false, + "deinterlacing": false + }, + { + "active": true, + "deinterlacing": false + }, + { + "active": false, + "deinterlacing": false + }, + { + "active": true, + "deinterlacing": false + } + ], + "draws": 337, + "glErrors": [] + }, + "liveMse": { + "source": [1920, 1080], + "native": { + "output": [1920, 1080], + "before": { + "mediaTime": 123.583979, + "totalFrames": 3093, + "dropped": 0, + "corrupted": 0 + }, + "after": { + "mediaTime": 138.585005, + "totalFrames": 3468, + "dropped": 0, + "corrupted": 0 + }, + "glError": 0 + }, + "uhd": { + "output": [3840, 2160], + "before": { + "mediaTime": 190.755053, + "totalFrames": 4785, + "dropped": 0, + "corrupted": 0 + }, + "after": { + "mediaTime": 205.755737, + "totalFrames": 5160, + "dropped": 0, + "corrupted": 0 + }, + "glError": 0 + } + }, + "unverified": ["Safari/WebKit", "Physical iPad", "Motion-compensated frame interpolation"] +} diff --git a/tools/video-quality/prepare-baseline.mjs b/tools/video-quality/prepare-baseline.mjs new file mode 100644 index 00000000..c5ccfb69 --- /dev/null +++ b/tools/video-quality/prepare-baseline.mjs @@ -0,0 +1,24 @@ +import { execFileSync } from "node:child_process"; +import { mkdir, writeFile } from "node:fs/promises"; +import { resolve } from "node:path"; + +const repo = resolve(import.meta.dirname, "../.."); +const ref = process.argv[2] ?? "3bf4ca8c5521eebcd8be76c387c3ecbe904d55ac"; +const revision = execFileSync("git", ["rev-parse", "--verify", `${ref}^{commit}`], { + cwd: repo, + encoding: "utf8", +}).trim(); +const output = resolve(repo, "build-video-quality/baseline"); +await mkdir(resolve(output, "filters"), { recursive: true }); +for (const path of ["fsr.ts", "presenters.ts", "filters/gl-utils.ts", "filters/types.ts", "filters/mosquito-nr.ts"]) { + let source = execFileSync("git", ["show", `${revision}:web-ui/src/playback-engine/render/${path}`], { + cwd: repo, + encoding: "utf8", + }); + // Expose the original class to the lab without registering a competing filter. + if (path.endsWith("mosquito-nr.ts")) + source = source.replace("class MosquitoNrFilter", "export class MosquitoNrFilter"); + await writeFile(resolve(output, path), source); +} +await writeFile(resolve(output, "revision.json"), JSON.stringify({ revision }, null, 2)); +console.log(`Prepared video enhancement baseline ${revision}`); diff --git a/tools/video-quality/vite.config.ts b/tools/video-quality/vite.config.ts new file mode 100644 index 00000000..4ed476c0 --- /dev/null +++ b/tools/video-quality/vite.config.ts @@ -0,0 +1,60 @@ +import { createReadStream } from "node:fs"; +import { mkdir, stat, writeFile } from "node:fs/promises"; +import { resolve } from "node:path"; +import { defineConfig } from "vite"; + +const repo = resolve(import.meta.dirname, "../.."); +const artifacts = resolve(repo, "build-video-quality"); + +export default defineConfig({ + define: { __QUALITY_BASELINE__: JSON.stringify(`/@fs/${artifacts}/baseline`) }, + server: { host: "127.0.0.1", port: 5186, strictPort: true, fs: { allow: [repo] } }, + plugins: [ + { + name: "quality-artifacts", + configureServer(server) { + server.middlewares.use(async (req, res, next) => { + const match = req.url?.match(/^\/(fixtures|results)\/([\w.-]+)$/); + if (!match) return next(); + const [, directory, name] = match; + const path = resolve(artifacts, directory, name); + if (req.method === "POST" && directory === "results") { + const chunks: Buffer[] = []; + let size = 0; + for await (const chunk of req) { + size += chunk.length; + if (size > 80 * 1024 * 1024) { + res.writeHead(413).end(); + return; + } + chunks.push(chunk); + } + await mkdir(resolve(artifacts, directory), { recursive: true }); + await writeFile(path, Buffer.concat(chunks)); + res.writeHead(201).end(); + } else if (req.method === "GET" || req.method === "HEAD") { + const info = await stat(path).catch(() => null); + if (!info?.isFile()) return res.writeHead(404).end(); + const range = req.headers.range?.match(/^bytes=(\d+)-(\d*)$/); + const start = range ? Number(range[1]) : 0; + const end = range?.[2] ? Math.min(Number(range[2]), info.size - 1) : info.size - 1; + if (start > end || start >= info.size) return res.writeHead(416).end(); + res.setHeader("Accept-Ranges", "bytes"); + res.setHeader("Content-Length", end - start + 1); + if (range) { + res.statusCode = 206; + res.setHeader("Content-Range", `bytes ${start}-${end}/${info.size}`); + } + if (name.endsWith(".mp4")) res.setHeader("Content-Type", "video/mp4"); + if (name.endsWith(".png")) res.setHeader("Content-Type", "image/png"); + if (req.method === "HEAD") return res.end(); + const stream = createReadStream(path, { start, end }); + res.on("close", () => stream.destroy()); + stream.on("error", () => res.destroy()); + stream.pipe(res); + } else res.writeHead(405).end(); + }); + }, + }, + ], +}); diff --git a/web-ui/src/playback-engine/render/denoise.ts b/web-ui/src/playback-engine/render/denoise.ts new file mode 100644 index 00000000..d1600cba --- /dev/null +++ b/web-ui/src/playback-engine/render/denoise.ts @@ -0,0 +1,237 @@ +import { createProgram, FRAMEBUFFER_VERTEX_SHADER } from "./filters/gl-utils"; + +/** + * Motion-adaptive, recursive video denoising at source resolution. RGB stores + * the filtered image; alpha stores the ORIGINAL luma. Comparing original luma + * across a five-pixel patch distinguishes changing detail from random noise, + * without an extra copy pass or a second history attachment. + * + * History is rejected on moving edges and clipped to the current neighborhood + * before blending. Chroma also participates in rejection (isoluminant motion). + * There is no lookahead, motion extrapolation, or added presentation latency. + */ +const FRAGMENT_SHADER = /*glsl*/ `#version 300 es +precision highp float; +precision highp sampler2D; + +uniform sampler2D u_input; +uniform sampler2D u_history; +uniform vec2 u_texelSize; +uniform bool u_hasHistory; +uniform bool u_flipY; + +in vec2 v_texCoord; +out vec4 outColor; + +const vec3 LUMA = vec3(0.2126, 0.7152, 0.0722); + +vec3 toYcc(vec3 rgb) { + float y = dot(rgb, LUMA); + return vec3(y, (rgb.b - y) / 1.8556, (rgb.r - y) / 1.5748); +} + +vec3 toRgb(vec3 ycc) { + float r = ycc.x + 1.5748 * ycc.z; + float b = ycc.x + 1.8556 * ycc.y; + return vec3(r, (ycc.x - LUMA.x * r - LUMA.z * b) / LUMA.y, b); +} + +void main() { + vec2 uv = v_texCoord; + vec2 dx = vec2(u_texelSize.x, 0.0); + vec2 dy = vec2(0.0, u_texelSize.y); + vec3 p[9]; + p[0] = toYcc(texture(u_input, uv - dx - dy).rgb); + p[1] = toYcc(texture(u_input, uv - dy).rgb); + p[2] = toYcc(texture(u_input, uv + dx - dy).rgb); + p[3] = toYcc(texture(u_input, uv - dx).rgb); + p[4] = toYcc(texture(u_input, uv).rgb); + p[5] = toYcc(texture(u_input, uv + dx).rgb); + p[6] = toYcc(texture(u_input, uv - dx + dy).rgb); + p[7] = toYcc(texture(u_input, uv + dy).rgb); + p[8] = toYcc(texture(u_input, uv + dx + dy).rgb); + vec3 c = p[4]; + + // History is framebuffer-backed, even when the input is a DOM upload. + vec2 huv = u_flipY ? vec2(uv.x, 1.0 - uv.y) : uv; + vec2 hdy = u_flipY ? -dy : dy; + vec4 history = texture(u_history, huv); + vec3 h = toYcc(history.rgb); + float d0 = c.x - history.a; + float dn = p[1].x - texture(u_history, huv - hdy).a; + float dw = p[3].x - texture(u_history, huv - dx).a; + float de = p[5].x - texture(u_history, huv + dx).a; + float ds = p[7].x - texture(u_history, huv + hdy).a; + float meanDelta = (d0 * 2.0 + dn + dw + de + ds) / 6.0; + float deltaVariance = max(0.0, (2.0*d0*d0 + dn*dn + dw*dw + de*de + ds*ds) / 6.0 - meanDelta*meanDelta); + float sigma = u_hasHistory ? clamp(sqrt(deltaVariance * 0.5), 0.0, 0.035) : 0.012; + + // Opposing differences reveal translating edges even if their average cancels. + float structureDelta = max(abs(de - dw), abs(ds - dn)); + float motion = abs(meanDelta); + float confidence = 1.0 - smoothstep(0.008 + sigma * 0.35, 0.024 + sigma * 0.65, motion); + confidence *= 1.0 - smoothstep(0.020 + sigma * 1.5, 0.060 + sigma * 2.0, structureDelta); + confidence *= 1.0 - smoothstep(0.025 + sigma, 0.070 + sigma, abs(d0)); + // A weak translating edge changes the center and at least two neighbors + // in the same direction. Random grain rarely agrees at all three pixels. + // This catches low-contrast motion without rejecting isolated noise peaks. + vec4 aligned = max(vec4(dn, dw, de, ds) * sign(d0), 0.0); + vec2 pairMin = min(aligned.xz, aligned.yw); + vec2 pairMax = max(aligned.xz, aligned.yw); + float secondLargest = max(min(pairMax.x, pairMax.y), max(pairMin.x, pairMin.y)); + float coherentDelta = min(abs(d0), secondLargest); + confidence *= 1.0 - smoothstep(0.009 + sigma * 0.5, 0.020 + sigma * 0.6, coherentDelta); + float cornerDelta = min(abs(d0), max(pairMax.x, pairMax.y)); + confidence *= 1.0 - smoothstep(0.004 + sigma * 1.8, 0.010 + sigma * 1.7, cornerDelta); + confidence *= 1.0 - smoothstep(0.025, 0.070, max(abs(c.y - h.y), abs(c.z - h.z))); + if (!u_hasHistory) confidence = 0.0; + + vec3 lo = c, hi = c, mean = vec3(0.0), moment = vec3(0.0); + vec3 spatial = vec3(0.0); + float weightSum = 0.0; + float rangeSigma = 0.018 + min(sigma, 0.025) * 1.5; + for (int i = 0; i < 9; ++i) { + vec3 v = p[i]; + lo = min(lo, v); + hi = max(hi, v); + mean += v; + moment += v * v; + // Both luma and chroma edges constrain the spatial average. + vec3 delta = v - c; + float distance = delta.x * delta.x + dot(delta.yz, delta.yz) * 0.5; + float w = exp2(-distance / (rangeSigma * rangeSigma)); + w *= i == 4 ? 2.0 : (i == 1 || i == 3 || i == 5 || i == 7 ? 1.0 : 0.5); + spatial += v * w; + weightSum += w; + } + spatial /= weightSum; + mean /= 9.0; + vec3 deviation = sqrt(max(moment / 9.0 - mean * mean, vec3(0.0))); + + // Repeated clean detail has sigma=0 and is left alone. Moving detail gets + // only mild spatial NR; stable noisy areas can average more confidently. + float noiseAmount = smoothstep(0.003, 0.018, sigma); + float lumaMix = noiseAmount * mix(0.12, 0.55, confidence); + float chromaMix = noiseAmount * mix(0.25, 0.65, confidence); + vec3 current = mix(c, spatial, vec3(lumaMix, chromaMix, chromaMix)); + + // Variance clipping prevents old silhouettes surviving a scene cut or + // disocclusion. Use both the variance box and the true neighborhood bounds. + vec3 extent = max(deviation * 1.25, vec3(0.004, 0.006, 0.006)); + // The current center is always a valid sample, including a thin line or a + // texture peak outside the variance box. Do not erase consistent detail. + vec3 clipped = clamp(h, min(current, max(lo, mean - extent)), max(current, min(hi, mean + extent))); + float innovation = abs(current.x - h.x); + confidence *= 1.0 - smoothstep(0.018 + sigma, 0.055 + sigma, innovation); + vec3 weight = confidence * vec3(0.90, 0.92, 0.92); + vec3 result = mix(current, clipped, weight); + outColor = vec4(clamp(toRgb(result), 0.0, 1.0), c.x); +} +`; + +interface HistoryTarget { + texture: WebGLTexture; + fbo: WebGLFramebuffer; +} + +export class TemporalDenoiser { + private program: WebGLProgram | null = null; + private texelSizeLocation: WebGLUniformLocation | null = null; + private flipYLocation: WebGLUniformLocation | null = null; + private hasHistoryLocation: WebGLUniformLocation | null = null; + private targets: HistoryTarget[] = []; + private width = 0; + private height = 0; + private writeIndex = 0; + private hasHistory = false; + private floatHistory = false; + + init(gl: WebGL2RenderingContext): void { + this.program = createProgram(gl, FRAMEBUFFER_VERTEX_SHADER, FRAGMENT_SHADER); + this.texelSizeLocation = gl.getUniformLocation(this.program, "u_texelSize"); + this.flipYLocation = gl.getUniformLocation(this.program, "u_flipY"); + this.hasHistoryLocation = gl.getUniformLocation(this.program, "u_hasHistory"); + this.floatHistory = !!gl.getExtension("EXT_color_buffer_float"); + // biome-ignore lint/correctness/useHookAtTopLevel: WebGL useProgram, not a React hook + gl.useProgram(this.program); + gl.uniform1i(gl.getUniformLocation(this.program, "u_input"), 0); + gl.uniform1i(gl.getUniformLocation(this.program, "u_history"), 1); + } + + render(gl: WebGL2RenderingContext, input: WebGLTexture, width: number, height: number, flipY: boolean): WebGLTexture { + if (!this.program) throw new Error("TemporalDenoiser.render() called before init()"); + this.ensureTargets(gl, width, height); + const target = this.targets[this.writeIndex]; + const previous = this.targets[1 - this.writeIndex]; + gl.bindFramebuffer(gl.FRAMEBUFFER, target.fbo); + gl.viewport(0, 0, width, height); + // biome-ignore lint/correctness/useHookAtTopLevel: WebGL useProgram, not a React hook + gl.useProgram(this.program); + gl.activeTexture(gl.TEXTURE0); + gl.bindTexture(gl.TEXTURE_2D, input); + gl.activeTexture(gl.TEXTURE1); + gl.bindTexture(gl.TEXTURE_2D, previous.texture); + gl.uniform2f(this.texelSizeLocation, 1 / width, 1 / height); + gl.uniform1i(this.flipYLocation, flipY ? 1 : 0); + gl.uniform1i(this.hasHistoryLocation, this.hasHistory ? 1 : 0); + gl.drawArrays(gl.TRIANGLES, 0, 3); + this.writeIndex = 1 - this.writeIndex; + this.hasHistory = true; + return target.texture; + } + + /** Call on seeks, source/stage changes, and before re-rendering the same frame. */ + reset(): void { + this.hasHistory = false; + } + + private ensureTargets(gl: WebGL2RenderingContext, width: number, height: number): void { + if (this.width === width && this.height === height && this.targets.length === 2) return; + this.releaseTransientResources(gl); + try { + for (let i = 0; i < 2; i++) { + const texture = gl.createTexture(); + const fbo = gl.createFramebuffer(); + if (!texture || !fbo) { + gl.deleteTexture(texture); + gl.deleteFramebuffer(fbo); + throw new Error("Unable to allocate video denoise history"); + } + this.targets.push({ texture, fbo }); + gl.bindTexture(gl.TEXTURE_2D, texture); + // Half floats avoid accumulating 8-bit rounding errors in dark gradients. + gl.texStorage2D(gl.TEXTURE_2D, 1, this.floatHistory ? gl.RGBA16F : gl.RGBA8, width, height); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); + gl.bindFramebuffer(gl.FRAMEBUFFER, fbo); + gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, texture, 0); + if (gl.checkFramebufferStatus(gl.FRAMEBUFFER) !== gl.FRAMEBUFFER_COMPLETE) { + throw new Error("Incomplete video denoise framebuffer"); + } + } + } catch (error) { + this.releaseTransientResources(gl); + throw error; + } + this.width = width; + this.height = height; + } + + releaseTransientResources(gl: WebGL2RenderingContext): void { + for (const target of this.targets) { + gl.deleteTexture(target.texture); + gl.deleteFramebuffer(target.fbo); + } + this.targets = []; + this.width = this.height = this.writeIndex = 0; + this.reset(); + } + + destroy(gl: WebGL2RenderingContext): void { + this.releaseTransientResources(gl); + gl.deleteProgram(this.program); + this.program = null; + } +} diff --git a/web-ui/src/playback-engine/render/filters/gl-utils.ts b/web-ui/src/playback-engine/render/filters/gl-utils.ts index d8ced765..2d8d816c 100644 --- a/web-ui/src/playback-engine/render/filters/gl-utils.ts +++ b/web-ui/src/playback-engine/render/filters/gl-utils.ts @@ -20,20 +20,26 @@ export function createProgram(gl: WebGL2RenderingContext, vertexSource: string, if (!program) { throw new Error("Failed to create program"); } - const vs = compileShader(gl, gl.VERTEX_SHADER, vertexSource); - const fs = compileShader(gl, gl.FRAGMENT_SHADER, fragmentSource); - gl.attachShader(program, vs); - gl.attachShader(program, fs); - gl.linkProgram(program); - // Shaders are owned by the program after linking - gl.deleteShader(vs); - gl.deleteShader(fs); - if (!gl.getProgramParameter(program, gl.LINK_STATUS) && !gl.isContextLost()) { - const info = gl.getProgramInfoLog(program); + let vs: WebGLShader | null = null; + let fs: WebGLShader | null = null; + try { + vs = compileShader(gl, gl.VERTEX_SHADER, vertexSource); + fs = compileShader(gl, gl.FRAGMENT_SHADER, fragmentSource); + gl.attachShader(program, vs); + gl.attachShader(program, fs); + gl.linkProgram(program); + if (!gl.getProgramParameter(program, gl.LINK_STATUS) && !gl.isContextLost()) { + throw new Error(`Program link failed: ${gl.getProgramInfoLog(program)}`); + } + return program; + } catch (error) { gl.deleteProgram(program); - throw new Error(`Program link failed: ${info}`); + throw error; + } finally { + // Also release a successfully compiled vertex shader if fragment compilation fails. + if (vs) gl.deleteShader(vs); + if (fs) gl.deleteShader(fs); } - return program; } /** diff --git a/web-ui/src/playback-engine/render/filters/mosquito-nr.ts b/web-ui/src/playback-engine/render/filters/mosquito-nr.ts deleted file mode 100644 index 659ff239..00000000 --- a/web-ui/src/playback-engine/render/filters/mosquito-nr.ts +++ /dev/null @@ -1,205 +0,0 @@ -import { createProgram, FRAMEBUFFER_VERTEX_SHADER } from "./gl-utils"; -import { type RenderParams, registerFilter, type VideoFilter } from "./types"; - -/** - * Spatial compression-artifact reduction ahead of FSR EASU: mosquito/ringing - * around high-contrast edges (burned-in subtitles, jersey numbers, silhouettes) - * plus light chroma bleed and blocking in flat regions. - * - * A 3×3 box would melt soft motion-blurred edges (running players into grass). - * Taps are luma-range weighted instead (cheap bilateral): similar neighbors - * average, real edges keep the center. Extra ±2-px cross taps widen support - * for ringing that sits 2 px off an edge without a full 5×5. - * - * - `onEdge` (center Sobel) reduces luma NR on the sharpest strokes so numbers - * and glyph interiors stay crisp; chroma NR is not gated (color bleed lives - * on the stroke itself). - * - `nearEdge` (wider luma range) is the halo where mosquito lives. - * - Flat regions get a modest luma mix to take the edge off 8×8 blocking. - * - * Thresholds are full-range 0–1 luma (Sobel abs-sum is ~0–4). RANGE_SIGMA is - * mosquito amplitude, not edge height — a white-on-green number (~0.6 luma - * jump) stays unmixed. Raising LUMA_NEAR eats halos harder; raising - * RANGE_SIGMA starts to smear silhouettes. - * - * Shader uniforms: - * - u_input: current frame (raw video upload or a prior framebuffer). - * - u_texelSize: 1/width, 1/height. - * - u_flipY: 1 when sampling a raw DOM video upload. - */ - -const FRAGMENT_SHADER = /*glsl*/ `#version 300 es -precision highp float; - -uniform sampler2D u_input; -uniform vec2 u_texelSize; - -in vec2 v_texCoord; -out vec4 outColor; - -const vec3 LUMA = vec3(0.2126, 0.7152, 0.0722); - -// Center Sobel abs-sum. A white/black step is ~3.2; protect only the -// sharpest transitions (numbers, subtitle strokes). Soft silhouettes sit -// lower and stay eligible for range-weighted NR. -const float EDGE_LO = 1.4; -const float EDGE_HI = 2.6; - -// Wider (3×3 plus ±2-px cross) luma range. Nearby high-contrast edges push -// this toward 0.5–1.0; mosquito-only neighborhoods sit lower. -const float NEAR_LO = 0.08; -const float NEAR_HI = 0.38; - -// 3×3 luma range for the flat-region mix. Low-bitrate grass/jersey blocking -// is ~0.02–0.10; keep this modest so faces in cleaner sources do not go plastic. -const float FLAT_LO = 0.02; -const float FLAT_HI = 0.12; - -// Luma delta at which a neighbor is excluded. Mosquito speckle is below this; -// a jersey-number / grass step is far above it. -const float RANGE_SIGMA = 0.11; - -const float LUMA_NEAR = 0.82; -const float LUMA_FLAT = 0.16; -const float EDGE_KEEP = 0.45; -const float CHROMA_BASE = 0.38; -const float CHROMA_NEAR = 0.72; -const float CHROMA_FLAT = 0.28; - -vec3 sampleRgb(vec2 uv) { - return texture(u_input, uv).rgb; -} - -float rangeW(float luma, float centerLuma) { - return 1.0 - smoothstep(0.0, RANGE_SIGMA, abs(luma - centerLuma)); -} - -void accum(inout vec3 acc, inout float wSum, vec3 color, float luma, float centerLuma) { - float w = rangeW(luma, centerLuma); - acc += color * w; - wSum += w; -} - -void main() { - vec2 uv = v_texCoord; - vec2 dx = vec2(u_texelSize.x, 0.0); - vec2 dy = vec2(0.0, u_texelSize.y); - - vec3 nw = sampleRgb(uv - dx - dy); - vec3 n = sampleRgb(uv - dy); - vec3 ne = sampleRgb(uv + dx - dy); - vec3 w = sampleRgb(uv - dx); - vec3 c = sampleRgb(uv); - vec3 e = sampleRgb(uv + dx); - vec3 sw = sampleRgb(uv - dx + dy); - vec3 s = sampleRgb(uv + dy); - vec3 se = sampleRgb(uv + dx + dy); - - vec3 nn = sampleRgb(uv - 2.0 * dy); - vec3 ss = sampleRgb(uv + 2.0 * dy); - vec3 ww = sampleRgb(uv - 2.0 * dx); - vec3 ee = sampleRgb(uv + 2.0 * dx); - - float nwL = dot(nw, LUMA); - float nL = dot(n, LUMA); - float neL = dot(ne, LUMA); - float wL = dot(w, LUMA); - float cL = dot(c, LUMA); - float eL = dot(e, LUMA); - float swL = dot(sw, LUMA); - float sL = dot(s, LUMA); - float seL = dot(se, LUMA); - float nnL = dot(nn, LUMA); - float ssL = dot(ss, LUMA); - float wwL = dot(ww, LUMA); - float eeL = dot(ee, LUMA); - - float gx = -nwL - 2.0 * wL - swL + neL + 2.0 * eL + seL; - float gy = -nwL - 2.0 * nL - neL + swL + 2.0 * sL + seL; - float grad = abs(gx) + abs(gy); - float onEdge = smoothstep(EDGE_LO, EDGE_HI, grad); - - float min3 = min(min(min(nwL, nL), min(neL, wL)), min(min(cL, eL), min(swL, min(sL, seL)))); - float max3 = max(max(max(nwL, nL), max(neL, wL)), max(max(cL, eL), max(swL, max(sL, seL)))); - float range3 = max3 - min3; - - float min5 = min(min3, min(min(nnL, ssL), min(wwL, eeL))); - float max5 = max(max3, max(max(nnL, ssL), max(wwL, eeL))); - float range5 = max5 - min5; - - float nearEdge = smoothstep(NEAR_LO, NEAR_HI, range5); - float flatness = 1.0 - smoothstep(FLAT_LO, FLAT_HI, range3); - - vec3 acc = vec3(0.0); - float wSum = 0.0; - accum(acc, wSum, nw, nwL, cL); - accum(acc, wSum, n, nL, cL); - accum(acc, wSum, ne, neL, cL); - accum(acc, wSum, w, wL, cL); - accum(acc, wSum, c, cL, cL); - accum(acc, wSum, e, eL, cL); - accum(acc, wSum, sw, swL, cL); - accum(acc, wSum, s, sL, cL); - accum(acc, wSum, se, seL, cL); - accum(acc, wSum, nn, nnL, cL); - accum(acc, wSum, ss, ssL, cL); - accum(acc, wSum, ww, wwL, cL); - accum(acc, wSum, ee, eeL, cL); - vec3 filtered = acc / max(wSum, 1e-6); - - float lumaMix = clamp((nearEdge * LUMA_NEAR + flatness * LUMA_FLAT) * mix(1.0, EDGE_KEEP, onEdge), 0.0, 1.0); - float chromaMix = clamp(CHROMA_BASE + nearEdge * CHROMA_NEAR + flatness * CHROMA_FLAT, 0.0, 1.0); - - float yC = cL; - float yB = dot(filtered, LUMA); - float yOut = mix(yC, yB, lumaMix); - - vec2 chromaC = vec2(c.b - yC, c.r - yC); - vec2 chromaB = vec2(filtered.b - yB, filtered.r - yB); - vec2 chromaOut = mix(chromaC, chromaB, chromaMix); - - float b = yOut + chromaOut.x; - float r = yOut + chromaOut.y; - float g = (yOut - LUMA.x * r - LUMA.z * b) / LUMA.y; - - outColor = vec4(clamp(vec3(r, g, b), 0.0, 1.0), 1.0); -} -`; - -class MosquitoNrFilter implements VideoFilter { - readonly name = "mosquito-nr"; - readonly historyFrames = 0; - - private program: WebGLProgram | null = null; - private uTexelSize: WebGLUniformLocation | null = null; - private uFlipY: WebGLUniformLocation | null = null; - - init(gl: WebGL2RenderingContext): void { - this.program = createProgram(gl, FRAMEBUFFER_VERTEX_SHADER, FRAGMENT_SHADER); - // biome-ignore lint/correctness/useHookAtTopLevel: WebGL useProgram, not a React hook - gl.useProgram(this.program); - gl.uniform1i(gl.getUniformLocation(this.program, "u_input"), 0); - this.uTexelSize = gl.getUniformLocation(this.program, "u_texelSize"); - this.uFlipY = gl.getUniformLocation(this.program, "u_flipY"); - } - - render(gl: WebGL2RenderingContext, textures: WebGLTexture[], params: RenderParams): void { - if (!this.program) return; - // biome-ignore lint/correctness/useHookAtTopLevel: WebGL useProgram, not a React hook - gl.useProgram(this.program); - gl.activeTexture(gl.TEXTURE0); - gl.bindTexture(gl.TEXTURE_2D, textures[0]); - gl.uniform2f(this.uTexelSize, 1 / params.width, 1 / params.height); - gl.uniform1i(this.uFlipY, params.flipY ? 1 : 0); - gl.drawArrays(gl.TRIANGLES, 0, 3); - } - - destroy(gl: WebGL2RenderingContext): void { - if (this.program) { - gl.deleteProgram(this.program); - this.program = null; - } - } -} - -registerFilter("mosquito-nr", () => new MosquitoNrFilter()); diff --git a/web-ui/src/playback-engine/render/fsr.ts b/web-ui/src/playback-engine/render/fsr.ts index 4ccddf05..3dac52fb 100644 --- a/web-ui/src/playback-engine/render/fsr.ts +++ b/web-ui/src/playback-engine/render/fsr.ts @@ -4,8 +4,7 @@ import type { Presenter } from "./presenters"; /** * AMD FidelityFX Super Resolution 1 (FSR1) upscale presenter: EASU * (Edge-Adaptive Spatial Upsampling) followed by RCAS (Robust Contrast - * Adaptive Sharpening), replacing the Catmull-Rom bicubic presenter as the - * enhancement path's upscaler. + * Adaptive Sharpening). * * Ported to WebGL2 GLSL ES 300 from AMD's reference (ffx_fsr1.h, MIT * licensed, Copyright (c) 2021 Advanced Micro Devices, Inc.), following the @@ -17,18 +16,15 @@ import type { Presenter } from "./presenters"; * derived from the RGB taps, and the resulting kernel weights are shared * across all three channels, since the weights never depend on tap color. * - * RCAS's contrast/sharpness lobe replaces the old dedicated "sharpen" filter - * (see filters/sharpen.ts, removed), so its output also folds in the same - * mild contrast/saturation lift that filter used to apply. + * RCAS sharpens luminance only, with noise rejection and local extrema + * bounds. It leaves the source transfer curve, saturation, and chroma alone. */ /** * EASU (Edge-Adaptive Spatial Upsampling), ported from AMD's FsrEasuF. * - * easuRcp/easuRsqrt: AMD's AAxxRcpF1/RsqF1 fast bit-hack approximations. - * Unlike a native 1.0/x, these return a large finite value at x=0 instead of - * Infinity -- flat image regions hit x=0 constantly, and a subsequent - * 0 * Infinity would poison the result with NaN. + * Reciprocals are guarded at zero. Exact arithmetic avoids bias in low + * contrast detail and handles flat black/white regions without NaNs. * * easuTap (AMD's FsrEasuTap): accumulates one Lanczos-2-approximation tap * into the RGB/weight accumulators, via a base*window product that avoids @@ -57,6 +53,7 @@ import type { Presenter } from "./presenters"; const EASU_FRAGMENT_SHADER = /*glsl*/ `#version 300 es precision highp float; precision highp int; +precision highp sampler2D; uniform sampler2D u_input; uniform vec2 u_srcSize; @@ -68,17 +65,17 @@ out vec4 outColor; const vec3 LUMA = vec3(0.2126, 0.7152, 0.0722); float easuRcp(float x) { - return uintBitsToFloat(0x7ef07ebbu - floatBitsToUint(x)); + return 1.0 / max(x, 1e-8); } float easuRsqrt(float x) { - return uintBitsToFloat(0x5f347d74u - (floatBitsToUint(x) >> 1u)); + return inversesqrt(max(x, 1e-8)); } vec3 min3(vec3 a, vec3 b, vec3 c) { return min(a, min(b, c)); } vec3 max3(vec3 a, vec3 b, vec3 c) { return max(a, max(b, c)); } -vec3 sampleSrc(vec2 pixelPos) { - return texture(u_input, (pixelPos + 0.5) / u_srcSize).rgb; +vec4 sampleSrc(vec2 pixelPos) { + return texture(u_input, (pixelPos + 0.5) / u_srcSize); } void easuTap( @@ -148,31 +145,31 @@ void main() { vec2 fp = floor(pp); pp -= fp; - vec3 b = sampleSrc(fp + vec2(0.0, -1.0)); - vec3 c = sampleSrc(fp + vec2(1.0, -1.0)); - vec3 e = sampleSrc(fp + vec2(-1.0, 0.0)); - vec3 f = sampleSrc(fp + vec2(0.0, 0.0)); - vec3 g = sampleSrc(fp + vec2(1.0, 0.0)); - vec3 h = sampleSrc(fp + vec2(2.0, 0.0)); - vec3 i = sampleSrc(fp + vec2(-1.0, 1.0)); - vec3 j = sampleSrc(fp + vec2(0.0, 1.0)); - vec3 k = sampleSrc(fp + vec2(1.0, 1.0)); - vec3 l = sampleSrc(fp + vec2(2.0, 1.0)); - vec3 n = sampleSrc(fp + vec2(0.0, 2.0)); - vec3 o = sampleSrc(fp + vec2(1.0, 2.0)); - - float bL = dot(b, LUMA); - float cL = dot(c, LUMA); - float eL = dot(e, LUMA); - float fL = dot(f, LUMA); - float gL = dot(g, LUMA); - float hL = dot(h, LUMA); - float iL = dot(i, LUMA); - float jL = dot(j, LUMA); - float kL = dot(k, LUMA); - float lL = dot(l, LUMA); - float nL = dot(n, LUMA); - float oL = dot(o, LUMA); + vec4 b = sampleSrc(fp + vec2(0.0, -1.0)); + vec4 c = sampleSrc(fp + vec2(1.0, -1.0)); + vec4 e = sampleSrc(fp + vec2(-1.0, 0.0)); + vec4 f = sampleSrc(fp + vec2(0.0, 0.0)); + vec4 g = sampleSrc(fp + vec2(1.0, 0.0)); + vec4 h = sampleSrc(fp + vec2(2.0, 0.0)); + vec4 i = sampleSrc(fp + vec2(-1.0, 1.0)); + vec4 j = sampleSrc(fp + vec2(0.0, 1.0)); + vec4 k = sampleSrc(fp + vec2(1.0, 1.0)); + vec4 l = sampleSrc(fp + vec2(2.0, 1.0)); + vec4 n = sampleSrc(fp + vec2(0.0, 2.0)); + vec4 o = sampleSrc(fp + vec2(1.0, 2.0)); + + float bL = dot(b.rgb, LUMA); + float cL = dot(c.rgb, LUMA); + float eL = dot(e.rgb, LUMA); + float fL = dot(f.rgb, LUMA); + float gL = dot(g.rgb, LUMA); + float hL = dot(h.rgb, LUMA); + float iL = dot(i.rgb, LUMA); + float jL = dot(j.rgb, LUMA); + float kL = dot(k.rgb, LUMA); + float lL = dot(l.rgb, LUMA); + float nL = dot(n.rgb, LUMA); + float oL = dot(o.rgb, LUMA); vec2 dir = vec2(0.0); float len = 0.0; @@ -198,77 +195,54 @@ void main() { vec3 accumColor = vec3(0.0); float accumWeight = 0.0; - easuTap(accumColor, accumWeight, vec2(0.0, -1.0) - pp, dir, len2, lobe, clip, b); - easuTap(accumColor, accumWeight, vec2(1.0, -1.0) - pp, dir, len2, lobe, clip, c); - easuTap(accumColor, accumWeight, vec2(-1.0, 1.0) - pp, dir, len2, lobe, clip, i); - easuTap(accumColor, accumWeight, vec2(0.0, 1.0) - pp, dir, len2, lobe, clip, j); - easuTap(accumColor, accumWeight, vec2(0.0, 0.0) - pp, dir, len2, lobe, clip, f); - easuTap(accumColor, accumWeight, vec2(-1.0, 0.0) - pp, dir, len2, lobe, clip, e); - easuTap(accumColor, accumWeight, vec2(1.0, 1.0) - pp, dir, len2, lobe, clip, k); - easuTap(accumColor, accumWeight, vec2(2.0, 1.0) - pp, dir, len2, lobe, clip, l); - easuTap(accumColor, accumWeight, vec2(2.0, 0.0) - pp, dir, len2, lobe, clip, h); - easuTap(accumColor, accumWeight, vec2(1.0, 0.0) - pp, dir, len2, lobe, clip, g); - easuTap(accumColor, accumWeight, vec2(1.0, 2.0) - pp, dir, len2, lobe, clip, o); - easuTap(accumColor, accumWeight, vec2(0.0, 2.0) - pp, dir, len2, lobe, clip, n); + easuTap(accumColor, accumWeight, vec2(0.0, -1.0) - pp, dir, len2, lobe, clip, b.rgb); + easuTap(accumColor, accumWeight, vec2(1.0, -1.0) - pp, dir, len2, lobe, clip, c.rgb); + easuTap(accumColor, accumWeight, vec2(-1.0, 1.0) - pp, dir, len2, lobe, clip, i.rgb); + easuTap(accumColor, accumWeight, vec2(0.0, 1.0) - pp, dir, len2, lobe, clip, j.rgb); + easuTap(accumColor, accumWeight, vec2(0.0, 0.0) - pp, dir, len2, lobe, clip, f.rgb); + easuTap(accumColor, accumWeight, vec2(-1.0, 0.0) - pp, dir, len2, lobe, clip, e.rgb); + easuTap(accumColor, accumWeight, vec2(1.0, 1.0) - pp, dir, len2, lobe, clip, k.rgb); + easuTap(accumColor, accumWeight, vec2(2.0, 1.0) - pp, dir, len2, lobe, clip, l.rgb); + easuTap(accumColor, accumWeight, vec2(2.0, 0.0) - pp, dir, len2, lobe, clip, h.rgb); + easuTap(accumColor, accumWeight, vec2(1.0, 0.0) - pp, dir, len2, lobe, clip, g.rgb); + easuTap(accumColor, accumWeight, vec2(1.0, 2.0) - pp, dir, len2, lobe, clip, o.rgb); + easuTap(accumColor, accumWeight, vec2(0.0, 2.0) - pp, dir, len2, lobe, clip, n.rgb); vec3 rgb = accumColor / accumWeight; - vec3 lo = min(min3(f, g, j), k); - vec3 hi = max(max3(f, g, j), k); + vec3 lo = min(min3(f.rgb, g.rgb, j.rgb), k.rgb); + vec3 hi = max(max3(f.rgb, g.rgb, j.rgb), k.rgb); rgb = clamp(rgb, lo, hi); - outColor = vec4(clamp(rgb, 0.0, 1.0), 1.0); + // Carry the amount removed by NR into sharpening. RGB10_A2 provides four + // noise levels without increasing the intermediate's bandwidth. + float noise = mix(mix(abs(f.a - fL), abs(g.a - gL), pp.x), + mix(abs(j.a - jL), abs(k.a - kL), pp.x), pp.y); + outColor = vec4(clamp(rgb, 0.0, 1.0), clamp(noise * 32.0, 0.0, 1.0)); } `; /** - * RCAS (Robust Contrast Adaptive Sharpening), ported from AMD's FsrRcasF. - * - * SHARPNESS: AMD scale, 0.0 = strongest sharpening, higher N = N stops - * (halvings) weaker. RCAS_LIMIT (AMD FSR_RCAS_LIMIT): clamps the sharpening - * lobe to avoid unnatural results. CONTRAST/SATURATION: the same mild tone - * lift the old dedicated "sharpen" enhancement filter used to apply. - * - * rcasRcp: medium-precision reciprocal (fast bit-hack estimate plus one - * Newton-Raphson refinement step), used in place of AMD's plain division - * because RCAS's clipping-lobe math divides by neighborhood min/max ranges - * that legitimately hit exactly zero on flat video content (solid black - * letterboxing, blown-out highlights), where native division would yield - * 0/0 = NaN. This approximation returns a large finite value at zero - * instead, so the following multiply-by-numerator collapses cleanly to zero - * rather than propagating NaN. - * - * main reads the 3x3 cross neighborhood around the output pixel: - * b - * d e f - * h - * derives a sharpening lobe from the local min/max contrast range (clamped - * by RCAS_LIMIT/SHARPNESS), de-weights it in flat/noisy regions (comparing - * the 4-neighbor average against the center to avoid amplifying compression - * noise/grain; 0.9 is stronger than AMD's 0.5 so leftover mosquito around - * edges is less likely to be re-sharpened), and blends the cross taps with - * the center by that lobe before applying the CONTRAST/SATURATION lift. + * Contrast-adaptive luma sharpening after EASU. A symmetric cross detects + * isolated noise; the output stays inside the local luma range and RGB gamut. + * Keeping chroma and the transfer curve unchanged avoids color fringes, + * crushed near-black detail, and clipped highlights. */ const RCAS_FRAGMENT_SHADER = /*glsl*/ `#version 300 es precision highp float; precision highp int; +precision highp sampler2D; uniform sampler2D u_input; uniform vec2 u_texelSize; uniform bool u_flipY; +uniform bool u_upscaled; out vec4 outColor; const vec3 LUMA = vec3(0.2126, 0.7152, 0.0722); -const float SHARPNESS = 0.2; +const float SHARPNESS = 0.45; const float RCAS_LIMIT = 0.25 - 1.0 / 16.0; -const float CONTRAST = 1.04; -const float SATURATION = 1.03; - -float rcasRcp(float a) { - float b = uintBitsToFloat(0x7ef19fffu - floatBitsToUint(a)); - return b * (-b * a + 2.0); -} float min3(float a, float b, float c) { return min(a, min(b, c)); } float max3(float a, float b, float c) { return max(a, max(b, c)); } @@ -278,7 +252,8 @@ void main() { if (u_flipY) uv.y = 1.0 - uv.y; vec3 b = texture(u_input, uv + vec2(0.0, -1.0) * u_texelSize).rgb; vec3 d = texture(u_input, uv + vec2(-1.0, 0.0) * u_texelSize).rgb; - vec3 e = texture(u_input, uv).rgb; + vec4 center = texture(u_input, uv); + vec3 e = center.rgb; vec3 f = texture(u_input, uv + vec2(1.0, 0.0) * u_texelSize).rgb; vec3 h = texture(u_input, uv + vec2(0.0, 1.0) * u_texelSize).rgb; @@ -291,23 +266,25 @@ void main() { float mn = min(min3(bL, dL, fL), hL); float mx = max(max3(bL, dL, fL), hL); - float hitMin = min(mn, eL) * rcasRcp(4.0 * mx); - float hitMax = (1.0 - max(mx, eL)) * rcasRcp(4.0 * mn - 4.0); + float hitMin = min(mn, eL) / max(4.0 * mx, 1e-6); + float hitMax = (1.0 - max(mx, eL)) / min(4.0 * mn - 4.0, -1e-6); float lobeShape = max(-hitMin, hitMax); float lobe = max(-RCAS_LIMIT, min(lobeShape, 0.0)) * exp2(-SHARPNESS); float mn5 = min(mn, eL); float mx5 = max(mx, eL); float noise = 0.25 * (bL + dL + fL + hL) - eL; - noise = clamp(abs(noise) * rcasRcp(mx5 - mn5), 0.0, 1.0); + noise = clamp(abs(noise) / max(mx5 - mn5, 1e-6), 0.0, 1.0); lobe *= 1.0 - 0.9 * noise; + float removedNoise = u_upscaled ? center.a : clamp(abs(center.a - eL) * 32.0, 0.0, 1.0); + lobe *= 1.0 - 0.85 * removedNoise; - float rcp = rcasRcp(4.0 * lobe + 1.0); - vec3 rgb = (lobe * (b + d + f + h) + e) * rcp; - - rgb = (rgb - 0.5) * CONTRAST + 0.5; - float y = dot(rgb, LUMA); - rgb = mix(vec3(y), rgb, SATURATION); + float edge = max(abs(fL - dL), abs(hL - bL)); + lobe *= smoothstep(2.0 / 255.0, 12.0 / 255.0, edge); + float y = (lobe * (bL + dL + fL + hL) + eL) / (4.0 * lobe + 1.0); + float lo = max(mn5 - eL, -min3(e.r, e.g, e.b)); + float hi = min(mx5 - eL, 1.0 - max3(e.r, e.g, e.b)); + vec3 rgb = e + clamp(y - eL, lo, hi); outColor = vec4(clamp(rgb, 0.0, 1.0), 1.0); } @@ -325,6 +302,8 @@ interface IntermediateTarget { * RCAS (intermediate -> bound framebuffer). RCAS also runs standalone * (skipping EASU) when the output is not larger than the source, so picture * enhancement still sharpens at native size instead of doing nothing. + * Input RGB must be denoised, with original luma in alpha (TemporalDenoiser's + * output). Alpha informs noise-aware sharpening; it is not image opacity. */ export class FsrPresenter implements Presenter { readonly name = "fsr-present"; @@ -339,6 +318,7 @@ export class FsrPresenter implements Presenter { private rcasInputLocation: WebGLUniformLocation | null = null; private rcasTexelSizeLocation: WebGLUniformLocation | null = null; private rcasFlipYLocation: WebGLUniformLocation | null = null; + private rcasUpscaledLocation: WebGLUniformLocation | null = null; private intermediate: IntermediateTarget | null = null; @@ -359,6 +339,7 @@ export class FsrPresenter implements Presenter { this.rcasInputLocation = gl.getUniformLocation(this.rcasProgram, "u_input"); this.rcasTexelSizeLocation = gl.getUniformLocation(this.rcasProgram, "u_texelSize"); this.rcasFlipYLocation = gl.getUniformLocation(this.rcasProgram, "u_flipY"); + this.rcasUpscaledLocation = gl.getUniformLocation(this.rcasProgram, "u_upscaled"); } present( @@ -381,7 +362,7 @@ export class FsrPresenter implements Presenter { const upscaling = dstWidth > srcWidth + 0.5 || dstHeight > srcHeight + 0.5; if (!upscaling) { - this.runRcas(gl, texture, dstWidth, dstHeight, outputFbo, flipY); + this.runRcas(gl, texture, dstWidth, dstHeight, outputFbo, flipY, false); return; } @@ -408,7 +389,7 @@ export class FsrPresenter implements Presenter { // The intermediate is a framebuffer-rendered texture (native orientation), // regardless of whether the EASU input needed a flip. - this.runRcas(gl, target.texture, dstWidth, dstHeight, outputFbo, false); + this.runRcas(gl, target.texture, dstWidth, dstHeight, outputFbo, false, true); } private runRcas( @@ -418,6 +399,7 @@ export class FsrPresenter implements Presenter { height: number, targetFbo: WebGLFramebuffer | null, flipY: boolean, + upscaled: boolean, ): void { gl.bindFramebuffer(gl.FRAMEBUFFER, targetFbo); gl.viewport(0, 0, width, height); @@ -428,6 +410,7 @@ export class FsrPresenter implements Presenter { gl.uniform1i(this.rcasInputLocation, 0); gl.uniform2f(this.rcasTexelSizeLocation, 1 / width, 1 / height); gl.uniform1i(this.rcasFlipYLocation, flipY ? 1 : 0); + gl.uniform1i(this.rcasUpscaledLocation, upscaled ? 1 : 0); gl.drawArrays(gl.TRIANGLES, 0, 3); } @@ -442,7 +425,7 @@ export class FsrPresenter implements Presenter { const texture = gl.createTexture(); if (!texture) return null; gl.bindTexture(gl.TEXTURE_2D, texture); - gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, width, height, 0, gl.RGBA, gl.UNSIGNED_BYTE, null); + gl.texStorage2D(gl.TEXTURE_2D, 1, gl.RGB10_A2, width, height); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); @@ -495,5 +478,6 @@ export class FsrPresenter implements Presenter { this.rcasInputLocation = null; this.rcasTexelSizeLocation = null; this.rcasFlipYLocation = null; + this.rcasUpscaledLocation = null; } } diff --git a/web-ui/src/playback-engine/render/index.ts b/web-ui/src/playback-engine/render/index.ts index a862df57..24a8c4bb 100644 --- a/web-ui/src/playback-engine/render/index.ts +++ b/web-ui/src/playback-engine/render/index.ts @@ -1,5 +1,4 @@ import "./filters/bwdif"; -import "./filters/mosquito-nr"; import type { PlayerRenderState, PlayerVideoScanType } from "../types"; import Log from "../utils/logger"; import { isRenderResolutionEligible, type RenderStageName, VideoRenderer } from "./renderer"; diff --git a/web-ui/src/playback-engine/render/renderer.ts b/web-ui/src/playback-engine/render/renderer.ts index bfcf245a..5b2d04c1 100644 --- a/web-ui/src/playback-engine/render/renderer.ts +++ b/web-ui/src/playback-engine/render/renderer.ts @@ -1,4 +1,5 @@ import Log from "../utils/logger"; +import { TemporalDenoiser } from "./denoise"; import { createFilter, type RenderParams, type VideoFilter } from "./filters/types"; import { FsrPresenter } from "./fsr"; import { PassthroughPresenter, type Presenter } from "./presenters"; @@ -17,16 +18,6 @@ export function isRenderResolutionEligible(width: number, height: number): boole return width > 0 && width <= GATE_MAX_WIDTH && height > 0 && height <= GATE_MAX_HEIGHT; } -/** - * Post-stage enhancement filters, applied in order between the source stage - * and presentation. All are registered in the filter registry and must be - * stateless (historyFrames = 0): the renderer re-runs the whole list per - * frame and assumes channel/stage switches need no per-filter reset. - * mosquito-nr runs at source resolution so FSR EASU does not reconstruct - * compression speckle around high-contrast edges. - */ -const ENHANCEMENT_FILTER_NAMES: readonly string[] = ["mosquito-nr"]; - /** * Safety ceiling for the enhanced canvas backing store, so a very large * display rect (or a stray devicePixelRatio) cannot push the per-frame @@ -42,11 +33,19 @@ interface RenderTarget { height: number; } +interface PendingField { + presentAt: number; + enhanced: boolean; + texture: WebGLTexture; + width: number; + height: number; +} + /** * WebGL2 render loop. It pulls decoded frames from the