diff --git a/web-ui/src/playback-engine/render/denoise.ts b/web-ui/src/playback-engine/render/denoise.ts new file mode 100644 index 00000000..d1600cba --- /dev/null +++ b/web-ui/src/playback-engine/render/denoise.ts @@ -0,0 +1,237 @@ +import { createProgram, FRAMEBUFFER_VERTEX_SHADER } from "./filters/gl-utils"; + +/** + * Motion-adaptive, recursive video denoising at source resolution. RGB stores + * the filtered image; alpha stores the ORIGINAL luma. Comparing original luma + * across a five-pixel patch distinguishes changing detail from random noise, + * without an extra copy pass or a second history attachment. + * + * History is rejected on moving edges and clipped to the current neighborhood + * before blending. Chroma also participates in rejection (isoluminant motion). + * There is no lookahead, motion extrapolation, or added presentation latency. + */ +const FRAGMENT_SHADER = /*glsl*/ `#version 300 es +precision highp float; +precision highp sampler2D; + +uniform sampler2D u_input; +uniform sampler2D u_history; +uniform vec2 u_texelSize; +uniform bool u_hasHistory; +uniform bool u_flipY; + +in vec2 v_texCoord; +out vec4 outColor; + +const vec3 LUMA = vec3(0.2126, 0.7152, 0.0722); + +vec3 toYcc(vec3 rgb) { + float y = dot(rgb, LUMA); + return vec3(y, (rgb.b - y) / 1.8556, (rgb.r - y) / 1.5748); +} + +vec3 toRgb(vec3 ycc) { + float r = ycc.x + 1.5748 * ycc.z; + float b = ycc.x + 1.8556 * ycc.y; + return vec3(r, (ycc.x - LUMA.x * r - LUMA.z * b) / LUMA.y, b); +} + +void main() { + vec2 uv = v_texCoord; + vec2 dx = vec2(u_texelSize.x, 0.0); + vec2 dy = vec2(0.0, u_texelSize.y); + vec3 p[9]; + p[0] = toYcc(texture(u_input, uv - dx - dy).rgb); + p[1] = toYcc(texture(u_input, uv - dy).rgb); + p[2] = toYcc(texture(u_input, uv + dx - dy).rgb); + p[3] = toYcc(texture(u_input, uv - dx).rgb); + p[4] = toYcc(texture(u_input, uv).rgb); + p[5] = toYcc(texture(u_input, uv + dx).rgb); + p[6] = toYcc(texture(u_input, uv - dx + dy).rgb); + p[7] = toYcc(texture(u_input, uv + dy).rgb); + p[8] = toYcc(texture(u_input, uv + dx + dy).rgb); + vec3 c = p[4]; + + // History is framebuffer-backed, even when the input is a DOM upload. + vec2 huv = u_flipY ? vec2(uv.x, 1.0 - uv.y) : uv; + vec2 hdy = u_flipY ? -dy : dy; + vec4 history = texture(u_history, huv); + vec3 h = toYcc(history.rgb); + float d0 = c.x - history.a; + float dn = p[1].x - texture(u_history, huv - hdy).a; + float dw = p[3].x - texture(u_history, huv - dx).a; + float de = p[5].x - texture(u_history, huv + dx).a; + float ds = p[7].x - texture(u_history, huv + hdy).a; + float meanDelta = (d0 * 2.0 + dn + dw + de + ds) / 6.0; + float deltaVariance = max(0.0, (2.0*d0*d0 + dn*dn + dw*dw + de*de + ds*ds) / 6.0 - meanDelta*meanDelta); + float sigma = u_hasHistory ? clamp(sqrt(deltaVariance * 0.5), 0.0, 0.035) : 0.012; + + // Opposing differences reveal translating edges even if their average cancels. + float structureDelta = max(abs(de - dw), abs(ds - dn)); + float motion = abs(meanDelta); + float confidence = 1.0 - smoothstep(0.008 + sigma * 0.35, 0.024 + sigma * 0.65, motion); + confidence *= 1.0 - smoothstep(0.020 + sigma * 1.5, 0.060 + sigma * 2.0, structureDelta); + confidence *= 1.0 - smoothstep(0.025 + sigma, 0.070 + sigma, abs(d0)); + // A weak translating edge changes the center and at least two neighbors + // in the same direction. Random grain rarely agrees at all three pixels. + // This catches low-contrast motion without rejecting isolated noise peaks. + vec4 aligned = max(vec4(dn, dw, de, ds) * sign(d0), 0.0); + vec2 pairMin = min(aligned.xz, aligned.yw); + vec2 pairMax = max(aligned.xz, aligned.yw); + float secondLargest = max(min(pairMax.x, pairMax.y), max(pairMin.x, pairMin.y)); + float coherentDelta = min(abs(d0), secondLargest); + confidence *= 1.0 - smoothstep(0.009 + sigma * 0.5, 0.020 + sigma * 0.6, coherentDelta); + float cornerDelta = min(abs(d0), max(pairMax.x, pairMax.y)); + confidence *= 1.0 - smoothstep(0.004 + sigma * 1.8, 0.010 + sigma * 1.7, cornerDelta); + confidence *= 1.0 - smoothstep(0.025, 0.070, max(abs(c.y - h.y), abs(c.z - h.z))); + if (!u_hasHistory) confidence = 0.0; + + vec3 lo = c, hi = c, mean = vec3(0.0), moment = vec3(0.0); + vec3 spatial = vec3(0.0); + float weightSum = 0.0; + float rangeSigma = 0.018 + min(sigma, 0.025) * 1.5; + for (int i = 0; i < 9; ++i) { + vec3 v = p[i]; + lo = min(lo, v); + hi = max(hi, v); + mean += v; + moment += v * v; + // Both luma and chroma edges constrain the spatial average. + vec3 delta = v - c; + float distance = delta.x * delta.x + dot(delta.yz, delta.yz) * 0.5; + float w = exp2(-distance / (rangeSigma * rangeSigma)); + w *= i == 4 ? 2.0 : (i == 1 || i == 3 || i == 5 || i == 7 ? 1.0 : 0.5); + spatial += v * w; + weightSum += w; + } + spatial /= weightSum; + mean /= 9.0; + vec3 deviation = sqrt(max(moment / 9.0 - mean * mean, vec3(0.0))); + + // Repeated clean detail has sigma=0 and is left alone. Moving detail gets + // only mild spatial NR; stable noisy areas can average more confidently. + float noiseAmount = smoothstep(0.003, 0.018, sigma); + float lumaMix = noiseAmount * mix(0.12, 0.55, confidence); + float chromaMix = noiseAmount * mix(0.25, 0.65, confidence); + vec3 current = mix(c, spatial, vec3(lumaMix, chromaMix, chromaMix)); + + // Variance clipping prevents old silhouettes surviving a scene cut or + // disocclusion. Use both the variance box and the true neighborhood bounds. + vec3 extent = max(deviation * 1.25, vec3(0.004, 0.006, 0.006)); + // The current center is always a valid sample, including a thin line or a + // texture peak outside the variance box. Do not erase consistent detail. + vec3 clipped = clamp(h, min(current, max(lo, mean - extent)), max(current, min(hi, mean + extent))); + float innovation = abs(current.x - h.x); + confidence *= 1.0 - smoothstep(0.018 + sigma, 0.055 + sigma, innovation); + vec3 weight = confidence * vec3(0.90, 0.92, 0.92); + vec3 result = mix(current, clipped, weight); + outColor = vec4(clamp(toRgb(result), 0.0, 1.0), c.x); +} +`; + +interface HistoryTarget { + texture: WebGLTexture; + fbo: WebGLFramebuffer; +} + +export class TemporalDenoiser { + private program: WebGLProgram | null = null; + private texelSizeLocation: WebGLUniformLocation | null = null; + private flipYLocation: WebGLUniformLocation | null = null; + private hasHistoryLocation: WebGLUniformLocation | null = null; + private targets: HistoryTarget[] = []; + private width = 0; + private height = 0; + private writeIndex = 0; + private hasHistory = false; + private floatHistory = false; + + init(gl: WebGL2RenderingContext): void { + this.program = createProgram(gl, FRAMEBUFFER_VERTEX_SHADER, FRAGMENT_SHADER); + this.texelSizeLocation = gl.getUniformLocation(this.program, "u_texelSize"); + this.flipYLocation = gl.getUniformLocation(this.program, "u_flipY"); + this.hasHistoryLocation = gl.getUniformLocation(this.program, "u_hasHistory"); + this.floatHistory = !!gl.getExtension("EXT_color_buffer_float"); + // biome-ignore lint/correctness/useHookAtTopLevel: WebGL useProgram, not a React hook + gl.useProgram(this.program); + gl.uniform1i(gl.getUniformLocation(this.program, "u_input"), 0); + gl.uniform1i(gl.getUniformLocation(this.program, "u_history"), 1); + } + + render(gl: WebGL2RenderingContext, input: WebGLTexture, width: number, height: number, flipY: boolean): WebGLTexture { + if (!this.program) throw new Error("TemporalDenoiser.render() called before init()"); + this.ensureTargets(gl, width, height); + const target = this.targets[this.writeIndex]; + const previous = this.targets[1 - this.writeIndex]; + gl.bindFramebuffer(gl.FRAMEBUFFER, target.fbo); + gl.viewport(0, 0, width, height); + // biome-ignore lint/correctness/useHookAtTopLevel: WebGL useProgram, not a React hook + gl.useProgram(this.program); + gl.activeTexture(gl.TEXTURE0); + gl.bindTexture(gl.TEXTURE_2D, input); + gl.activeTexture(gl.TEXTURE1); + gl.bindTexture(gl.TEXTURE_2D, previous.texture); + gl.uniform2f(this.texelSizeLocation, 1 / width, 1 / height); + gl.uniform1i(this.flipYLocation, flipY ? 1 : 0); + gl.uniform1i(this.hasHistoryLocation, this.hasHistory ? 1 : 0); + gl.drawArrays(gl.TRIANGLES, 0, 3); + this.writeIndex = 1 - this.writeIndex; + this.hasHistory = true; + return target.texture; + } + + /** Call on seeks, source/stage changes, and before re-rendering the same frame. */ + reset(): void { + this.hasHistory = false; + } + + private ensureTargets(gl: WebGL2RenderingContext, width: number, height: number): void { + if (this.width === width && this.height === height && this.targets.length === 2) return; + this.releaseTransientResources(gl); + try { + for (let i = 0; i < 2; i++) { + const texture = gl.createTexture(); + const fbo = gl.createFramebuffer(); + if (!texture || !fbo) { + gl.deleteTexture(texture); + gl.deleteFramebuffer(fbo); + throw new Error("Unable to allocate video denoise history"); + } + this.targets.push({ texture, fbo }); + gl.bindTexture(gl.TEXTURE_2D, texture); + // Half floats avoid accumulating 8-bit rounding errors in dark gradients. + gl.texStorage2D(gl.TEXTURE_2D, 1, this.floatHistory ? gl.RGBA16F : gl.RGBA8, width, height); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); + gl.bindFramebuffer(gl.FRAMEBUFFER, fbo); + gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, texture, 0); + if (gl.checkFramebufferStatus(gl.FRAMEBUFFER) !== gl.FRAMEBUFFER_COMPLETE) { + throw new Error("Incomplete video denoise framebuffer"); + } + } + } catch (error) { + this.releaseTransientResources(gl); + throw error; + } + this.width = width; + this.height = height; + } + + releaseTransientResources(gl: WebGL2RenderingContext): void { + for (const target of this.targets) { + gl.deleteTexture(target.texture); + gl.deleteFramebuffer(target.fbo); + } + this.targets = []; + this.width = this.height = this.writeIndex = 0; + this.reset(); + } + + destroy(gl: WebGL2RenderingContext): void { + this.releaseTransientResources(gl); + gl.deleteProgram(this.program); + this.program = null; + } +} diff --git a/web-ui/src/playback-engine/render/filters/gl-utils.ts b/web-ui/src/playback-engine/render/filters/gl-utils.ts index d8ced765..2d8d816c 100644 --- a/web-ui/src/playback-engine/render/filters/gl-utils.ts +++ b/web-ui/src/playback-engine/render/filters/gl-utils.ts @@ -20,20 +20,26 @@ export function createProgram(gl: WebGL2RenderingContext, vertexSource: string, if (!program) { throw new Error("Failed to create program"); } - const vs = compileShader(gl, gl.VERTEX_SHADER, vertexSource); - const fs = compileShader(gl, gl.FRAGMENT_SHADER, fragmentSource); - gl.attachShader(program, vs); - gl.attachShader(program, fs); - gl.linkProgram(program); - // Shaders are owned by the program after linking - gl.deleteShader(vs); - gl.deleteShader(fs); - if (!gl.getProgramParameter(program, gl.LINK_STATUS) && !gl.isContextLost()) { - const info = gl.getProgramInfoLog(program); + let vs: WebGLShader | null = null; + let fs: WebGLShader | null = null; + try { + vs = compileShader(gl, gl.VERTEX_SHADER, vertexSource); + fs = compileShader(gl, gl.FRAGMENT_SHADER, fragmentSource); + gl.attachShader(program, vs); + gl.attachShader(program, fs); + gl.linkProgram(program); + if (!gl.getProgramParameter(program, gl.LINK_STATUS) && !gl.isContextLost()) { + throw new Error(`Program link failed: ${gl.getProgramInfoLog(program)}`); + } + return program; + } catch (error) { gl.deleteProgram(program); - throw new Error(`Program link failed: ${info}`); + throw error; + } finally { + // Also release a successfully compiled vertex shader if fragment compilation fails. + if (vs) gl.deleteShader(vs); + if (fs) gl.deleteShader(fs); } - return program; } /** diff --git a/web-ui/src/playback-engine/render/filters/mosquito-nr.ts b/web-ui/src/playback-engine/render/filters/mosquito-nr.ts deleted file mode 100644 index 659ff239..00000000 --- a/web-ui/src/playback-engine/render/filters/mosquito-nr.ts +++ /dev/null @@ -1,205 +0,0 @@ -import { createProgram, FRAMEBUFFER_VERTEX_SHADER } from "./gl-utils"; -import { type RenderParams, registerFilter, type VideoFilter } from "./types"; - -/** - * Spatial compression-artifact reduction ahead of FSR EASU: mosquito/ringing - * around high-contrast edges (burned-in subtitles, jersey numbers, silhouettes) - * plus light chroma bleed and blocking in flat regions. - * - * A 3×3 box would melt soft motion-blurred edges (running players into grass). - * Taps are luma-range weighted instead (cheap bilateral): similar neighbors - * average, real edges keep the center. Extra ±2-px cross taps widen support - * for ringing that sits 2 px off an edge without a full 5×5. - * - * - `onEdge` (center Sobel) reduces luma NR on the sharpest strokes so numbers - * and glyph interiors stay crisp; chroma NR is not gated (color bleed lives - * on the stroke itself). - * - `nearEdge` (wider luma range) is the halo where mosquito lives. - * - Flat regions get a modest luma mix to take the edge off 8×8 blocking. - * - * Thresholds are full-range 0–1 luma (Sobel abs-sum is ~0–4). RANGE_SIGMA is - * mosquito amplitude, not edge height — a white-on-green number (~0.6 luma - * jump) stays unmixed. Raising LUMA_NEAR eats halos harder; raising - * RANGE_SIGMA starts to smear silhouettes. - * - * Shader uniforms: - * - u_input: current frame (raw video upload or a prior framebuffer). - * - u_texelSize: 1/width, 1/height. - * - u_flipY: 1 when sampling a raw DOM video upload. - */ - -const FRAGMENT_SHADER = /*glsl*/ `#version 300 es -precision highp float; - -uniform sampler2D u_input; -uniform vec2 u_texelSize; - -in vec2 v_texCoord; -out vec4 outColor; - -const vec3 LUMA = vec3(0.2126, 0.7152, 0.0722); - -// Center Sobel abs-sum. A white/black step is ~3.2; protect only the -// sharpest transitions (numbers, subtitle strokes). Soft silhouettes sit -// lower and stay eligible for range-weighted NR. -const float EDGE_LO = 1.4; -const float EDGE_HI = 2.6; - -// Wider (3×3 plus ±2-px cross) luma range. Nearby high-contrast edges push -// this toward 0.5–1.0; mosquito-only neighborhoods sit lower. -const float NEAR_LO = 0.08; -const float NEAR_HI = 0.38; - -// 3×3 luma range for the flat-region mix. Low-bitrate grass/jersey blocking -// is ~0.02–0.10; keep this modest so faces in cleaner sources do not go plastic. -const float FLAT_LO = 0.02; -const float FLAT_HI = 0.12; - -// Luma delta at which a neighbor is excluded. Mosquito speckle is below this; -// a jersey-number / grass step is far above it. -const float RANGE_SIGMA = 0.11; - -const float LUMA_NEAR = 0.82; -const float LUMA_FLAT = 0.16; -const float EDGE_KEEP = 0.45; -const float CHROMA_BASE = 0.38; -const float CHROMA_NEAR = 0.72; -const float CHROMA_FLAT = 0.28; - -vec3 sampleRgb(vec2 uv) { - return texture(u_input, uv).rgb; -} - -float rangeW(float luma, float centerLuma) { - return 1.0 - smoothstep(0.0, RANGE_SIGMA, abs(luma - centerLuma)); -} - -void accum(inout vec3 acc, inout float wSum, vec3 color, float luma, float centerLuma) { - float w = rangeW(luma, centerLuma); - acc += color * w; - wSum += w; -} - -void main() { - vec2 uv = v_texCoord; - vec2 dx = vec2(u_texelSize.x, 0.0); - vec2 dy = vec2(0.0, u_texelSize.y); - - vec3 nw = sampleRgb(uv - dx - dy); - vec3 n = sampleRgb(uv - dy); - vec3 ne = sampleRgb(uv + dx - dy); - vec3 w = sampleRgb(uv - dx); - vec3 c = sampleRgb(uv); - vec3 e = sampleRgb(uv + dx); - vec3 sw = sampleRgb(uv - dx + dy); - vec3 s = sampleRgb(uv + dy); - vec3 se = sampleRgb(uv + dx + dy); - - vec3 nn = sampleRgb(uv - 2.0 * dy); - vec3 ss = sampleRgb(uv + 2.0 * dy); - vec3 ww = sampleRgb(uv - 2.0 * dx); - vec3 ee = sampleRgb(uv + 2.0 * dx); - - float nwL = dot(nw, LUMA); - float nL = dot(n, LUMA); - float neL = dot(ne, LUMA); - float wL = dot(w, LUMA); - float cL = dot(c, LUMA); - float eL = dot(e, LUMA); - float swL = dot(sw, LUMA); - float sL = dot(s, LUMA); - float seL = dot(se, LUMA); - float nnL = dot(nn, LUMA); - float ssL = dot(ss, LUMA); - float wwL = dot(ww, LUMA); - float eeL = dot(ee, LUMA); - - float gx = -nwL - 2.0 * wL - swL + neL + 2.0 * eL + seL; - float gy = -nwL - 2.0 * nL - neL + swL + 2.0 * sL + seL; - float grad = abs(gx) + abs(gy); - float onEdge = smoothstep(EDGE_LO, EDGE_HI, grad); - - float min3 = min(min(min(nwL, nL), min(neL, wL)), min(min(cL, eL), min(swL, min(sL, seL)))); - float max3 = max(max(max(nwL, nL), max(neL, wL)), max(max(cL, eL), max(swL, max(sL, seL)))); - float range3 = max3 - min3; - - float min5 = min(min3, min(min(nnL, ssL), min(wwL, eeL))); - float max5 = max(max3, max(max(nnL, ssL), max(wwL, eeL))); - float range5 = max5 - min5; - - float nearEdge = smoothstep(NEAR_LO, NEAR_HI, range5); - float flatness = 1.0 - smoothstep(FLAT_LO, FLAT_HI, range3); - - vec3 acc = vec3(0.0); - float wSum = 0.0; - accum(acc, wSum, nw, nwL, cL); - accum(acc, wSum, n, nL, cL); - accum(acc, wSum, ne, neL, cL); - accum(acc, wSum, w, wL, cL); - accum(acc, wSum, c, cL, cL); - accum(acc, wSum, e, eL, cL); - accum(acc, wSum, sw, swL, cL); - accum(acc, wSum, s, sL, cL); - accum(acc, wSum, se, seL, cL); - accum(acc, wSum, nn, nnL, cL); - accum(acc, wSum, ss, ssL, cL); - accum(acc, wSum, ww, wwL, cL); - accum(acc, wSum, ee, eeL, cL); - vec3 filtered = acc / max(wSum, 1e-6); - - float lumaMix = clamp((nearEdge * LUMA_NEAR + flatness * LUMA_FLAT) * mix(1.0, EDGE_KEEP, onEdge), 0.0, 1.0); - float chromaMix = clamp(CHROMA_BASE + nearEdge * CHROMA_NEAR + flatness * CHROMA_FLAT, 0.0, 1.0); - - float yC = cL; - float yB = dot(filtered, LUMA); - float yOut = mix(yC, yB, lumaMix); - - vec2 chromaC = vec2(c.b - yC, c.r - yC); - vec2 chromaB = vec2(filtered.b - yB, filtered.r - yB); - vec2 chromaOut = mix(chromaC, chromaB, chromaMix); - - float b = yOut + chromaOut.x; - float r = yOut + chromaOut.y; - float g = (yOut - LUMA.x * r - LUMA.z * b) / LUMA.y; - - outColor = vec4(clamp(vec3(r, g, b), 0.0, 1.0), 1.0); -} -`; - -class MosquitoNrFilter implements VideoFilter { - readonly name = "mosquito-nr"; - readonly historyFrames = 0; - - private program: WebGLProgram | null = null; - private uTexelSize: WebGLUniformLocation | null = null; - private uFlipY: WebGLUniformLocation | null = null; - - init(gl: WebGL2RenderingContext): void { - this.program = createProgram(gl, FRAMEBUFFER_VERTEX_SHADER, FRAGMENT_SHADER); - // biome-ignore lint/correctness/useHookAtTopLevel: WebGL useProgram, not a React hook - gl.useProgram(this.program); - gl.uniform1i(gl.getUniformLocation(this.program, "u_input"), 0); - this.uTexelSize = gl.getUniformLocation(this.program, "u_texelSize"); - this.uFlipY = gl.getUniformLocation(this.program, "u_flipY"); - } - - render(gl: WebGL2RenderingContext, textures: WebGLTexture[], params: RenderParams): void { - if (!this.program) return; - // biome-ignore lint/correctness/useHookAtTopLevel: WebGL useProgram, not a React hook - gl.useProgram(this.program); - gl.activeTexture(gl.TEXTURE0); - gl.bindTexture(gl.TEXTURE_2D, textures[0]); - gl.uniform2f(this.uTexelSize, 1 / params.width, 1 / params.height); - gl.uniform1i(this.uFlipY, params.flipY ? 1 : 0); - gl.drawArrays(gl.TRIANGLES, 0, 3); - } - - destroy(gl: WebGL2RenderingContext): void { - if (this.program) { - gl.deleteProgram(this.program); - this.program = null; - } - } -} - -registerFilter("mosquito-nr", () => new MosquitoNrFilter()); diff --git a/web-ui/src/playback-engine/render/fsr.ts b/web-ui/src/playback-engine/render/fsr.ts index 4ccddf05..af19ec5a 100644 --- a/web-ui/src/playback-engine/render/fsr.ts +++ b/web-ui/src/playback-engine/render/fsr.ts @@ -4,8 +4,7 @@ import type { Presenter } from "./presenters"; /** * AMD FidelityFX Super Resolution 1 (FSR1) upscale presenter: EASU * (Edge-Adaptive Spatial Upsampling) followed by RCAS (Robust Contrast - * Adaptive Sharpening), replacing the Catmull-Rom bicubic presenter as the - * enhancement path's upscaler. + * Adaptive Sharpening). * * Ported to WebGL2 GLSL ES 300 from AMD's reference (ffx_fsr1.h, MIT * licensed, Copyright (c) 2021 Advanced Micro Devices, Inc.), following the @@ -17,18 +16,15 @@ import type { Presenter } from "./presenters"; * derived from the RGB taps, and the resulting kernel weights are shared * across all three channels, since the weights never depend on tap color. * - * RCAS's contrast/sharpness lobe replaces the old dedicated "sharpen" filter - * (see filters/sharpen.ts, removed), so its output also folds in the same - * mild contrast/saturation lift that filter used to apply. + * RCAS sharpens luminance with noise rejection and local extrema bounds, + * then applies a mild contrast and saturation lift. */ /** * EASU (Edge-Adaptive Spatial Upsampling), ported from AMD's FsrEasuF. * - * easuRcp/easuRsqrt: AMD's AAxxRcpF1/RsqF1 fast bit-hack approximations. - * Unlike a native 1.0/x, these return a large finite value at x=0 instead of - * Infinity -- flat image regions hit x=0 constantly, and a subsequent - * 0 * Infinity would poison the result with NaN. + * Reciprocals are guarded at zero. Exact arithmetic avoids bias in low + * contrast detail and handles flat black/white regions without NaNs. * * easuTap (AMD's FsrEasuTap): accumulates one Lanczos-2-approximation tap * into the RGB/weight accumulators, via a base*window product that avoids @@ -57,6 +53,7 @@ import type { Presenter } from "./presenters"; const EASU_FRAGMENT_SHADER = /*glsl*/ `#version 300 es precision highp float; precision highp int; +precision highp sampler2D; uniform sampler2D u_input; uniform vec2 u_srcSize; @@ -68,17 +65,17 @@ out vec4 outColor; const vec3 LUMA = vec3(0.2126, 0.7152, 0.0722); float easuRcp(float x) { - return uintBitsToFloat(0x7ef07ebbu - floatBitsToUint(x)); + return 1.0 / max(x, 1e-8); } float easuRsqrt(float x) { - return uintBitsToFloat(0x5f347d74u - (floatBitsToUint(x) >> 1u)); + return inversesqrt(max(x, 1e-8)); } vec3 min3(vec3 a, vec3 b, vec3 c) { return min(a, min(b, c)); } vec3 max3(vec3 a, vec3 b, vec3 c) { return max(a, max(b, c)); } -vec3 sampleSrc(vec2 pixelPos) { - return texture(u_input, (pixelPos + 0.5) / u_srcSize).rgb; +vec4 sampleSrc(vec2 pixelPos) { + return texture(u_input, (pixelPos + 0.5) / u_srcSize); } void easuTap( @@ -148,31 +145,31 @@ void main() { vec2 fp = floor(pp); pp -= fp; - vec3 b = sampleSrc(fp + vec2(0.0, -1.0)); - vec3 c = sampleSrc(fp + vec2(1.0, -1.0)); - vec3 e = sampleSrc(fp + vec2(-1.0, 0.0)); - vec3 f = sampleSrc(fp + vec2(0.0, 0.0)); - vec3 g = sampleSrc(fp + vec2(1.0, 0.0)); - vec3 h = sampleSrc(fp + vec2(2.0, 0.0)); - vec3 i = sampleSrc(fp + vec2(-1.0, 1.0)); - vec3 j = sampleSrc(fp + vec2(0.0, 1.0)); - vec3 k = sampleSrc(fp + vec2(1.0, 1.0)); - vec3 l = sampleSrc(fp + vec2(2.0, 1.0)); - vec3 n = sampleSrc(fp + vec2(0.0, 2.0)); - vec3 o = sampleSrc(fp + vec2(1.0, 2.0)); - - float bL = dot(b, LUMA); - float cL = dot(c, LUMA); - float eL = dot(e, LUMA); - float fL = dot(f, LUMA); - float gL = dot(g, LUMA); - float hL = dot(h, LUMA); - float iL = dot(i, LUMA); - float jL = dot(j, LUMA); - float kL = dot(k, LUMA); - float lL = dot(l, LUMA); - float nL = dot(n, LUMA); - float oL = dot(o, LUMA); + vec4 b = sampleSrc(fp + vec2(0.0, -1.0)); + vec4 c = sampleSrc(fp + vec2(1.0, -1.0)); + vec4 e = sampleSrc(fp + vec2(-1.0, 0.0)); + vec4 f = sampleSrc(fp + vec2(0.0, 0.0)); + vec4 g = sampleSrc(fp + vec2(1.0, 0.0)); + vec4 h = sampleSrc(fp + vec2(2.0, 0.0)); + vec4 i = sampleSrc(fp + vec2(-1.0, 1.0)); + vec4 j = sampleSrc(fp + vec2(0.0, 1.0)); + vec4 k = sampleSrc(fp + vec2(1.0, 1.0)); + vec4 l = sampleSrc(fp + vec2(2.0, 1.0)); + vec4 n = sampleSrc(fp + vec2(0.0, 2.0)); + vec4 o = sampleSrc(fp + vec2(1.0, 2.0)); + + float bL = dot(b.rgb, LUMA); + float cL = dot(c.rgb, LUMA); + float eL = dot(e.rgb, LUMA); + float fL = dot(f.rgb, LUMA); + float gL = dot(g.rgb, LUMA); + float hL = dot(h.rgb, LUMA); + float iL = dot(i.rgb, LUMA); + float jL = dot(j.rgb, LUMA); + float kL = dot(k.rgb, LUMA); + float lL = dot(l.rgb, LUMA); + float nL = dot(n.rgb, LUMA); + float oL = dot(o.rgb, LUMA); vec2 dir = vec2(0.0); float len = 0.0; @@ -198,77 +195,56 @@ void main() { vec3 accumColor = vec3(0.0); float accumWeight = 0.0; - easuTap(accumColor, accumWeight, vec2(0.0, -1.0) - pp, dir, len2, lobe, clip, b); - easuTap(accumColor, accumWeight, vec2(1.0, -1.0) - pp, dir, len2, lobe, clip, c); - easuTap(accumColor, accumWeight, vec2(-1.0, 1.0) - pp, dir, len2, lobe, clip, i); - easuTap(accumColor, accumWeight, vec2(0.0, 1.0) - pp, dir, len2, lobe, clip, j); - easuTap(accumColor, accumWeight, vec2(0.0, 0.0) - pp, dir, len2, lobe, clip, f); - easuTap(accumColor, accumWeight, vec2(-1.0, 0.0) - pp, dir, len2, lobe, clip, e); - easuTap(accumColor, accumWeight, vec2(1.0, 1.0) - pp, dir, len2, lobe, clip, k); - easuTap(accumColor, accumWeight, vec2(2.0, 1.0) - pp, dir, len2, lobe, clip, l); - easuTap(accumColor, accumWeight, vec2(2.0, 0.0) - pp, dir, len2, lobe, clip, h); - easuTap(accumColor, accumWeight, vec2(1.0, 0.0) - pp, dir, len2, lobe, clip, g); - easuTap(accumColor, accumWeight, vec2(1.0, 2.0) - pp, dir, len2, lobe, clip, o); - easuTap(accumColor, accumWeight, vec2(0.0, 2.0) - pp, dir, len2, lobe, clip, n); + easuTap(accumColor, accumWeight, vec2(0.0, -1.0) - pp, dir, len2, lobe, clip, b.rgb); + easuTap(accumColor, accumWeight, vec2(1.0, -1.0) - pp, dir, len2, lobe, clip, c.rgb); + easuTap(accumColor, accumWeight, vec2(-1.0, 1.0) - pp, dir, len2, lobe, clip, i.rgb); + easuTap(accumColor, accumWeight, vec2(0.0, 1.0) - pp, dir, len2, lobe, clip, j.rgb); + easuTap(accumColor, accumWeight, vec2(0.0, 0.0) - pp, dir, len2, lobe, clip, f.rgb); + easuTap(accumColor, accumWeight, vec2(-1.0, 0.0) - pp, dir, len2, lobe, clip, e.rgb); + easuTap(accumColor, accumWeight, vec2(1.0, 1.0) - pp, dir, len2, lobe, clip, k.rgb); + easuTap(accumColor, accumWeight, vec2(2.0, 1.0) - pp, dir, len2, lobe, clip, l.rgb); + easuTap(accumColor, accumWeight, vec2(2.0, 0.0) - pp, dir, len2, lobe, clip, h.rgb); + easuTap(accumColor, accumWeight, vec2(1.0, 0.0) - pp, dir, len2, lobe, clip, g.rgb); + easuTap(accumColor, accumWeight, vec2(1.0, 2.0) - pp, dir, len2, lobe, clip, o.rgb); + easuTap(accumColor, accumWeight, vec2(0.0, 2.0) - pp, dir, len2, lobe, clip, n.rgb); vec3 rgb = accumColor / accumWeight; - vec3 lo = min(min3(f, g, j), k); - vec3 hi = max(max3(f, g, j), k); + vec3 lo = min(min3(f.rgb, g.rgb, j.rgb), k.rgb); + vec3 hi = max(max3(f.rgb, g.rgb, j.rgb), k.rgb); rgb = clamp(rgb, lo, hi); - outColor = vec4(clamp(rgb, 0.0, 1.0), 1.0); + // Carry the amount removed by NR into sharpening. RGB10_A2 provides four + // noise levels without increasing the intermediate's bandwidth. + float noise = mix(mix(abs(f.a - fL), abs(g.a - gL), pp.x), + mix(abs(j.a - jL), abs(k.a - kL), pp.x), pp.y); + outColor = vec4(clamp(rgb, 0.0, 1.0), clamp(noise * 32.0, 0.0, 1.0)); } `; /** - * RCAS (Robust Contrast Adaptive Sharpening), ported from AMD's FsrRcasF. - * - * SHARPNESS: AMD scale, 0.0 = strongest sharpening, higher N = N stops - * (halvings) weaker. RCAS_LIMIT (AMD FSR_RCAS_LIMIT): clamps the sharpening - * lobe to avoid unnatural results. CONTRAST/SATURATION: the same mild tone - * lift the old dedicated "sharpen" enhancement filter used to apply. - * - * rcasRcp: medium-precision reciprocal (fast bit-hack estimate plus one - * Newton-Raphson refinement step), used in place of AMD's plain division - * because RCAS's clipping-lobe math divides by neighborhood min/max ranges - * that legitimately hit exactly zero on flat video content (solid black - * letterboxing, blown-out highlights), where native division would yield - * 0/0 = NaN. This approximation returns a large finite value at zero - * instead, so the following multiply-by-numerator collapses cleanly to zero - * rather than propagating NaN. - * - * main reads the 3x3 cross neighborhood around the output pixel: - * b - * d e f - * h - * derives a sharpening lobe from the local min/max contrast range (clamped - * by RCAS_LIMIT/SHARPNESS), de-weights it in flat/noisy regions (comparing - * the 4-neighbor average against the center to avoid amplifying compression - * noise/grain; 0.9 is stronger than AMD's 0.5 so leftover mosquito around - * edges is less likely to be re-sharpened), and blends the cross taps with - * the center by that lobe before applying the CONTRAST/SATURATION lift. + * Contrast-adaptive luma sharpening, with or without EASU. A symmetric cross + * detects isolated noise; sharpened luma stays inside the local range and RGB + * gamut before a mild contrast and saturation lift. */ const RCAS_FRAGMENT_SHADER = /*glsl*/ `#version 300 es precision highp float; precision highp int; +precision highp sampler2D; uniform sampler2D u_input; uniform vec2 u_texelSize; +uniform vec2 u_scale; uniform bool u_flipY; +uniform bool u_upscaled; out vec4 outColor; const vec3 LUMA = vec3(0.2126, 0.7152, 0.0722); -const float SHARPNESS = 0.2; -const float RCAS_LIMIT = 0.25 - 1.0 / 16.0; +const float SHARPNESS = 0.45; const float CONTRAST = 1.04; const float SATURATION = 1.03; - -float rcasRcp(float a) { - float b = uintBitsToFloat(0x7ef19fffu - floatBitsToUint(a)); - return b * (-b * a + 2.0); -} +const float RCAS_LIMIT = 0.25 - 1.0 / 16.0; float min3(float a, float b, float c) { return min(a, min(b, c)); } float max3(float a, float b, float c) { return max(a, max(b, c)); } @@ -278,7 +254,8 @@ void main() { if (u_flipY) uv.y = 1.0 - uv.y; vec3 b = texture(u_input, uv + vec2(0.0, -1.0) * u_texelSize).rgb; vec3 d = texture(u_input, uv + vec2(-1.0, 0.0) * u_texelSize).rgb; - vec3 e = texture(u_input, uv).rgb; + vec4 center = texture(u_input, uv); + vec3 e = center.rgb; vec3 f = texture(u_input, uv + vec2(1.0, 0.0) * u_texelSize).rgb; vec3 h = texture(u_input, uv + vec2(0.0, 1.0) * u_texelSize).rgb; @@ -291,24 +268,31 @@ void main() { float mn = min(min3(bL, dL, fL), hL); float mx = max(max3(bL, dL, fL), hL); - float hitMin = min(mn, eL) * rcasRcp(4.0 * mx); - float hitMax = (1.0 - max(mx, eL)) * rcasRcp(4.0 * mn - 4.0); + float hitMin = min(mn, eL) / max(4.0 * mx, 1e-6); + float hitMax = (1.0 - max(mx, eL)) / min(4.0 * mn - 4.0, -1e-6); float lobeShape = max(-hitMin, hitMax); float lobe = max(-RCAS_LIMIT, min(lobeShape, 0.0)) * exp2(-SHARPNESS); float mn5 = min(mn, eL); float mx5 = max(mx, eL); float noise = 0.25 * (bL + dL + fL + hL) - eL; - noise = clamp(abs(noise) * rcasRcp(mx5 - mn5), 0.0, 1.0); + noise = clamp(abs(noise) / max(mx5 - mn5, 1e-6), 0.0, 1.0); lobe *= 1.0 - 0.9 * noise; - - float rcp = rcasRcp(4.0 * lobe + 1.0); - vec3 rgb = (lobe * (b + d + f + h) + e) * rcp; + float removedNoise = u_upscaled ? center.a : clamp(abs(center.a - eL) * 32.0, 0.0, 1.0); + lobe *= 1.0 - 0.85 * removedNoise; + + // Express output-pixel differences in source-pixel units so upscaling + // does not suppress sharpening just because adjacent samples get closer. + vec2 gradient = abs(vec2(fL - dL, hL - bL)) * u_scale; + float edge = max(gradient.x, gradient.y); + lobe *= smoothstep(2.0 / 255.0, 12.0 / 255.0, edge); + float y = (lobe * (bL + dL + fL + hL) + eL) / (4.0 * lobe + 1.0); + float lo = max(mn5 - eL, -min3(e.r, e.g, e.b)); + float hi = min(mx5 - eL, 1.0 - max3(e.r, e.g, e.b)); + vec3 rgb = e + clamp(y - eL, lo, hi); rgb = (rgb - 0.5) * CONTRAST + 0.5; - float y = dot(rgb, LUMA); - rgb = mix(vec3(y), rgb, SATURATION); - + rgb = mix(vec3(dot(rgb, LUMA)), rgb, SATURATION); outColor = vec4(clamp(rgb, 0.0, 1.0), 1.0); } `; @@ -322,9 +306,11 @@ interface IntermediateTarget { /** * FSR1 upscale presenter: EASU (source -> intermediate, at output size) then - * RCAS (intermediate -> bound framebuffer). RCAS also runs standalone - * (skipping EASU) when the output is not larger than the source, so picture - * enhancement still sharpens at native size instead of doing nothing. + * RCAS (intermediate -> bound framebuffer). When the output is no larger + * than the source, skip EASU and run RCAS directly so enhancement still + * sharpens and adjusts contrast and saturation at native size. + * Input RGB must be denoised, with original luma in alpha (TemporalDenoiser's + * output). Alpha informs noise-aware sharpening; it is not image opacity. */ export class FsrPresenter implements Presenter { readonly name = "fsr-present"; @@ -338,7 +324,9 @@ export class FsrPresenter implements Presenter { private rcasProgram: WebGLProgram | null = null; private rcasInputLocation: WebGLUniformLocation | null = null; private rcasTexelSizeLocation: WebGLUniformLocation | null = null; + private rcasScaleLocation: WebGLUniformLocation | null = null; private rcasFlipYLocation: WebGLUniformLocation | null = null; + private rcasUpscaledLocation: WebGLUniformLocation | null = null; private intermediate: IntermediateTarget | null = null; @@ -358,7 +346,9 @@ export class FsrPresenter implements Presenter { } this.rcasInputLocation = gl.getUniformLocation(this.rcasProgram, "u_input"); this.rcasTexelSizeLocation = gl.getUniformLocation(this.rcasProgram, "u_texelSize"); + this.rcasScaleLocation = gl.getUniformLocation(this.rcasProgram, "u_scale"); this.rcasFlipYLocation = gl.getUniformLocation(this.rcasProgram, "u_flipY"); + this.rcasUpscaledLocation = gl.getUniformLocation(this.rcasProgram, "u_upscaled"); } present( @@ -381,7 +371,7 @@ export class FsrPresenter implements Presenter { const upscaling = dstWidth > srcWidth + 0.5 || dstHeight > srcHeight + 0.5; if (!upscaling) { - this.runRcas(gl, texture, dstWidth, dstHeight, outputFbo, flipY); + this.runRcas(gl, texture, dstWidth, dstHeight, outputFbo, flipY, false); return; } @@ -408,7 +398,17 @@ export class FsrPresenter implements Presenter { // The intermediate is a framebuffer-rendered texture (native orientation), // regardless of whether the EASU input needed a flip. - this.runRcas(gl, target.texture, dstWidth, dstHeight, outputFbo, false); + this.runRcas( + gl, + target.texture, + dstWidth, + dstHeight, + outputFbo, + false, + true, + Math.max(1, dstWidth / srcWidth), + Math.max(1, dstHeight / srcHeight), + ); } private runRcas( @@ -418,6 +418,9 @@ export class FsrPresenter implements Presenter { height: number, targetFbo: WebGLFramebuffer | null, flipY: boolean, + upscaled: boolean, + scaleX = 1, + scaleY = 1, ): void { gl.bindFramebuffer(gl.FRAMEBUFFER, targetFbo); gl.viewport(0, 0, width, height); @@ -427,7 +430,9 @@ export class FsrPresenter implements Presenter { gl.bindTexture(gl.TEXTURE_2D, inputTexture); gl.uniform1i(this.rcasInputLocation, 0); gl.uniform2f(this.rcasTexelSizeLocation, 1 / width, 1 / height); + gl.uniform2f(this.rcasScaleLocation, scaleX, scaleY); gl.uniform1i(this.rcasFlipYLocation, flipY ? 1 : 0); + gl.uniform1i(this.rcasUpscaledLocation, upscaled ? 1 : 0); gl.drawArrays(gl.TRIANGLES, 0, 3); } @@ -442,7 +447,7 @@ export class FsrPresenter implements Presenter { const texture = gl.createTexture(); if (!texture) return null; gl.bindTexture(gl.TEXTURE_2D, texture); - gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, width, height, 0, gl.RGBA, gl.UNSIGNED_BYTE, null); + gl.texStorage2D(gl.TEXTURE_2D, 1, gl.RGB10_A2, width, height); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); @@ -494,6 +499,8 @@ export class FsrPresenter implements Presenter { this.easuFlipYLocation = null; this.rcasInputLocation = null; this.rcasTexelSizeLocation = null; + this.rcasScaleLocation = null; this.rcasFlipYLocation = null; + this.rcasUpscaledLocation = null; } } diff --git a/web-ui/src/playback-engine/render/index.ts b/web-ui/src/playback-engine/render/index.ts index a862df57..3b07e34c 100644 --- a/web-ui/src/playback-engine/render/index.ts +++ b/web-ui/src/playback-engine/render/index.ts @@ -1,5 +1,4 @@ import "./filters/bwdif"; -import "./filters/mosquito-nr"; import type { PlayerRenderState, PlayerVideoScanType } from "../types"; import Log from "../utils/logger"; import { isRenderResolutionEligible, type RenderStageName, VideoRenderer } from "./renderer"; @@ -88,9 +87,6 @@ export function createVideoRenderPipeline( const desiredStage = (): RenderStageName => autoDeinterlaceEnabled && scanType === "interlaced" ? "bwdif" : "passthrough"; - const formatVideoSize = () => - video.videoWidth > 0 && video.videoHeight > 0 ? `${video.videoWidth}x${video.videoHeight}` : "unknown"; - const renderer = new VideoRenderer( video, canvas, @@ -108,9 +104,8 @@ export function createVideoRenderPipeline( ); renderer.setPictureEnhancementEnabled(pictureEnhancementEnabled); - renderer.onFrameOutsideRenderGate = () => { + renderer.onFrameSizeChange = () => { if (destroyed) return; - lastEligibility = null; apply(); }; @@ -160,12 +155,16 @@ export function createVideoRenderPipeline( }; const apply = () => { + const { width, height } = renderer.frameSize; const eligible = - video.videoWidth > 0 && video.videoHeight > 0 && isRenderResolutionEligible(video.videoWidth, video.videoHeight); + isRenderResolutionEligible(width, height) && isRenderResolutionEligible(video.videoWidth, video.videoHeight); if (eligible !== lastEligibility) { - if (eligible) Log.i(TAG, `Render gate enabled for ${formatVideoSize()}`); - else if (video.videoWidth > 0 && video.videoHeight > 0) { - Log.i(TAG, `Render gate disabled for ${formatVideoSize()}; falling back to raw video`); + if (eligible) Log.i(TAG, `Render gate enabled for ${width}x${height}`); + else if (width > 0 && height > 0) { + Log.i( + TAG, + `Render gate disabled for ${width}x${height} (${video.videoWidth}x${video.videoHeight} display); falling back to raw video`, + ); } lastEligibility = eligible; } @@ -187,12 +186,6 @@ export function createVideoRenderPipeline( setActive(true); }; - const handleVideoResize = () => { - if (destroyed) return; - apply(); - }; - video.addEventListener("resize", handleVideoResize); - apply(); return { @@ -228,7 +221,6 @@ export function createVideoRenderPipeline( }, destroy() { destroyed = true; - video.removeEventListener("resize", handleVideoResize); renderer.destroy(); }, }; diff --git a/web-ui/src/playback-engine/render/renderer.ts b/web-ui/src/playback-engine/render/renderer.ts index bfcf245a..054001fe 100644 --- a/web-ui/src/playback-engine/render/renderer.ts +++ b/web-ui/src/playback-engine/render/renderer.ts @@ -1,4 +1,5 @@ import Log from "../utils/logger"; +import { TemporalDenoiser } from "./denoise"; import { createFilter, type RenderParams, type VideoFilter } from "./filters/types"; import { FsrPresenter } from "./fsr"; import { PassthroughPresenter, type Presenter } from "./presenters"; @@ -17,16 +18,6 @@ export function isRenderResolutionEligible(width: number, height: number): boole return width > 0 && width <= GATE_MAX_WIDTH && height > 0 && height <= GATE_MAX_HEIGHT; } -/** - * Post-stage enhancement filters, applied in order between the source stage - * and presentation. All are registered in the filter registry and must be - * stateless (historyFrames = 0): the renderer re-runs the whole list per - * frame and assumes channel/stage switches need no per-filter reset. - * mosquito-nr runs at source resolution so FSR EASU does not reconstruct - * compression speckle around high-contrast edges. - */ -const ENHANCEMENT_FILTER_NAMES: readonly string[] = ["mosquito-nr"]; - /** * Safety ceiling for the enhanced canvas backing store, so a very large * display rect (or a stray devicePixelRatio) cannot push the per-frame @@ -42,11 +33,19 @@ interface RenderTarget { height: number; } +interface PendingField { + presentAt: number; + enhanced: boolean; + texture: WebGLTexture; + width: number; + height: number; +} + /** * WebGL2 render loop. It pulls decoded frames from the