diff --git a/.gitignore b/.gitignore index 81b687fe8c4..df6630aa3d2 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ lerna-debug.log* .deepsec node_modules +apps/cli/skill/cap-demo/node_modules/ dist dist-ssr *.local diff --git a/Cargo.lock b/Cargo.lock index 9a10a9e4c08..88ca9aca6df 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1213,6 +1213,7 @@ dependencies = [ "flume", "futures", "image 0.25.8", + "include_dir", "kameo", "keyring", "libc", @@ -4984,6 +4985,25 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8a5a9a0ff0086c7a148acb942baaabeadf9504d10400b5a05645853729b9cd2" +[[package]] +name = "include_dir" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "923d117408f1e49d914f1a379a309cffe4f18c05cf4e3d12e613a15fc81bd0dd" +dependencies = [ + "include_dir_macros", +] + +[[package]] +name = "include_dir_macros" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cab85a7ed0bd5f0e76d93846e0147172bed2e2d3f859bcc33a8d9699cad1a75" +dependencies = [ + "proc-macro2", + "quote", +] + [[package]] name = "indenter" version = "0.3.4" diff --git a/apps/cli/Cargo.toml b/apps/cli/Cargo.toml index e8fb5b520af..16648093c5c 100644 --- a/apps/cli/Cargo.toml +++ b/apps/cli/Cargo.toml @@ -28,6 +28,7 @@ kameo = "0.17.2" flume = { workspace = true } futures = { workspace = true } dirs = "6.0.0" +include_dir = "0.7" image = "0.25.2" chrono = "0.4.31" base64 = "0.22.1" diff --git a/apps/cli/build.rs b/apps/cli/build.rs new file mode 100644 index 00000000000..b99bd3ea57d --- /dev/null +++ b/apps/cli/build.rs @@ -0,0 +1,36 @@ +use std::path::Path; + +// The cap-demo skill directory is embedded wholesale into the `cap` binary via +// `include_dir!` in src/agents.rs. That macro has no exclusion filter and does +// not respect .gitignore, so anything on disk under skill/cap-demo at build +// time gets baked into every binary built from this checkout. The skill's own +// docs instruct running `npm install` inside that directory (it uses +// Playwright), which would silently embed the entire node_modules tree — +// including playwright-core — bloating the shipped CLI and making builds +// non-reproducible. Guard against that here, loudly, at build time. +fn main() { + let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR is set"); + let node_modules = Path::new(&manifest_dir) + .join("skill") + .join("cap-demo") + .join("node_modules"); + + // Re-run this check whenever the embedded skill tree changes (including + // when node_modules appears or disappears). + println!("cargo:rerun-if-changed=skill/cap-demo"); + + if node_modules.exists() { + println!( + "cargo:warning=skill/cap-demo/node_modules exists and would be embedded into the cap binary by include_dir!" + ); + panic!( + "\n\nskill/cap-demo/node_modules exists at build time.\n\ + The entire skill/cap-demo directory is embedded into the `cap` binary via \ + include_dir! (src/agents.rs), which does not respect .gitignore, so building \ + now would bake the whole node_modules tree (including playwright-core) into \ + every binary built from this checkout.\n\n\ + Fix: rm -rf apps/cli/skill/cap-demo/node_modules\n\ + (npm dependencies there are only needed at cap-demo runtime, never at build time.)\n\n" + ); + } +} diff --git a/apps/cli/skill/cap-demo/SKILL.md b/apps/cli/skill/cap-demo/SKILL.md new file mode 100644 index 00000000000..a0b9b92dc09 --- /dev/null +++ b/apps/cli/skill/cap-demo/SKILL.md @@ -0,0 +1,200 @@ +--- +name: cap-demo +description: Generate a cinematic 3D product-demo video from any URL — scouts the page, records it with virtual input, and treats it with Cap's 3D camera, brand background, and music. +--- + +# cap-demo + +Give it a URL, get back a short, cinematic product-demo video. The pipeline +scouts the page headlessly, shoots a headed browser with **virtual input only** +(CDP mouse/keyboard — the user's real cursor is never touched), records with +Cap's CLI, then treats the recording with Cap's 3D camera, a brand-matched +background gradient, a synthetic cursor, and music, and exports an mp4. + +> **macOS only (for now).** This skill records via Cap's window capture and +> resolves a cached Playwright Chromium under +> `~/Library/Caches/ms-playwright`, both of which are macOS-specific today. +> Apple Silicon is assumed (the bundled Chromium path is `chrome-mac-arm64`). + +## The skill directory + +`` below is the directory this `SKILL.md` was installed into. `cap +agents install` writes it next to the main `cap` skill, so it is one of: + +- Claude: `~/.claude/skills/cap-demo` +- Cursor: `~/.cursor/skills/cap-demo` +- Codex: `/skills/cap-demo` (default `~/.codex/skills/cap-demo`) + +There is no `cap-demo` binary on PATH — the `cap-demo` command is just +`node /cap-demo.mjs`. Alias it if you like: + +``` +alias cap-demo='node /cap-demo.mjs' +``` + +## Requirements + +- **macOS on Apple Silicon.** Windows/Linux are not supported yet. +- **Node 18+**, with `playwright-core` vendored into the skill. Run `npm + install` once inside `` on first use. It **reuses an already-cached + Chromium** under `~/Library/Caches/ms-playwright`; if none is cached, run + `npx playwright install chromium` once. +- **`python3`** and **`ffmpeg`/`ffprobe`** on PATH. +- The **`cap` CLI** on PATH (Cap Desktop, https://cap.so) with macOS + **screen-recording permission** granted to it. Both stages otherwise resolve + the binary from `CAP_BIN`; if neither is set they error clearly. + +## One-liner + +``` +cap-demo https://website.com +``` + +Output lands at `/-demo.mp4` (default `/cap-demo//-demo.mp4`). + +## Two modes + +- **Deterministic** (`cap-demo `): fixed heuristics, no agent in the loop. + Scores the best CTA, picks click-through vs scroll, paints a brand gradient + from the page colors, picks music by brand darkness. Good, ~70% quality. Use + it for batch runs or when you cannot watch the frames. +- **Agent-driven recipe** (recommended, ~95% quality): the agent runs the + same steps but frame-QAs each stage and applies judgment — sets the brand + color by eye, matches export fps to the real capture rate, reshoots on a + dead-end CTA, regrades until the beats read. This is the path that produced + the shipped demos. Follow the recipe below. + +## The agent recipe + +Run the two stages by hand so you can inspect between them. Both scripts live in +this skill; the orchestrator just chains them. + +### 1. Scout + shoot + +``` +node /lib/scout-shoot.mjs [--story click|scroll] +``` + +- Writes the recording to `/.cap` and the beat log to + `/.timeline.json`. +- Prints a final JSON line: `{"slug","story","scout":{title,hero,accent,pageBg,ctaText}}`. + Read it to see what it decided (story, chosen CTA, brand colors). +- It already: kills stale test-chrome, closes Finder windows, dismisses cookie + banners, injects the shimmer div, matches the capture window by exact page + title (asserts a single match), and logs the virtual cursor path. + +**Story choice.** Leave it to auto for most sites (a strong content CTA → +`click`, otherwise `scroll`). Force `--story scroll` for product one-pagers that +showcase best as a scroll of their own sections, or when the top CTA dead-ends +at a booking/login page. + +### 2. Frame-QA the raw recording (before treating) + +Extract a few beat frames and **look at them**: + +``` +ffmpeg -y -ss -i /.cap/content/segments/segment-0/display.mp4 -frames:v 1 /tmp/raw-.png +``` + +Check: the right window/content is captured (no leftover footage from a prior +shoot), no cookie banner leaked in, the CTA click did **not** dead-end at a +booking calendar or login form, and the **end** of the clip is clean (no desktop +or Finder window bleeding into the bottom of the capture). If any of that is +wrong, reshoot with `--story scroll` or a better landmark before spending an +export. + +### 3. Treat + export + +``` +python3 /lib/treat.py [--music ID] [--quality 4k|hd] [--bg-gradient FROM_HEX,TO_HEX] +``` + +- Aligns beats to video (tail-anchored: the video can be **shorter** than the + event log — never trust event times blindly), trims dead time, aims three 3D + shots at the logged landmarks, paints the brand gradient, synthesizes the + cursor track from the bundled cursor assets, copies music, exports the mp4. +- **Set the brand background by eye** when the site is gradient-heavy or + light/pastel: `getComputedStyle` lies on those (e.g. a site that looks white + with lavender accents can sample as black). Look at the raw frames and pass + `--bg-gradient FROM_HEX,TO_HEX` (light sites → a soft light gradient; dark + sites → a deep tint of the brand hue). +- **Match export fps to the real capture rate.** `cap record --detach` can + engage late and the window capture stalls timestamps on static pixels, so the + true rate is often ~58, not 60. Check it and avoid the 58-vs-60 judder: + `ffprobe -v error -select_streams v:0 -show_entries stream=nb_frames,duration -of csv=p=0 /content/segments/segment-0/display.mp4`, + then pass `--quality hd` (60) only if the capture is really ~60, otherwise + leave fps at the source rate (omit `--quality`). + +### 4. Frame-QA the export, then regrade if needed + +Extract 4 beat frames from `/-demo.mp4` and look: each beat +readable, a typing shot aimed at the **text-entry point** (not the field +center), no window/desktop bleed, clean fps, music fades feel right. If a shot +reads too tight or the aim is off, adjust and re-export. The reference 3D poses +were tuned on full-desktop captures; a browser-window capture fills the card +more, so distances can read ~30% too tight — back the zoom off if so. + +### 5. Deliver + +Ship `/-demo.mp4`. + +## Hard editorial rules (encode these every time) + +- **12s max** total. The tail is shaved evenly if the cut runs long. +- **A camera cut MUST be a content cut.** Every 3D perspective change lands on a + different section/page — the shot boundary sits exactly on the clip cut or the + scroll into new content. Never cut the camera mid-idle. +- **Cut on action, trim dead time.** Cut on the click; resume on the loaded + page. Trim page-loads and loading-state frames — never let a blur-up or + spinner leak into a shot tail (frame-check the cut points). +- **Aim at content, not the container.** Shots are aimed at landmarks (hero, the + clicked CTA, the destination header), not copied pans. Blur focus rides the + aim. +- **One motion system.** All emphasis lives in the 3D shot (a dolly-in is the + click punch). Never stack 2D zoom segments on 3D shots — the two systems fight + and read as jarring. +- **Hands off.** Virtual input only; the user's real cursor/mouse is never + moved. The on-screen cursor is synthesized post-hoc from the logged glide path. + +## Options reference + +Orchestrator (`cap-demo [flags]`): + +- `--out DIR` — output directory (default `/cap-demo/`). +- `--slug NAME` — project slug (default: the URL host, dashed). +- `--music ID` — music track id (see below). Default: chosen by brand darkness + (dark → `lofi-cinematic-pulsebox`, light → `sunday-mood-lofi-cafe-upbeat-bluelike`). +- `--quality 4k|hd` — `4k` = 3840x2160 / 60fps / maximum / filesize-optimized; + `hd` = 60fps / maximum. Omit for the source-rate default. +- `--story click|scroll` — force the storyboard (default: auto). + +`lib/treat.py` extra flags (when running the stages by hand): + +- `--bg-gradient FROM_HEX,TO_HEX` — override the brand gradient by eye + (e.g. `--bg-gradient E4DCF8,C6BAEA` for a light lavender, or `261A40,0C0914` + for a deep purple). + +**Bundled music** (a premium/dark/light/moody spread): +`lofi-cinematic-pulsebox` (moody/premium), `lofi-hip-hop-leberch` (dark), +`sunday-mood-lofi-cafe-upbeat-bluelike` (light/upbeat), `lofi-smooth-pulsebox` +(light/smooth). Music resolves only from the skill's bundled +`assets/music/.mp3`; an unknown id errors. + +**Binary resolution** (both scripts): Cap binary = env `CAP_BIN`, else `cap` on +PATH, else a clear error. Chromium = newest cached +`~/Library/Caches/ms-playwright/chromium-*` (macOS), else the pinned +`chromium-1228` build. + +## For dev tools / typing showcases (closeup variant) + +For dev tools with a live terminal or code on screen, a bespoke storyboard beats +the generic click-through. Scout for `[class*=terminal]` / `pre` / `code`, +scroll each to center, dwell ~2.7s (live terminals animate = free motion). The +**closeup shot recipe**: a gentle tilt (tiltX 10-13, tiltY 9-13, rotateX -4) so +text stays legible, a tight slow push-in zoom (0.72-0.82), and blur focus locked +ON the element (focusX = the element's fraction, small focusSize ~0.55). Keep it +a single continuous clip (no editorial cut) so scrolls stay smooth — cuts are +3D-only. Aim typing shots at the **text-entry region**, not the field center. +Rebuild a bespoke shoot from `lib/scout-shoot.mjs` as the base when a site +deserves it (edit a fresh copy — do not sed/splice shot arrays, overlapping +offsets corrupt the file). diff --git a/apps/cli/skill/cap-demo/assets/cursor_0.png b/apps/cli/skill/cap-demo/assets/cursor_0.png new file mode 100644 index 00000000000..379db242e6b Binary files /dev/null and b/apps/cli/skill/cap-demo/assets/cursor_0.png differ diff --git a/apps/cli/skill/cap-demo/assets/cursor_2.png b/apps/cli/skill/cap-demo/assets/cursor_2.png new file mode 100644 index 00000000000..1d9a4a98936 Binary files /dev/null and b/apps/cli/skill/cap-demo/assets/cursor_2.png differ diff --git a/apps/cli/skill/cap-demo/assets/cursors.json b/apps/cli/skill/cap-demo/assets/cursors.json new file mode 100644 index 00000000000..17989c59cf3 --- /dev/null +++ b/apps/cli/skill/cap-demo/assets/cursors.json @@ -0,0 +1,12 @@ +{ + "0": { + "imagePath": "content/cursors/cursor_0.png", + "hotspot": { "x": 0.17857142857142858, "y": 0.125 }, + "shape": "MacOS|TahoeArrow" + }, + "2": { + "imagePath": "content/cursors/cursor_2.png", + "hotspot": { "x": 0.40625, "y": 0.25 }, + "shape": "MacOS|TahoePointingHand" + } +} diff --git a/apps/cli/skill/cap-demo/assets/music/lofi-cinematic-pulsebox.mp3 b/apps/cli/skill/cap-demo/assets/music/lofi-cinematic-pulsebox.mp3 new file mode 100644 index 00000000000..8a178d61cee Binary files /dev/null and b/apps/cli/skill/cap-demo/assets/music/lofi-cinematic-pulsebox.mp3 differ diff --git a/apps/cli/skill/cap-demo/assets/music/lofi-hip-hop-leberch.mp3 b/apps/cli/skill/cap-demo/assets/music/lofi-hip-hop-leberch.mp3 new file mode 100644 index 00000000000..28baaa633cb Binary files /dev/null and b/apps/cli/skill/cap-demo/assets/music/lofi-hip-hop-leberch.mp3 differ diff --git a/apps/cli/skill/cap-demo/assets/music/lofi-smooth-pulsebox.mp3 b/apps/cli/skill/cap-demo/assets/music/lofi-smooth-pulsebox.mp3 new file mode 100644 index 00000000000..0614d15731e Binary files /dev/null and b/apps/cli/skill/cap-demo/assets/music/lofi-smooth-pulsebox.mp3 differ diff --git a/apps/cli/skill/cap-demo/assets/music/sunday-mood-lofi-cafe-upbeat-bluelike.mp3 b/apps/cli/skill/cap-demo/assets/music/sunday-mood-lofi-cafe-upbeat-bluelike.mp3 new file mode 100644 index 00000000000..8cbf35c1da0 Binary files /dev/null and b/apps/cli/skill/cap-demo/assets/music/sunday-mood-lofi-cafe-upbeat-bluelike.mp3 differ diff --git a/apps/cli/skill/cap-demo/cap-demo.mjs b/apps/cli/skill/cap-demo/cap-demo.mjs new file mode 100644 index 00000000000..7ce5d1a0d46 --- /dev/null +++ b/apps/cli/skill/cap-demo/cap-demo.mjs @@ -0,0 +1,171 @@ +#!/usr/bin/env node +// cap-demo orchestrator: URL in, cinematic 3D product-demo video out. +// Chains SCOUT + SHOOT (lib/scout-shoot.mjs) -> TREAT + EXPORT (lib/treat.py). +// +// Usage: cap-demo [--out DIR] [--slug NAME] [--music ID] [--quality 4k|hd] [--story click|scroll] +import { spawn } from "node:child_process"; +import { existsSync, mkdirSync } from "node:fs"; +import os from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const LIB = join(__dirname, "lib"); + +// --- CLI ---------------------------------------------------------------------- +const argv = process.argv.slice(2); +const opts = { out: null, slug: null, music: null, quality: null, story: null }; +const positionals = []; +for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + const take = () => argv[++i]; + if (a === "--out" || a === "-o") opts.out = take(); + else if (a.startsWith("--out=")) opts.out = a.slice(6); + else if (a === "--slug") opts.slug = take(); + else if (a.startsWith("--slug=")) opts.slug = a.slice(7); + else if (a === "--music") opts.music = take(); + else if (a.startsWith("--music=")) opts.music = a.slice(8); + else if (a === "--quality") opts.quality = take(); + else if (a.startsWith("--quality=")) opts.quality = a.slice(10); + else if (a === "--story") opts.story = take(); + else if (a.startsWith("--story=")) opts.story = a.slice(8); + else if (a === "-h" || a === "--help") { + usage(); + process.exit(0); + } else positionals.push(a); +} + +function usage() { + console.log( + "Usage: cap-demo [--out DIR] [--slug NAME] [--music ID] [--quality 4k|hd] [--story click|scroll]", + ); +} + +const url = positionals[0]; +if (!url) { + usage(); + fail("missing "); +} +if (opts.quality && opts.quality !== "4k" && opts.quality !== "hd") + fail("--quality must be 4k or hd"); +if (opts.story && opts.story !== "click" && opts.story !== "scroll") + fail("--story must be click or scroll"); + +function fail(msg) { + console.error(`cap-demo: ${msg}`); + process.exit(1); +} + +function slugFromUrl(u) { + let host; + try { + host = new URL(/^https?:\/\//i.test(u) ? u : `https://${u}`).host; + } catch { + host = u; + } + return ( + host + .replace(/^www\./i, "") + .toLowerCase() + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-+|-+$/g, "") || "demo" + ); +} + +const slug = opts.slug || slugFromUrl(url); +const outDir = opts.out || join(os.tmpdir(), "cap-demo", slug); +mkdirSync(outDir, { recursive: true }); + +// playwright-core is vendored into the skill; give a clear hint if missing. +if (!existsSync(join(__dirname, "node_modules", "playwright-core"))) { + fail( + `playwright-core not installed. Run: (cd "${__dirname}" && npm install)`, + ); +} + +// --- run a child, streaming stdout to the user and optionally capturing it ----- +function run(cmd, cmdArgs, { capture = false } = {}) { + return new Promise((resolve, reject) => { + const child = spawn(cmd, cmdArgs, { stdio: ["ignore", "pipe", "inherit"] }); + let buf = ""; + child.stdout.on("data", (chunk) => { + process.stdout.write(chunk); + if (capture) buf += chunk.toString(); + }); + child.on("error", reject); + child.on("close", (code) => + code === 0 + ? resolve(buf) + : reject(new Error(`${cmd} exited with code ${code}`)), + ); + }); +} + +// Parse the scout's final machine-readable JSON line (last line with a `scout` key). +function parseScoutResult(out) { + const lines = out + .split("\n") + .map((l) => l.trim()) + .filter(Boolean); + for (let i = lines.length - 1; i >= 0; i--) { + if (!lines[i].startsWith("{")) continue; + try { + const o = JSON.parse(lines[i]); + if (o && o.scout) return o; + } catch {} + } + return null; +} + +// Dark brand -> moody cinematic; light brand -> upbeat lofi. +function defaultMusic(scoutResult) { + const bg = scoutResult?.scout?.pageBg; + const light = + Array.isArray(bg) && bg.reduce((a, b) => a + b, 0) / bg.length > 150; + return light + ? "sunday-mood-lofi-cafe-upbeat-bluelike" + : "lofi-cinematic-pulsebox"; +} + +// --- pipeline ----------------------------------------------------------------- +(async () => { + console.log(`cap-demo: ${url} -> ${outDir}`); + console.log( + `[1/2] scout + shoot (slug=${slug}${opts.story ? `, story=${opts.story}` : ""})`, + ); + + const shootArgs = [join(LIB, "scout-shoot.mjs"), url, outDir, slug]; + if (opts.story) shootArgs.push("--story", opts.story); + + let scoutOut; + try { + scoutOut = await run(process.execPath, shootArgs, { capture: true }); + } catch (e) { + fail(`scout/shoot failed: ${e.message}`); + } + + const scoutResult = parseScoutResult(scoutOut); + const music = opts.music || defaultMusic(scoutResult); + const decided = scoutResult + ? `story=${scoutResult.story} cta=${scoutResult.scout.ctaText ?? "none"}` + : "story=?"; + + console.log( + `[2/2] treat + export (music=${music}${opts.quality ? `, quality=${opts.quality}` : ""})`, + ); + const treatArgs = [join(LIB, "treat.py"), outDir, slug, "--music", music]; + if (opts.quality) treatArgs.push("--quality", opts.quality); + + try { + await run("python3", treatArgs, { capture: false }); + } catch (e) { + fail(`treat/export failed: ${e.message}`); + } + + const mp4 = join(outDir, `${slug}-demo.mp4`); + console.log(""); + console.log(`Done: ${mp4}`); + console.log( + ` ${decided}, music=${music}${opts.quality ? `, ${opts.quality}` : ""}`, + ); +})(); diff --git a/apps/cli/skill/cap-demo/lib/scout-shoot.mjs b/apps/cli/skill/cap-demo/lib/scout-shoot.mjs new file mode 100644 index 00000000000..66d4b104b76 --- /dev/null +++ b/apps/cli/skill/cap-demo/lib/scout-shoot.mjs @@ -0,0 +1,419 @@ +// cap-demo SCOUT + SHOOT (adapted from director2.mjs; logic preserved verbatim, +// only I/O, path/binary resolution, CLI args and a final result line changed). +// +// Fully virtual input: CDP mouse/keyboard only (the user's real cursor is never +// touched); the cursor you see in the final video is synthesized from the logged +// glide path by treat.py. A near-invisible shimmer dot keeps the window capture +// emitting frames so the video clock never freezes on static content. +// +// Usage: node scout-shoot.mjs [--story click|scroll] + +import { execFileSync, execSync } from "node:child_process"; +import { existsSync, mkdirSync, readdirSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { chromium } from "playwright-core"; + +// --- CLI ---------------------------------------------------------------------- +const argv = process.argv.slice(2); +const positionals = []; +let forceStory; +for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + if (a === "--story") { + forceStory = argv[++i]; + } else if (a.startsWith("--story=")) { + forceStory = a.slice("--story=".length); + } else { + positionals.push(a); + } +} +const [url, outDir, slug] = positionals; +if (!url || !outDir || !slug) { + throw new Error( + "usage: node scout-shoot.mjs [--story click|scroll]", + ); +} +if (forceStory && forceStory !== "click" && forceStory !== "scroll") { + throw new Error(`--story must be 'click' or 'scroll' (got '${forceStory}')`); +} +mkdirSync(outDir, { recursive: true }); + +// --- binary resolution -------------------------------------------------------- +// Cap binary: env CAP_BIN, else `cap` on PATH, else a clear error. +function resolveCap() { + if (process.env.CAP_BIN) return process.env.CAP_BIN; + try { + const p = execSync("command -v cap", { encoding: "utf8" }).trim(); + if (p) return p; + } catch {} + throw new Error( + "cap CLI not found on PATH. Install Cap Desktop (https://cap.so) or set CAP_BIN.", + ); +} +const CAP = resolveCap(); + +// Chromium: newest cached ms-playwright chromium-*; fall back to the pinned build. +function resolveChromium() { + const HOME = process.env.HOME; + const suffix = + "chrome-mac-arm64/Google Chrome for Testing.app/Contents/MacOS/Google Chrome for Testing"; + const fallback = `${HOME}/Library/Caches/ms-playwright/chromium-1228/${suffix}`; + try { + const base = `${HOME}/Library/Caches/ms-playwright`; + const dirs = readdirSync(base) + .filter((d) => /^chromium-\d+$/.test(d)) + .map((d) => ({ d, n: parseInt(d.split("-")[1], 10) })) + .sort((a, b) => b.n - a.n); + for (const { d } of dirs) { + const exe = join(base, d, suffix); + if (existsSync(exe)) return exe; + } + } catch {} + return fallback; +} +const EXE = resolveChromium(); + +const PROJECT = join(outDir, `${slug}.cap`); +const TIMELINE = join(outDir, `${slug}.timeline.json`); + +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); +const jitter = (b, s) => b + (Math.random() - 0.5) * s; + +// Stale/closing test-chrome windows stay enumerable in `cap record windows` and +// the first-match can grab the WRONG one. Also close Finder/preview windows so +// they never composite into the bottom of the window capture. +try { + execSync(`pkill -f "Google Chrome for Testing" || true`); +} catch {} +try { + execSync( + `osascript -e 'tell application "Finder" to close every window' || true`, + ); +} catch {} +await sleep(800); + +const browser = await chromium.launch({ + executablePath: EXE, + headless: false, + ignoreDefaultArgs: ["--enable-automation"], + args: [ + "--window-size=1560,1000", + "--window-position=120,60", + "--disable-blink-features=AutomationControlled", + ], +}); +const page = await browser.newPage({ viewport: null }); +await page.goto(url, { waitUntil: "load", timeout: 60000 }); +await sleep(2600); + +// Dismiss cookie / consent banners so they never leak into a shot. +await page + .evaluate(() => { + const re = + /^(accept|accept all|allow all|agree|i agree|got it|ok|reject all|decline|dismiss|close)$/i; + for (const el of document.querySelectorAll("button, a, [role=button]")) { + const t = (el.textContent || "").trim(); + if (re.test(t)) { + try { + el.click(); + } catch {} + } + } + }) + .catch(() => {}); +await sleep(400); + +const geo = await page.evaluate(() => ({ + chromeTop: window.outerHeight - window.innerHeight, + iw: window.innerWidth, + ih: window.innerHeight, +})); +// Window-fraction mapping for aim targets and the synthetic cursor track. +const toFrac = (b) => ({ + x: (b.x + b.width / 2) / geo.iw, + y: (geo.chromeTop + b.y + b.height / 2) / (geo.ih + geo.chromeTop), +}); +const pointFrac = (p) => ({ + x: p.x / geo.iw, + y: (geo.chromeTop + p.y) / (geo.ih + geo.chromeTop), +}); + +// Shimmer dot: 3px, 2% opacity, orbits 2px in a corner. Invisible to the eye, +// visible to the encoder. Re-injected after navigations. +const injectShimmer = () => + page + .evaluate(() => { + if (document.getElementById("__cap_shimmer")) return; + const d = document.createElement("div"); + d.id = "__cap_shimmer"; + d.style.cssText = + "position:fixed;right:6px;bottom:6px;width:3px;height:3px;background:rgba(127,127,127,0.02);z-index:2147483647;pointer-events:none;"; + document.documentElement.appendChild(d); + let t = 0; + const tick = () => { + t += 0.25; + d.style.transform = `translate(${Math.sin(t) * 2}px, ${Math.cos(t) * 2}px)`; + requestAnimationFrame(tick); + }; + requestAnimationFrame(tick); + }) + .catch(() => {}); +await injectShimmer(); + +// --- Scout -------------------------------------------------------------------- +const scout = await page.evaluate(() => { + const vis = (el) => { + const r = el.getBoundingClientRect(); + const s = getComputedStyle(el); + return ( + r.width > 8 && + r.height > 8 && + s.visibility !== "hidden" && + s.display !== "none" && + r.y < innerHeight * 1.1 + ); + }; + const hero = + [...document.querySelectorAll("h1")].find(vis) ?? + [...document.querySelectorAll("h2")].find(vis); + const heroBox = hero ? hero.getBoundingClientRect() : null; + const sameHost = (href) => { + try { + return new URL(href, location.href).host === location.host; + } catch { + return false; + } + }; + // Content destinations tell the story; conversion CTAs dead-end at forms. + const ctaWords = + /how it works|features|product|explore|browse|see|watch|demo|learn|courses|membership|events|about|pricing|showcase|gallery|tools|docs/i; + const ctaBad = + /try|free|sign ?up|register|get started|start now|join|book|download|log ?in|subscribe|get a demo|book a demo|request a demo|contact|talk to/i; + const candidates = [...document.querySelectorAll("a, button")] + .filter(vis) + .map((el) => { + const r = el.getBoundingClientRect(); + const s = getComputedStyle(el); + const text = (el.textContent || "").trim(); + const href = el.tagName === "A" ? el.href : null; + const bg = s.backgroundColor; + const filled = bg && bg !== "rgba(0, 0, 0, 0)" && bg !== "transparent"; + let score = 0; + if (ctaWords.test(text)) score += 3; + if (filled) score += 2; + if (r.width > 110) score += 1; + if (r.y > 120 && r.y < innerHeight * 0.85) score += 2; + if (href && sameHost(href) && !href.includes("#")) score += 2; + if (href && new URL(href).pathname === location.pathname) score -= 3; + if (/login|log in|sign in/i.test(text)) score -= 4; + if (ctaBad.test(text)) score -= 5; + if ( + href && + /\/(blog|changelog|careers|jobs|news|press)(\/|$)/i.test( + new URL(href).pathname, + ) + ) + score -= 5; + if (/↗|↗️|external/i.test(text)) score -= 3; + return { + text: text.slice(0, 48), + href, + score, + box: { x: r.x, y: r.y, width: r.width, height: r.height }, + filled, + bg, + }; + }) + .filter((c) => c.text && c.score >= 5) + .sort((a, b) => b.score - a.score); + const parse = (c) => { + const m = c?.match(/rgba?\((\d+),\s*(\d+),\s*(\d+)/); + return m ? [+m[1], +m[2], +m[3]] : null; + }; + let pageBg = null; + for (const el of [ + document.body, + document.documentElement, + ...document.querySelectorAll("main, header, section"), + ]) { + const c = parse(getComputedStyle(el).backgroundColor); + if (c) { + pageBg = c; + break; + } + } + const filledCta = candidates.find((c) => c.filled); + return { + title: document.title, + heroBox, + candidates: candidates.slice(0, 5), + pageBg, + accent: filledCta ? parse(filledCta.bg) : null, + }; +}); +const cta = scout.candidates[0] ?? null; +const story = forceStory ?? (cta?.href ? "click" : "scroll"); +console.log( + `STORY=${story} cta=${cta?.text ?? "none"} accent=${scout.accent} bg=${scout.pageBg}`, +); + +// --- Virtual cursor ----------------------------------------------------------- +const events = []; +const cursorLog = { moves: [], clicks: [] }; +let t0 = 0; +const now = () => (Date.now() - t0) / 1000; +let vpos = { x: geo.iw * 0.55, y: geo.ih * 0.45 }; + +const logMove = () => { + const f = pointFrac(vpos); + cursorLog.moves.push({ t: now(), x: f.x, y: f.y }); +}; +const vmove = async (p) => { + vpos = p; + await page.mouse.move(p.x, p.y).catch(() => {}); + logMove(); +}; +async function glide(to, ms) { + const from = { ...vpos }; + const steps = Math.max(8, Math.round(ms / 16)); + for (let i = 1; i <= steps; i++) { + const t = i / steps, + e = t * t * (3 - 2 * t); + await vmove({ + x: from.x + (to.x - from.x) * e, + y: from.y + (to.y - from.y) * e, + }); + await sleep(ms / steps); + } +} +const vclick = async () => { + cursorLog.clicks.push({ t: now(), down: true }); + await page.mouse.down().catch(() => {}); + await sleep(70); + await page.mouse.up().catch(() => {}); + cursorLog.clicks.push({ t: now() - 0.001, down: false }); +}; +const center = (b) => ({ x: b.x + b.width / 2, y: b.y + b.height / 2 }); + +const pageTitle = await page.title(); +const windows = JSON.parse( + execFileSync(CAP, ["record", "windows", "--json"], { encoding: "utf8" }), +); +const chromeWins = windows.filter( + (w) => + /Chrome for Testing/i.test(w.ownerName ?? "") || + /Chrome for Testing/i.test(w.name ?? ""), +); +let win = chromeWins.find((w) => (w.name ?? "").trim() === pageTitle.trim()); +if (!win && chromeWins.length === 1) win = chromeWins[0]; +if (!win) + throw new Error( + `window not found (title="${pageTitle}", ${chromeWins.length} chrome windows: ${chromeWins.map((w) => w.name).join(" | ")})`, + ); + +execFileSync( + CAP, + [ + "record", + "start", + "--detach", + "--window", + String(win.id), + "--fps", + "60", + "--path", + PROJECT, + ], + { + encoding: "utf8", + }, +); +t0 = Date.now(); +const mark = (n, extra = {}) => { + events.push({ name: n, t: now(), ...extra }); + console.log(`[${now().toFixed(2)}s] ${n}`); +}; + +try { + logMove(); + if (scout.heroBox) mark("hero_frac", toFrac(scout.heroBox)); + await sleep(1300); + + if (story === "click" && cta) { + const c = center(cta.box); + mark("cta_frac", toFrac(cta.box)); + mark("drift_to_cta"); + await glide({ x: c.x + 16, y: c.y + 24 }, 700); + await glide(c, 380); + await sleep(420); + mark("click_cta"); + await vclick(); + await page.waitForLoadState("load", { timeout: 25000 }).catch(() => {}); + await injectShimmer(); + await sleep(1100); + mark("page_ready"); + const head2 = await page + .locator("h1, h2") + .first() + .boundingBox() + .catch(() => null); + if (head2 && head2.y < geo.ih) mark("page2_frac", toFrac(head2)); + await glide({ x: geo.iw * 0.55, y: geo.ih * 0.5 }, 800); + await sleep(1100); + } else { + mark("drift_hero"); + await glide({ x: geo.iw * 0.5, y: geo.ih * 0.45 }, 900); + await sleep(500); + mark("scroll1_start"); + await page.evaluate( + (d) => window.scrollBy({ top: d, behavior: "smooth" }), + Math.round(geo.ih * 0.95), + ); + await glide({ x: vpos.x + 90, y: vpos.y + 120 }, 900); + await sleep(800); + mark("scroll1_settled"); + await sleep(1500); + } + + mark("scroll_start"); + await page.evaluate(() => window.scrollBy({ top: 700, behavior: "smooth" })); + await glide({ x: vpos.x + 110, y: vpos.y + 130 }, 900); + await sleep(800); + mark("scroll_settled"); + // Gentle idle drift for the settle; shimmer keeps the clock honest anyway. + const until = Date.now() + 2300; + while (Date.now() < until) { + await vmove({ x: vpos.x + jitter(0, 2.4), y: vpos.y + jitter(0, 2.4) }); + await sleep(150); + } + mark("end"); +} finally { + execFileSync(CAP, ["record", "stop", "--path", PROJECT], { + encoding: "utf8", + }); + writeFileSync( + TIMELINE, + JSON.stringify( + { story, scout: { ...scout, candidates: undefined }, events, cursorLog }, + null, + 1, + ), + ); + await browser.close(); +} + +// Final machine-readable line: what the scout decided (orchestrator/agent reads this). +console.log( + JSON.stringify({ + slug, + story, + scout: { + title: scout.title, + hero: scout.heroBox, + accent: scout.accent, + pageBg: scout.pageBg, + ctaText: cta?.text ?? null, + }, + }), +); +console.log("DONE"); diff --git a/apps/cli/skill/cap-demo/lib/treat.py b/apps/cli/skill/cap-demo/lib/treat.py new file mode 100644 index 00000000000..c7e667cf917 --- /dev/null +++ b/apps/cli/skill/cap-demo/lib/treat.py @@ -0,0 +1,354 @@ +#!/usr/bin/env python3 +"""cap-demo TREAT + EXPORT (adapted from treat.py; all shot math, brand-gradient +and editorial logic preserved verbatim, only I/O, binary/asset/music resolution +and CLI args changed). + +Aligns beats to video, trims, aims 3D shots at landmarks, paints a brand +background, synthesizes the cursor track from the bundled cursor assets, copies +music, writes project-config.json, and exports via the Cap CLI. + +Usage: python3 treat.py [--music ID] [--quality 4k|hd] [--bg-gradient FROM_HEX,TO_HEX] +""" +import argparse +import colorsys +import json +import math +import os +import shutil +import subprocess +import sys + +SKILL_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +ASSETS = os.path.join(SKILL_DIR, "assets") + + +def resolve_cap(): + """Cap binary: env CAP_BIN, else `cap` on PATH, else a clear error.""" + if os.environ.get("CAP_BIN"): + return os.environ["CAP_BIN"] + p = shutil.which("cap") + if p: + return p + raise SystemExit( + "cap CLI not found on PATH. Install Cap Desktop (https://cap.so) or set CAP_BIN." + ) + + +def resolve_music(music_id): + bundled = os.path.join(ASSETS, "music", f"{music_id}.mp3") + if os.path.exists(bundled): + return bundled + raise SystemExit( + f"music not found: {music_id} (looked in {os.path.join(ASSETS, 'music')})" + ) + + +def parse_bg_override(spec): + parts = [p.strip() for p in spec.split(",")] + if len(parts) != 2: + raise SystemExit("--bg-gradient must be FROM_HEX,TO_HEX (e.g. 261A40,0C0914)") + + def hexrgb(h): + h = h.lstrip("#") + if len(h) != 6: + raise SystemExit(f"invalid hex color: {h}") + return [int(h[i:i + 2], 16) for i in (0, 2, 4)] + + return {"type": "gradient", "from": hexrgb(parts[0]), "to": hexrgb(parts[1]), "angle": 135} + + +ap = argparse.ArgumentParser() +ap.add_argument("outDir") +ap.add_argument("slug") +ap.add_argument("--music", default="lofi-cinematic-pulsebox") +ap.add_argument("--quality", choices=["4k", "hd"], default=None) +ap.add_argument("--bg-gradient", dest="bg_gradient", default=None) +args = ap.parse_args() + +outDir = args.outDir +slug = args.slug +music = args.music +CAP = resolve_cap() + +proj = os.path.join(outDir, f"{slug}.cap") + +with open(os.path.join(outDir, f"{slug}.timeline.json")) as f: + tl = json.load(f) +events = {e["name"]: e for e in tl["events"]} +story = tl["story"] +scout = tl["scout"] + +video = f"{proj}/content/segments/segment-0/display.mp4" +probe = subprocess.run( + ["ffprobe", "-v", "quiet", "-show_entries", "format=duration", "-of", "csv=p=0", video], + capture_output=True, text=True).stdout.strip() +vdur = float(probe) +dims = subprocess.run( + ["ffprobe", "-v", "quiet", "-show_entries", "stream=width,height", "-of", "csv=p=0", video], + capture_output=True, text=True).stdout.strip() +W, H = [int(x) for x in dims.split(",")[:2]] +aspect = W / H +hx, hy = (1.0, 1.0 / aspect) if aspect >= 1 else (aspect, 1.0) + +# Tail-anchored alignment: micro-drift keeps frames flowing to the end. +offset = events["end"]["t"] - vdur +def v(name): + return max(0.0, events[name]["t"] - offset) + +# --- camera math --------------------------------------------------------------- +def rot(axis, deg): + t = math.radians(deg); c, s = math.cos(t), math.sin(t) + if axis == "x": return [[1, 0, 0], [0, c, -s], [0, s, c]] + if axis == "y": return [[c, 0, s], [0, 1, 0], [-s, 0, c]] + return [[c, -s, 0], [s, c, 0], [0, 0, 1]] + +def mul(a, b): + return [[sum(a[i][k] * b[k][j] for k in range(3)) for j in range(3)] for i in range(3)] + +def aim_pan(p, frac): + R = mul(mul(rot("y", p["tiltY"]), mul(rot("x", p["tiltX"]), rot("z", p["roll"]))), + mul(rot("y", p["rotateY"]), rot("x", p["rotateX"]))) + u = 0.5 + (frac[0] - 0.5) * 0.9 + vv = 0.5 + (frac[1] - 0.5) * 0.9 + X, Y = (2 * u - 1) * hx, (1 - 2 * vv) * hy + return (-(R[0][0] * X + R[0][1] * Y), -(R[1][0] * X + R[1][1] * Y)) + +def props(**kw): + base = {"tiltX": 0.0, "tiltY": 0.0, "roll": 0.0, "rotateX": 0.0, "rotateY": 0.0, + "fov": 45.0, "zoom": 2.0, "panX": 0.0, "panY": 0.0} + base.update(kw); return base + +def clamp_cut(a, b, ceiling, floor=0.0, min_dur=0.4): + """Force a (start, end) cut forward within [floor, ceiling] by at least min_dur.""" + a = max(floor, min(a, ceiling - min_dur)) + b = max(a + min_dur, min(b, ceiling)) + return (a, b) + +def kf(t, val, first): + k = {"time": t, "value": val} + k["outEasing" if first else "inEasing"] = [0.0, 0.0] if first else [1.0, 1.0] + return k + +def tracks_between(p0, p1, length): + return {key: [kf(0.0, p0[key], True), kf(length, p1[key], False)] + for key in p0 if abs(p0[key] - p1[key]) > 1e-6} + +def blur(st, fo, sz): + return {"mode": "radial", "strength": st, "falloff": fo, "focusX": 0.5, "focusY": 0.46, + "focusSize": sz, "angle": 0.0, "dirPosition": 0.5, "bokeh": True} + +def shot(a, b, p0, p1, bl): + return {"start": a, "end": b, "enabled": True, "properties": p0, "blur": bl, + "tracks": tracks_between(p0, p1, b - a), "transitionIn": 0.0, "transitionOut": 0.0} + +def frac_of(name, default): + e = events.get(name) + return (e["x"], e["y"]) if e and "x" in e else default + +# --- brand background ---------------------------------------------------------- +def brand_gradient(): + accent = scout.get("accent") + bg = scout.get("pageBg") + def tune(rgb, lo, hi, sat_cap=0.75): + r, g, b = [c / 255 for c in rgb] + h, l, s = colorsys.rgb_to_hls(r, g, b) + l = min(max(l, lo), hi) + s = min(s, sat_cap) + r, g, b = colorsys.hls_to_rgb(h, l, s) + return [int(r * 255), int(g * 255), int(b * 255)] + if accent and bg: + return {"type": "gradient", "from": tune(accent, 0.18, 0.42), "to": tune(bg, 0.10, 0.30), "angle": 135} + if accent: + base = tune(accent, 0.18, 0.42) + return {"type": "gradient", "from": base, "to": tune(accent, 0.08, 0.2), "angle": 135} + if bg and (sum(bg) / 3) > 150: # light brand -> soft light gradient + return {"type": "gradient", "from": tune(bg, 0.86, 0.94, 0.25), "to": tune(bg, 0.72, 0.82, 0.35), "angle": 135} + return {"type": "gradient", "from": [38, 40, 58], "to": [16, 17, 26], "angle": 135} + +bg_source = parse_bg_override(args.bg_gradient) if args.bg_gradient else brand_gradient() + +# --- editorial cut ------------------------------------------------------------- +HERO = frac_of("hero_frac", (0.5, 0.4)) + +if story == "click": + CTA = frac_of("cta_frac", (0.5, 0.6)) + PAGE2 = frac_of("page2_frac", (0.42, 0.45)) + click = v("click_cta") + ready = v("page_ready") + scroll = v("scroll_start") + cutA = (0.25, min(click + 0.4, ready - 0.1)) + cutB = (ready + 0.5, vdur - 0.05) +else: + CTA = HERO + PAGE2 = (0.5, 0.45) + s1 = v("scroll1_start") + scroll = v("scroll_start") + cutA = (0.25, s1 + 0.9) # hero through the first scroll's start + cutB = (s1 + 1.1, vdur - 0.05) # arrive in section 2 mid-scroll + +# A late-starting or stalled capture can align marker events near zero (see +# v()'s clamp above), which can push a cut's computed end before its start. +# Force both cuts forward within the actual recording bounds. +cutA = clamp_cut(*cutA, vdur) +cutB = clamp_cut(*cutB, vdur) + +clip_cut = cutA[1] - cutA[0] +DUR = clip_cut + (cutB[1] - cutB[0]) +# 12s cap: shave the tail evenly if long. +if DUR > 12.0: + trim = DUR - 12.0 + cutB = (cutB[0], cutB[1] - trim) + DUR = 12.0 + +scroll_out = clip_cut + (scroll - cutB[0]) +avail = DUR - clip_cut +if avail >= 3.4: + b2 = min(max(scroll_out + 0.45, clip_cut + 1.2), DUR - 2.2) +else: + # A short or stalled capture can leave too little room for the 1.2s/2.2s + # pacing minimums above (their sum, 3.4s); the fixed DUR - 2.2 ceiling can + # then fall below clip_cut and reverse the second shot's boundaries. + # Split whatever duration is available instead so both shots stay forward. + margin = avail * 0.25 + b2 = min(max(scroll_out + 0.45, clip_cut + margin), DUR - margin) + +s1a = props(tiltX=26.0, tiltY=-22.0, roll=1.0, zoom=1.05) +s1a["panX"], s1a["panY"] = aim_pan(s1a, HERO) +s1b = props(tiltX=26.0, tiltY=-26.0, roll=1.0, zoom=0.86) +s1b["panX"], s1b["panY"] = aim_pan(s1b, CTA) +s2a = props(tiltX=24.8, tiltY=17.04, rotateX=-40.0, rotateY=18.0, fov=60.0, zoom=0.8) +s2a["panX"], s2a["panY"] = aim_pan(s2a, PAGE2) +s2b = props(tiltX=34.19, tiltY=15.28, rotateX=-40.0, rotateY=9.0, fov=60.0, zoom=0.74) +s2b["panX"], s2b["panY"] = aim_pan(s2b, PAGE2) +s3a = props(rotateX=-14.0, zoom=0.95) +s3a["panX"], s3a["panY"] = aim_pan(s3a, (0.5, 0.45)) +s3b = props(rotateX=-14.0, zoom=1.6) + + +# --- synthetic cursor track ---------------------------------------------------- +# The shoot used CDP input only; the on-screen cursor in the final render is +# drawn from this data. Overwrites any stray real-cursor data Cap recorded. +# Cursor assets are bundled with the skill (assets/cursor_0.png, cursor_2.png, +# assets/cursors.json). +if "cursorLog" in tl: + with open(os.path.join(ASSETS, "cursors.json")) as f: + cursors_meta = json.load(f) + with open(f"{proj}/recording-meta.json") as f: + meta = json.load(f) + start_time = meta["segments"][0].get("display", {}).get("start_time", 0.0) or 0.0 + + os.makedirs(f"{proj}/content/cursors", exist_ok=True) + for cid in ("0", "2"): + src = os.path.join(ASSETS, f"cursor_{cid}.png") + shutil.copy(src, f"{proj}/content/cursors/cursor_{cid}.png") + meta["cursors"] = {cid: cursors_meta[cid] for cid in ("0", "2")} + meta["segments"][0]["cursor"] = "content/segments/segment-0/cursor.json" + with open(f"{proj}/recording-meta.json", "w") as f: + json.dump(meta, f, indent=2) + + def ms(t): + return max(0.0, (t - offset + start_time)) * 1000.0 + + moves = [ + {"active_modifiers": [], "cursor_id": "0", "time_ms": ms(m["t"]), "x": m["x"], "y": m["y"]} + for m in tl["cursorLog"]["moves"] + ] + clicks = [ + {"active_modifiers": [], "cursor_num": 1, "cursor_id": "0", "time_ms": ms(c["t"]), "down": c["down"]} + for c in tl["cursorLog"]["clicks"] + ] + with open(f"{proj}/content/segments/segment-0/cursor.json", "w") as f: + json.dump({"moves": moves, "clicks": clicks}, f) + print(f"cursor track synthesized: {len(moves)} moves, {len(clicks)} clicks") + +os.makedirs(f"{proj}/assets/audio", exist_ok=True) +music_src = resolve_music(music) +dest = f"{proj}/assets/audio/library-{music}.mp3" +shutil.copy(music_src, dest) +mdur = float(subprocess.run( + ["ffprobe", "-v", "quiet", "-show_entries", "format=duration", "-of", "csv=p=0", dest], + capture_output=True, text=True).stdout.strip()) + +config = { + "aspectRatio": None, + "background": { + "source": bg_source, + "blur": 0.0, "padding": 10.0, "rounding": 7.5, "roundingType": "squircle", "inset": 0, + "crop": None, "displayPosition": None, "shadow": 73.6, + "advancedShadow": {"size": 14.4, "opacity": 68.1, "blur": 3.8}, "border": None, "frame": None, + }, + "timeline": { + "segments": [ + {"recordingSegment": 0, "timescale": 1.0, "start": cutA[0], "end": cutA[1], "name": None}, + {"recordingSegment": 0, "timescale": 1.0, "start": cutB[0], "end": cutB[1], "name": None}, + ], + "zoomSegments": [], + "camera3dSegments": [ + shot(0.0, clip_cut, s1a, s1b, blur(20.0, 0.7, 0.32)), + shot(clip_cut, b2, s2a, s2b, blur(18.0, 0.72, 0.5)), + shot(b2, DUR, s3a, s3b, blur(19.0, 0.67, 0.42)), + ], + "audioSegments": [{ + "start": 0.0, "end": DUR, "track": 0, + "path": f"assets/audio/library-{music}.mp3", + "name": music, "enabled": True, "trimStart": 0.0, + "volumeDb": 0.0, "fadeIn": 0.5, "fadeOut": 1.5, "duration": mdur, + }], + }, +} +with open(f"{proj}/project-config.json", "w") as f: + json.dump(config, f, indent=1) +print(f"{slug}: story={story} dur={DUR:.2f} cut={clip_cut:.2f} b2={b2:.2f} offset={offset:.2f} grad={config['background']['source']}") + +out_mp4 = os.path.join(outDir, f"{slug}-demo.mp4") + + +def source_fps(path): + """Dominant capture rate (mode of instantaneous fps). Window capture is VFR: + a few long stall gaps (page nav, heavy animation) drag the mean down, so the + mean is the wrong number to export at. The mode is the real rate the smooth + sections were captured at; matching it avoids resample judder.""" + try: + import re as _re + from collections import Counter + pts = subprocess.run( + ["ffprobe", "-v", "quiet", "-select_streams", "v", + "-show_entries", "frame=pts_time", "-of", "csv=p=0", path], + capture_output=True, text=True).stdout.replace(",", " ").split() + times = [float(x) for x in pts if x.strip()] + rates = Counter() + for a, b in zip(times, times[1:]): + d = b - a + if d > 1e-6: + rates[round(1.0 / d)] += 1 + if rates: + return max(2, min(60, rates.most_common(1)[0][0])) + except Exception: + pass + try: + import re as _re + meta = subprocess.run( + ["ffprobe", "-v", "quiet", "-count_frames", "-select_streams", "v", + "-show_entries", "stream=nb_read_frames", "-show_entries", "format=duration", + "-of", "csv=p=0", path], capture_output=True, text=True).stdout + nums = [x for x in _re.split(r"[,\n]", meta) if x.strip()] + return max(2, min(60, round(int(nums[0]) / float(nums[1])))) + except Exception: + return 60 + + +fps = str(source_fps(video)) +cmd = [CAP, "export", proj, "-o", out_mp4, "--fps", fps] +if args.quality == "4k": + cmd += ["--resolution", "3840x2160", "--quality", "maximum", "--optimize-filesize"] +elif args.quality == "hd": + cmd += ["--quality", "maximum"] +else: + cmd += ["--quality", "maximum", "--optimize-filesize"] +res = subprocess.run(cmd, capture_output=True, text=True) +if res.returncode != 0: + sys.stderr.write(res.stdout or "") + sys.stderr.write(res.stderr or "") + raise SystemExit(f"cap export failed (exit {res.returncode})") +print(f"exported {slug}-demo.mp4") +print(out_mp4) diff --git a/apps/cli/skill/cap-demo/package-lock.json b/apps/cli/skill/cap-demo/package-lock.json new file mode 100644 index 00000000000..d7c425d8949 --- /dev/null +++ b/apps/cli/skill/cap-demo/package-lock.json @@ -0,0 +1,30 @@ +{ + "name": "cap-demo", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "cap-demo", + "version": "1.0.0", + "dependencies": { + "playwright-core": "^1.62.1" + }, + "bin": { + "cap-demo": "cap-demo.mjs" + } + }, + "node_modules/playwright-core": { + "version": "1.62.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.62.1.tgz", + "integrity": "sha512-wPYSwEBJY9GHraISXqyqtx0na0LpO3XEX7jNDhntbex7tzUS7kLnZsOlFruFJB4Hi/rhDMjXGqHewDZ68nYZVw==", + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=20" + } + } + } +} diff --git a/apps/cli/skill/cap-demo/package.json b/apps/cli/skill/cap-demo/package.json new file mode 100644 index 00000000000..95915db47c6 --- /dev/null +++ b/apps/cli/skill/cap-demo/package.json @@ -0,0 +1,12 @@ +{ + "name": "cap-demo", + "version": "1.0.0", + "type": "module", + "description": "Generate a cinematic 3D product-demo video from any URL with Cap.", + "bin": { + "cap-demo": "./cap-demo.mjs" + }, + "dependencies": { + "playwright-core": "^1.62.1" + } +} diff --git a/apps/cli/skill/cap/SKILL.md b/apps/cli/skill/cap/SKILL.md index ad552ebc12a..68f6f60c095 100644 --- a/apps/cli/skill/cap/SKILL.md +++ b/apps/cli/skill/cap/SKILL.md @@ -115,3 +115,10 @@ cap upload out.mp4 --json ``` A stopped recording is complete only when `recordingMetaExists` is `true`. + +## Cinematic demo videos + +To generate a short, cinematic 3D product-demo video from a URL, use the installed +`cap-demo` skill (installed alongside this one). It scouts the page, records it with +virtual input, and treats it with Cap's 3D camera, a brand-matched background, and +music. Currently macOS-only. diff --git a/apps/cli/src/agents.rs b/apps/cli/src/agents.rs index 55d5a8ba6c2..739d9854a26 100644 --- a/apps/cli/src/agents.rs +++ b/apps/cli/src/agents.rs @@ -12,6 +12,23 @@ use crate::{OutputFormat, atomic, resolve_format, write_json}; const BUNDLED_SKILL: &str = include_str!("../skill/cap/SKILL.md"); +// The cap-demo companion skill ships as a directory tree (scripts, cursor and +// music assets) embedded into the binary alongside the main cap skill. +static CAP_DEMO_SKILL: include_dir::Dir<'static> = + include_dir::include_dir!("$CARGO_MANIFEST_DIR/skill/cap-demo"); + +// Flatten every embedded file (recursively) so each becomes its own install +// change. include_dir gives direct files via `files()` and nested dirs via +// `dirs()`; file paths are already relative to the embedded root. +fn collect_demo_files<'a>(dir: &'a include_dir::Dir<'a>, out: &mut Vec<&'a include_dir::File<'a>>) { + for file in dir.files() { + out.push(file); + } + for sub in dir.dirs() { + collect_demo_files(sub, out); + } +} + #[derive(Args)] pub struct AgentsArgs { #[command(subcommand)] @@ -97,6 +114,19 @@ fn skill_path(target: AgentTarget) -> Result { }) } +// The cap-demo skill installs as a directory sibling of the cap skill. +fn cap_demo_skill_dir(target: AgentTarget) -> Result { + let home = home_dir()?; + Ok(match target { + AgentTarget::Codex => std::env::var_os("CODEX_HOME") + .map(PathBuf::from) + .unwrap_or_else(|| home.join(".codex")) + .join("skills/cap-demo"), + AgentTarget::Claude => home.join(".claude/skills/cap-demo"), + AgentTarget::Cursor => home.join(".cursor/skills/cap-demo"), + }) +} + fn mcp_path(target: AgentTarget) -> Result { let home = home_dir()?; Ok(match target { @@ -262,6 +292,37 @@ impl InstallArgs { expected, content, }); + + // Ship the cap-demo companion skill as one change per embedded file. + let demo_dir = cap_demo_skill_dir(self.target)?; + let mut demo_files = Vec::new(); + collect_demo_files(&CAP_DEMO_SKILL, &mut demo_files); + demo_files.sort_by(|a, b| a.path().cmp(b.path())); + for file in demo_files { + let relative = file.path(); + let dest = demo_dir.join(relative); + let expected = read_optional(&dest)?; + let bytes = file.contents(); + let action = if expected.as_deref() == Some(bytes) { + "unchanged" + } else if expected.is_some() { + "replace" + } else { + "create" + }; + let content = (action != "unchanged").then(|| bytes.to_vec()); + changes.push(PlannedChange { + component: "skill", + path: dest, + action, + value: json!({ + "name": "cap-demo", + "file": relative.to_string_lossy(), + }), + expected, + content, + }); + } } if self.includes_mcp() { let path = mcp_path(self.target)?; diff --git a/apps/cli/src/selftest/playback.rs b/apps/cli/src/selftest/playback.rs index 0458443bcb9..83845d08aa7 100644 --- a/apps/cli/src/selftest/playback.rs +++ b/apps/cli/src/selftest/playback.rs @@ -912,6 +912,7 @@ mod fixture { caption_segments: Vec::new(), keyboard_segments: Vec::new(), audio_segments: Vec::new(), + camera3d_segments: Vec::new(), }), clips: vec![ClipConfiguration { index: 0, diff --git a/apps/desktop/src-tauri/examples/desktop-display-transport-benchmark.rs b/apps/desktop/src-tauri/examples/desktop-display-transport-benchmark.rs index cb0788461d8..35693bf8cb6 100644 --- a/apps/desktop/src-tauri/examples/desktop-display-transport-benchmark.rs +++ b/apps/desktop/src-tauri/examples/desktop-display-transport-benchmark.rs @@ -168,6 +168,7 @@ async fn load_recording( caption_segments: Vec::new(), keyboard_segments: Vec::new(), audio_segments: Vec::new(), + camera3d_segments: Vec::new(), }); } } diff --git a/apps/desktop/src-tauri/src/export.rs b/apps/desktop/src-tauri/src/export.rs index 45c61a2aef9..d0fbe410cb8 100644 --- a/apps/desktop/src-tauri/src/export.rs +++ b/apps/desktop/src-tauri/src/export.rs @@ -693,6 +693,7 @@ fn resolve_exporter_binary() -> Result { if let Some(dir) = exe.parent() { for candidate in adjacent_exporter_binary_candidates(dir) { if candidate.exists() { + warn_if_exporter_stale(&exe, &candidate); return Ok(candidate); } } @@ -702,6 +703,7 @@ fn resolve_exporter_binary() -> Result { for root in std::iter::once(cwd.as_path()).chain(cwd.ancestors()) { for candidate in exporter_binary_candidates(root) { if candidate.exists() { + warn_if_exporter_stale(&exe, &candidate); return Ok(candidate); } } @@ -714,6 +716,28 @@ fn resolve_exporter_binary() -> Result { )) } +/// Dev-loop trap: `tauri dev` rebuilds the app but not the exporter sidecar, +/// so exports can silently run renderer code from days earlier and disagree +/// with the preview. Debug builds log loudly when that is happening. +fn warn_if_exporter_stale(app_exe: &Path, exporter: &Path) { + if !cfg!(debug_assertions) { + return; + } + let mtime = |p: &Path| std::fs::metadata(p).and_then(|m| m.modified()).ok(); + if let (Some(app_time), Some(exporter_time)) = (mtime(app_exe), mtime(exporter)) + && let Ok(lag) = app_time.duration_since(exporter_time) + && lag.as_secs() > 60 + { + tracing::warn!( + exporter = %exporter.display(), + lag_secs = lag.as_secs(), + "Export worker binary is older than the app; exports may not match \ + the preview. Rebuild it (cargo build -p cap) and copy it over \ + target/debug/cap-exporter." + ); + } +} + fn exporter_binary_candidates(root: &Path) -> Vec { let mut candidates = Vec::new(); diff --git a/apps/desktop/src-tauri/src/import.rs b/apps/desktop/src-tauri/src/import.rs index 38c425d8cc2..94b4b89f722 100644 --- a/apps/desktop/src-tauri/src/import.rs +++ b/apps/desktop/src-tauri/src/import.rs @@ -379,6 +379,7 @@ fn ensure_project_timeline<'a>( caption_segments: Vec::new(), keyboard_segments: Vec::new(), audio_segments: Vec::new(), + camera3d_segments: Vec::new(), }); } diff --git a/apps/desktop/src-tauri/src/recording.rs b/apps/desktop/src-tauri/src/recording.rs index a12f05431d6..2d0ef1a004c 100644 --- a/apps/desktop/src-tauri/src/recording.rs +++ b/apps/desktop/src-tauri/src/recording.rs @@ -3874,6 +3874,7 @@ fn project_config_from_recording( caption_segments: Vec::new(), keyboard_segments: Vec::new(), audio_segments: Vec::new(), + camera3d_segments: Vec::new(), }); config diff --git a/apps/desktop/src/routes/editor/ConfigSidebar.tsx b/apps/desktop/src/routes/editor/ConfigSidebar.tsx index fce6f7a95eb..08cf05fc7dd 100644 --- a/apps/desktop/src/routes/editor/ConfigSidebar.tsx +++ b/apps/desktop/src/routes/editor/ConfigSidebar.tsx @@ -1,3 +1,4 @@ +import { Button } from "@cap/ui-solid"; import { NumberField } from "@kobalte/core"; import { Collapsible, @@ -31,9 +32,11 @@ import { createSignal, For, Index, + type JSX, lazy, on, onMount, + type ParentProps, Show, Suspense, type ValidComponent, @@ -75,15 +78,19 @@ import { type XY, type ZoomSegment, } from "~/utils/tauri"; +import IconLucideArrowLeftRight from "~icons/lucide/arrow-left-right"; import IconLucideBoxSelect from "~icons/lucide/box-select"; import IconLucideColumns2 from "~icons/lucide/columns-2"; import IconLucideEyeOff from "~icons/lucide/eye-off"; +import IconLucideFlipHorizontal2 from "~icons/lucide/flip-horizontal-2"; +import IconLucideFlipVertical2 from "~icons/lucide/flip-vertical-2"; import IconLucideGrid from "~icons/lucide/grid"; import IconLucideImageOff from "~icons/lucide/image-off"; import IconLucideKeyboard from "~icons/lucide/keyboard"; import IconLucideLaptop from "~icons/lucide/laptop"; import IconLucideMonitor from "~icons/lucide/monitor"; import IconLucideMoon from "~icons/lucide/moon"; +import IconLucideMoveRight from "~icons/lucide/move-right"; import IconLucideMusic from "~icons/lucide/music"; import IconLucidePalette from "~icons/lucide/palette"; import IconLucideRabbit from "~icons/lucide/rabbit"; @@ -130,6 +137,41 @@ import { TEXT_FONT_SIZE_MIN, type TextSegment, } from "./text"; +import { + ANGLE_PRESETS, + anglePresetMotion, + anglePresetPose, + applyMotionTemplate, + CAMERA3D_BLUR_MODE_SEEDS, + CAMERA3D_BOKEH_MAX_STRENGTH, + CAMERA3D_LIMITS, + CAMERA3D_MIN_SHOT_DURATION, + CAMERA3D_RESET_POSE, + CAMERA3D_SCENES, + CAMERA3D_TRANSITION_LIMITS, + type Camera3DAnglePreset, + type Camera3DBlurMode, + type Camera3DBlurScalarKey, + type Camera3DFlipAxis, + type Camera3DMotionEasing, + type Camera3DMotionTemplate, + type Camera3DProperties, + type Camera3DPropertyKey, + type Camera3DScene, + type Camera3DSegment, + camera3DPosesEqual, + camera3dBlurLimit, + cssPreviewTransform, + defaultCamera3DBlur, + flipCamera3DSegment, + getEndPose, + getMotionEasing, + getStartPose, + MOTION_EASINGS, + MOTION_TEMPLATES, + matchAnglePreset, + setMotion, +} from "./three-d"; import { ComingSoonTooltip, EditorButton, @@ -515,7 +557,8 @@ export function ConfigSidebar() { @@ -611,7 +658,8 @@ export function ConfigSidebar() { hidden: !!editorState.timeline.selection || editorState.timeline.audioPicker !== null || - editorState.timeline.audioReplace !== null, + editorState.timeline.audioReplace !== null || + editorState.timeline.camera3dSetup !== null, }} > setEditorState("timeline", "audioPicker", null)} /> + + {(setup) => ( + setEditorState("timeline", "camera3dSetup", null)} + /> + )} + { const index = editorState.timeline.audioReplace; @@ -1492,6 +1558,79 @@ export function ConfigSidebar() { )} + { + const camera3dSelection = selection(); + if (camera3dSelection.type !== "3d") return; + + const segments = camera3dSelection.indices + .map((index) => ({ + index, + segment: project.timeline?.camera3dSegments?.[index], + })) + .filter( + ( + item, + ): item is { index: number; segment: Camera3DSegment } => + item.segment !== undefined, + ); + + if (segments.length === 0) { + setEditorState("timeline", "selection", null); + return; + } + return { selection: camera3dSelection, segments }; + })()} + > + {(value) => ( +
+
+
+ + setEditorState("timeline", "selection", null) + } + leftIcon={} + > + Done + + + {value().segments.length} 3D{" "} + {value().segments.length === 1 + ? "segment" + : "segments"}{" "} + selected + +
+ { + projectActions.deleteCamera3DSegments( + value().segments.map((s) => s.index), + ); + }} + leftIcon={} + > + Delete + +
+ + {(item) => ( +
+ +
+ )} +
+
+ )} +
{ const sceneSelection = selection(); @@ -4070,6 +4209,910 @@ function MaskSegmentConfig(props: { ); } +const CAMERA3D_SLIDERS: Array<{ + key: Camera3DPropertyKey; + label: string; + unit: string; +}> = [ + { key: "tiltX", label: "Tilt X", unit: "°" }, + { key: "tiltY", label: "Tilt Y", unit: "°" }, + { key: "roll", label: "Roll", unit: "°" }, + { key: "rotateX", label: "Rotate X", unit: "°" }, + { key: "rotateY", label: "Rotate Y", unit: "°" }, + { key: "fov", label: "Field of view", unit: "°" }, + { key: "zoom", label: "Zoom", unit: "" }, + { key: "panX", label: "Pan X", unit: "" }, + { key: "panY", label: "Pan Y", unit: "" }, +]; + +function camera3dSliderIcon(key: Camera3DPropertyKey) { + switch (key) { + case "roll": + return ; + case "fov": + return ; + case "zoom": + return ; + case "panX": + case "panY": + return ; + default: + return ; + } +} + +const CAMERA3D_BLUR_MODE_OPTIONS: Array<{ + value: Camera3DBlurMode; + label: string; +}> = [ + { value: "none", label: "None" }, + { value: "radial", label: "Radial" }, + { value: "directional", label: "Directional" }, + { value: "tiltShift", label: "Tilt Shift" }, +]; + +type Camera3DBlurSlider = { + key: Camera3DBlurScalarKey; + label: string; + unit: string; +}; + +// Each mode exposes only the parameters it actually reads, in display order. +const CAMERA3D_BLUR_SLIDERS: Record< + Exclude, + Camera3DBlurSlider[] +> = { + radial: [ + { key: "strength", label: "Strength", unit: "" }, + { key: "focusX", label: "Focus X", unit: "" }, + { key: "focusY", label: "Focus Y", unit: "" }, + { key: "focusSize", label: "Focus size", unit: "" }, + { key: "falloff", label: "Falloff", unit: "" }, + ], + directional: [ + { key: "strength", label: "Strength", unit: "" }, + { key: "angle", label: "Angle", unit: "°" }, + { key: "dirPosition", label: "Position", unit: "" }, + { key: "falloff", label: "Falloff", unit: "" }, + ], + tiltShift: [ + { key: "strength", label: "Strength", unit: "" }, + { key: "focusY", label: "Scan", unit: "" }, + { key: "focusSize", label: "Focus size", unit: "" }, + { key: "angle", label: "Angle", unit: "°" }, + { key: "falloff", label: "Falloff", unit: "" }, + ], +}; + +function Camera3DTransitionInput(props: { + label: string; + value: number; + onChange: (value: number) => void; +}) { + const [text, setText] = createWritableMemo(() => props.value.toString()); + + return ( +
+ {props.label} +
+ { + if (Number.isNaN(value)) return; + props.onChange( + Math.min( + Math.max(value, CAMERA3D_TRANSITION_LIMITS.min), + CAMERA3D_TRANSITION_LIMITS.max, + ), + ); + }} + minValue={CAMERA3D_TRANSITION_LIMITS.min} + maxValue={CAMERA3D_TRANSITION_LIMITS.max} + step={CAMERA3D_TRANSITION_LIMITS.step} + > + { + if (text() === "" || Number.isNaN(props.value)) { + setText("0"); + props.onChange(0); + } + }} + class="w-20 p-1.5 border rounded-lg bg-gray-1 focus-visible:outline-hidden" + /> + + s +
+
+ ); +} + +const CAMERA3D_ANGLE_PREVIEW_HEIGHT = 30; +const CAMERA3D_TEMPLATE_PREVIEW_HEIGHT = 40; +/** Scenes lead the section and are three across, so their cards read taller. */ +const CAMERA3D_SCENE_PREVIEW_HEIGHT = 48; +/** The two pose cards are the panel's main control, so they read larger. */ +const CAMERA3D_POSE_PREVIEW_HEIGHT = 56; +const CAMERA3D_PREVIEW_TRANSITION = "700ms ease-in-out"; + +/** + * A `Field` that folds away. The header keeps the Field rhythm so a closed + * section reads as one more label in the column, with an optional summary that + * shows the state without opening it. + */ +function Camera3DSection( + props: ParentProps<{ + name: string; + icon: JSX.Element; + summary?: string; + open: boolean; + onOpenChange: (open: boolean) => void; + }>, +) { + return ( + + + {props.icon} + {props.name} + + {props.summary} + + + + +
{props.children}
+
+
+ ); +} + +// A CSS-3D stand-in for the renderer: the perspective distance reproduces the +// field of view at this card height and the rotations run in the renderer's +// order (camera orbit, then the content plane's own fold). +function Camera3DPosePreview(props: { + pose: Camera3DProperties; + height: number; + animate?: boolean; +}) { + const style = () => cssPreviewTransform(props.pose, props.height); + + return ( +
+
+
+ ); +} + +/** + * One scene, as a card: the pose its first shot opens on, drifting to that + * shot's end pose while hovered. Shared by the panel's Scenes row and the empty + * track's setup flow, so the two always offer the same thing. + */ +function Camera3DSceneCard(props: { + scene: Camera3DScene; + shotCount?: number; + selected?: boolean; + onClick: () => void; +}) { + const [hovered, setHovered] = createSignal(false); + const shotCount = () => props.shotCount ?? props.scene.shots.length; + + return ( + + ); +} + +/** + * The empty 3D track's setup flow: pick a look and how many cuts it makes, + * with the track drawing the result live underneath. Confirming lays the whole + * chain down in one step. + */ +function Camera3DSetupPanel(props: { + setup: { sceneId: string; shots: number }; + onClose: () => void; +}) { + const { projectActions, setEditorState, totalDuration } = useEditorContext(); + + const scene = () => + CAMERA3D_SCENES.find((item) => item.id === props.setup.sceneId) ?? + CAMERA3D_SCENES[0]; + + // A shot under the minimum is a glitch rather than a cut, so a short video + // simply offers fewer of them. + const maxShots = () => + Math.max( + 1, + Math.min( + scene().shots.length, + Math.floor(totalDuration() / CAMERA3D_MIN_SHOT_DURATION), + ), + ); + const shots = () => Math.min(props.setup.shots, maxShots()); + + const updateSetup = (changes: Partial<{ sceneId: string; shots: number }>) => + setEditorState("timeline", "camera3dSetup", (current) => + current ? { ...current, ...changes } : current, + ); + + return ( +
+
+ props.onClose()} + leftIcon={} + > + Close + + New 3D scene +
+ +

+ Lay a chain of camera moves over the whole video +

+ + }> +
+ + {(item) => ( + updateSetup({ sceneId: item.id })} + /> + )} + +
+
+ + +
+
+ index + 1)}> + {(count) => { + const tooShort = () => count > maxShots(); + return ( + // The title lives on the wrapper: a disabled button never + // hovers, so it would never show its own tooltip. +
+ +
+ ); + }} +
+
+

+ Shots split the video into separate camera moves. +

+
+
+ +
+ + +
+
+ ); +} + +function Camera3DSegmentConfig(props: { + segmentIndex: number; + segment: Camera3DSegment; +}) { + const { setProject, setEditorState, projectActions } = useEditorContext(); + + const updateSegment = (fn: (segment: Camera3DSegment) => void) => { + setProject( + "timeline", + "camera3dSegments", + produce((segments) => { + const target = segments?.[props.segmentIndex]; + if (!target) return; + fn(target); + }), + ); + }; + + // A 3D segment is one move: the pose it opens on and the pose it lands on. + // Everything in this panel reads and writes that pair, and the per-property + // keyframe tracks underneath are only how the renderer is fed. + const startPose = () => getStartPose(props.segment); + const endPose = () => getEndPose(props.segment); + const isStill = () => camera3DPosesEqual(startPose(), endPose()); + + // Which of the two poses the Camera sliders are pointed at. + const [editingEnd, setEditingEnd] = createSignal(false); + const selectedPose = () => (editingEnd() ? endPose() : startPose()); + + // Selecting another segment reuses this panel, so the card selection has to + // fall back to Start rather than carry over. + createEffect( + on( + () => props.segmentIndex, + () => setEditingEnd(false), + { defer: true }, + ), + ); + + // Parking the playhead on the pose being edited is what makes the canvas + // show it. The end pose is sampled a hair inside the segment so the playhead + // stays on this segment instead of falling into the next one. + const seekToPose = (end: boolean) => { + const time = end + ? Math.max(props.segment.end - 0.01, props.segment.start) + : props.segment.start; + batch(() => { + setEditorState("playbackTime", time); + setEditorState("previewTime", null); + }); + }; + + const selectPose = (end: boolean) => + batch(() => { + setEditingEnd(end); + seekToPose(end); + }); + + const writeMotion = ( + start: Camera3DProperties, + end: Camera3DProperties, + easing = getMotionEasing(props.segment), + ) => updateSegment((segment) => setMotion(segment, start, end, easing)); + + // A camera edit on a still shot moves both ends, so dialling in a hold never + // turns into an unrequested move. Once the shot moves, each card owns its + // own pose. + const writeSelectedPose = (pose: Camera3DProperties) => { + if (isStill()) writeMotion(pose, pose); + else if (editingEnd()) writeMotion(startPose(), pose); + else writeMotion(pose, endPose()); + }; + + const setPoseProperty = (key: Camera3DPropertyKey, value: number) => + writeSelectedPose({ ...selectedPose(), [key]: value }); + + const swapPoses = () => { + const start = startPose(); + writeMotion(endPose(), start); + }; + + const flipSegment = (axis: Camera3DFlipAxis) => + updateSegment((segment) => flipCamera3DSegment(segment, axis)); + + const makeStill = () => { + const start = startPose(); + writeMotion(start, start); + }; + + const resetCamera = () => writeSelectedPose({ ...CAMERA3D_RESET_POSE }); + + // The shot's identity is the pose it opens on, so the ring stays put while + // the end pose is being edited. + const activeAnglePresetId = () => matchAnglePreset(startPose()); + + const [hoveredTemplate, setHoveredTemplate] = createSignal( + null, + ); + + // Templates own the whole camera animation: the existing move is replaced + // and the playhead returns to the start so the result plays from its + // first pose. + const applyTemplate = (template: Camera3DMotionTemplate) => { + batch(() => { + updateSegment((segment) => { + applyMotionTemplate(segment, template); + }); + setEditingEnd(false); + setEditorState("playbackTime", props.segment.start); + setEditorState("previewTime", null); + }); + }; + + // Angle presets are moving shots too, exactly like the motion grid: the + // shot opens on the named pose and drifts. + const applyAnglePreset = (preset: Camera3DAnglePreset) => + applyTemplate(anglePresetMotion(preset)); + + // A scene replaces this one segment with its whole chain of shots, so the + // panel hands off to the project action that owns the splice. + const applyScene = (scene: Camera3DScene) => + projectActions.applyCamera3DScene(props.segmentIndex, scene.id); + + const motionEasing = () => getMotionEasing(props.segment); + + const blur = () => props.segment.blur; + + const blurSliders = () => { + const mode = blur().mode; + return mode === "none" ? [] : CAMERA3D_BLUR_SLIDERS[mode]; + }; + + // Blur is on by default now, so the closed section still has to say so. + const blurSummary = () => { + const mode = blur().mode; + if (mode === "none") return "Off"; + const label = + CAMERA3D_BLUR_MODE_OPTIONS.find((option) => option.value === mode) + ?.label ?? mode; + return `${label} ${Math.round(blur().strength)}`; + }; + + // Blur is segment-level and static: it is never part of the move. + const setBlurValue = (key: Camera3DBlurScalarKey, value: number) => + updateSegment((segment) => { + segment.blur[key] = value; + }); + + const setBlurMode = (mode: Camera3DBlurMode) => { + if (mode === blur().mode) return; + updateSegment((segment) => { + segment.blur.mode = mode; + const seed = CAMERA3D_BLUR_MODE_SEEDS[mode]; + for (const key of Object.keys(seed) as Camera3DBlurScalarKey[]) { + const value = seed[key]; + if (value !== undefined) segment.blur[key] = value; + } + }); + }; + + const setBokeh = (enabled: boolean) => { + updateSegment((segment) => { + segment.blur.bokeh = enabled; + if (!enabled) return; + // The bokeh kernel tops out at 20, so pull the strength down with the + // slider's new ceiling. + segment.blur.strength = Math.min( + segment.blur.strength, + CAMERA3D_BOKEH_MAX_STRENGTH, + ); + }); + }; + + const resetBlur = () => { + updateSegment((segment) => { + segment.blur = defaultCamera3DBlur(); + }); + }; + + // Section state is panel-local: it is how this user is reading the panel + // right now, not something the project should remember. + const [cameraOpen, setCameraOpen] = createSignal(false); + const [blurOpen, setBlurOpen] = createSignal(false); + const [advancedOpen, setAdvancedOpen] = createSignal(false); + + const poseCard = (label: string, end: boolean) => ( + + ); + + return ( +
+ }> +
+ {/* Scenes lead: one click lays a whole chained sequence over this + segment's range, where the rows below author a single shot. */} +
+ + {(scene) => ( + applyScene(scene)} + /> + )} + +
+
+ + {(preset) => ( + + )} + +
+
+ + {(template) => ( + + )} + +
+
+
+ }> +
+
+ {poseCard("Start", false)} + } + /> + {poseCard("End", true)} +
+
+ flipSegment("horizontal")} + tooltipText="Flip horizontal" + leftIcon={} + /> + flipSegment("vertical")} + tooltipText="Flip vertical" + leftIcon={} + /> + + Pick a template or edit the end pose to add motion +

+ } + > + +
+
+
+
+ } + summary={editingEnd() ? "End pose" : "Start pose"} + open={cameraOpen()} + onOpenChange={setCameraOpen} + > +
+ + {(slider) => ( +
+ + {camera3dSliderIcon(slider.key)} + {slider.label} + + setPoseProperty(slider.key, v[0])} + minValue={CAMERA3D_LIMITS[slider.key].min} + maxValue={CAMERA3D_LIMITS[slider.key].max} + step={CAMERA3D_LIMITS[slider.key].step} + formatTooltip={slider.unit} + /> +
+ )} +
+ } + onClick={resetCamera} + > + Reset camera + +
+
+ } + summary={blurSummary()} + open={blurOpen()} + onOpenChange={setBlurOpen} + > +
+ +
+ + options={CAMERA3D_BLUR_MODE_OPTIONS} + optionValue="value" + optionTextValue="label" + value={CAMERA3D_BLUR_MODE_OPTIONS.find( + (option) => option.value === blur().mode, + )} + onChange={(option) => { + if (option) setBlurMode(option.value); + }} + disallowEmptySelection + itemComponent={(itemProps) => ( + + as={KSelect.Item} + item={itemProps.item} + > + + {itemProps.item.rawValue.label} + + + )} + > + + class="flex-1 text-sm text-left truncate text-(--gray-500) font-normal"> + {(state) => {state.selectedOption().label}} + + + as={(iconProps) => ( + + )} + /> + + + + as={KSelect.Content} + class={cx(topSlideAnimateClasses, "z-50")} + > + + class="overflow-y-auto max-h-32" + as={KSelect.Listbox} + /> + + + +
+
+ + Pick a mode to blur everything outside the focus area. +

+ } + > + + {(slider) => { + const limit = () => camera3dBlurLimit(slider.key, blur()); + return ( +
+ {slider.label} + setBlurValue(slider.key, v[0])} + minValue={limit().min} + maxValue={limit().max} + step={limit().step} + formatTooltip={slider.unit} + /> +
+ ); + }} +
+ + + + } + onClick={resetBlur} + > + Turn blur off + +
+
+
+ } + open={advancedOpen()} + onOpenChange={setAdvancedOpen} + > +
+ +
+ + options={MOTION_EASINGS} + optionValue="id" + optionTextValue="label" + value={motionEasing()} + onChange={(option) => { + if (option) writeMotion(startPose(), endPose(), option); + }} + // A still shot has no span to shape, and nowhere to store a + // curve, so the picker would silently snap back. + disabled={isStill()} + disallowEmptySelection + itemComponent={(itemProps) => ( + + as={KSelect.Item} + item={itemProps.item} + > + + {itemProps.item.rawValue.label} + + + )} + > + + class="flex-1 text-sm text-left truncate text-(--gray-500) font-normal"> + {(state) => {state.selectedOption().label}} + + + as={(iconProps) => ( + + )} + /> + + + + as={KSelect.Content} + class={cx(topSlideAnimateClasses, "z-50")} + > + + class="overflow-y-auto max-h-32" + as={KSelect.Listbox} + /> + + + +
+
+
+ + updateSegment((segment) => { + segment.transitionIn = value; + }) + } + /> + + updateSegment((segment) => { + segment.transitionOut = value; + }) + } + /> +
+
+
+
+ ); +} + function ZoomSegmentPreview(props: { segmentIndex: number; segment: ZoomSegment; diff --git a/apps/desktop/src/routes/editor/Editor.tsx b/apps/desktop/src/routes/editor/Editor.tsx index f53a293e624..52c839204d4 100644 --- a/apps/desktop/src/routes/editor/Editor.tsx +++ b/apps/desktop/src/routes/editor/Editor.tsx @@ -129,6 +129,13 @@ function getPreviewProjectConfig( }; } + if (!editorState.timeline.tracks["3d"] && config.timeline) { + config.timeline = { + ...config.timeline, + camera3dSegments: [], + }; + } + return config; } @@ -566,6 +573,7 @@ function Inner() { return { caption: editorState.timeline.tracks.caption, keyboard: editorState.timeline.tracks.keyboard, + threeD: editorState.timeline.tracks["3d"], }; }, () => { diff --git a/apps/desktop/src/routes/editor/Player.tsx b/apps/desktop/src/routes/editor/Player.tsx index 2eae18aa726..1da2c7e40f4 100644 --- a/apps/desktop/src/routes/editor/Player.tsx +++ b/apps/desktop/src/routes/editor/Player.tsx @@ -108,6 +108,7 @@ export function PlayerContent() { sceneSegments: [], maskSegments: [], textSegments: [], + camera3dSegments: [], transitions: [], }), captionSegments: createCaptionTrackSegments(captionSegments), diff --git a/apps/desktop/src/routes/editor/Timeline/ThreeDTrack.tsx b/apps/desktop/src/routes/editor/Timeline/ThreeDTrack.tsx new file mode 100644 index 00000000000..59b18d8826a --- /dev/null +++ b/apps/desktop/src/routes/editor/Timeline/ThreeDTrack.tsx @@ -0,0 +1,771 @@ +import { createEventListenerMap } from "@solid-primitives/event-listener"; +import { cx } from "cva"; +import { Array, Option } from "effect"; +import { + batch, + createMemo, + createRoot, + createSignal, + Index, + Match, + Show, + Switch, +} from "solid-js"; +import { produce } from "solid-js/store"; +import { useEditorContext } from "../context"; +import { + CAMERA3D_SCENES, + defaultCamera3DSegment, + fitCamera3DMotionToSegment, + hasCamera3DMotion, +} from "../three-d"; +import { + useSegmentContext, + useTimelineContext, + useTrackContext, +} from "./context"; +import { + SegmentContent, + SegmentHandle, + SegmentRoot, + TrackRoot, + useSegmentTranslateX, + useSegmentWidth, + useSetPreviewTime, +} from "./Track"; + +export type ThreeDSegmentDragState = + | { type: "idle" } + | { type: "movePending" } + | { type: "moving" }; + +const MIN_THREE_D_SEGMENT_PIXEL_WIDTH = 40; +const MIN_NEW_SEGMENT_PIXEL_WIDTH = 80; +const MIN_NEW_SEGMENT_SECS_WIDTH = 1; + +/** What the setup flow opens on: the full showcase sequence. */ +const CAMERA3D_SETUP_SEED = { sceneId: "showcase", shots: 3 }; + +/** + * One shot of the scene the setup flow is offering, drawn where it would land. + * Non-interactive: the whole placeholder area belongs to the open flow. + */ +function Camera3DSetupGhost(props: { + segment: { start: number; end: number }; + label: string; +}) { + const translateX = useSegmentTranslateX(() => props.segment); + const width = useSegmentWidth(() => props.segment); + + return ( +
+ {props.label} +
+ ); +} + +export function ThreeDTrack(props: { + onDragStateChanged: (v: ThreeDSegmentDragState) => void; + handleUpdatePlayhead: (e: MouseEvent) => void; +}) { + const { + project, + setProject, + projectHistory, + setEditorState, + editorState, + totalDuration, + projectActions, + camera3DScenePreview, + } = useEditorContext(); + + const { duration, secsPerPixel } = useTimelineContext(); + const setPreviewTime = useSetPreviewTime(); + + const [creatingSegmentViaDrag, setCreatingSegmentViaDrag] = + createSignal(false); + + const hasCamera3DSegments = () => + (project.timeline?.camera3dSegments?.length ?? 0) > 0; + + const setup = () => editorState.timeline.camera3dSetup; + + const setupSegments = createMemo(() => { + const current = setup(); + if (!current) return []; + return camera3DScenePreview(current.sceneId, current.shots); + }); + + const setupShotLabel = (index: number) => { + const current = setup(); + const scene = current + ? CAMERA3D_SCENES.find((s) => s.id === current.sceneId) + : undefined; + return scene?.shots[index]?.name ?? `Shot ${index + 1}`; + }; + + const startSetup = () => + batch(() => { + setEditorState("timeline", "selection", null); + setEditorState("timeline", "audioPicker", null); + setEditorState("timeline", "audioReplace", null); + setEditorState("timeline", "camera3dSetup", { ...CAMERA3D_SETUP_SEED }); + }); + const selectedCamera3DIndices = createMemo(() => { + const selection = editorState.timeline.selection; + if (!selection || selection.type !== "3d") return null; + return new Set(selection.indices); + }); + + const newSegmentMinDuration = () => + Math.max( + MIN_NEW_SEGMENT_PIXEL_WIDTH * secsPerPixel(), + MIN_NEW_SEGMENT_SECS_WIDTH, + ); + + // Returns a start and end time for a new segment that can be inserted at the + // current previewTime, if conditions permit + const newSegmentDetails = () => { + if ( + creatingSegmentViaDrag() || + editorState.timeline.hoveredTrack !== "3d" || + editorState.previewTime === null + ) + return; + + // An empty track belongs to the setup flow, whether it is open or still + // only offering itself, so nothing is drag-created over it. + if (!hasCamera3DSegments() || setup()) return; + + const { previewTime } = editorState; + + const nextSegment = Array.findFirstWithIndex( + project.timeline?.camera3dSegments ?? [], + (s) => previewTime <= s.start, + ); + + const prevSegment = Array.findLastIndex( + project.timeline?.camera3dSegments ?? [], + (s) => previewTime >= s.start, + ).pipe( + Option.flatMap((index) => + Option.fromNullable(project.timeline?.camera3dSegments?.[index]).pipe( + Option.map((segment) => [segment, index] as const), + ), + ), + ); + + if ( + Option.isSome(prevSegment) && + previewTime > prevSegment.value[0].start && + previewTime < prevSegment.value[0].end + ) + return; + + const minDuration = newSegmentMinDuration(); + + if (Option.isSome(nextSegment)) { + if (Option.isSome(prevSegment)) { + const availableTime = + nextSegment.value[0].start - prevSegment.value[0].end; + + if (availableTime < minDuration) return; + } + + if (nextSegment.value[0].start - previewTime < 1) + return { + index: nextSegment.value[1], + start: nextSegment.value[0].start - minDuration, + end: nextSegment.value[0].start, + max: nextSegment.value[0].start, + }; + } + + return { + index: nextSegment.pipe(Option.map(([_, i]) => i)), + start: previewTime, + end: previewTime + minDuration, + max: nextSegment.pipe( + Option.map(([s]) => s.start), + Option.getOrElse(() => totalDuration()), + ), + }; + }; + + return ( + setEditorState("timeline", "hoveredTrack", "3d")} + onMouseLeave={() => setEditorState("timeline", "hoveredTrack", null)} + onMouseDown={(e) => { + if (e.button !== 0) return; + + const baseSegment = newSegmentDetails(); + if (!baseSegment) return; + + createRoot((dispose) => { + let segmentCreated = false; + let createdSegmentIndex = -1; + const initialMouseX = e.clientX; + const initialEndTime = baseSegment.end; + + const minDuration = newSegmentMinDuration; + + const createSegment = (endTime: number) => { + if (segmentCreated) return; + + batch(() => { + setProject("timeline", "camera3dSegments", (v) => v ?? []); + setProject( + "timeline", + "camera3dSegments", + produce((camera3dSegments) => { + camera3dSegments ??= []; + + let index = 0; + + for (let i = 0; i < camera3dSegments.length; i++) { + if (camera3dSegments[i].start < baseSegment.start) { + index = i + 1; + } + } + + const minEndTime = baseSegment.start + minDuration(); + + camera3dSegments.splice( + index, + 0, + defaultCamera3DSegment( + baseSegment.start, + Math.max(minEndTime, endTime), + ), + ); + + createdSegmentIndex = index; + }), + ); + setEditorState("timeline", "selection", { + type: "3d", + indices: [createdSegmentIndex], + }); + }); + segmentCreated = true; + }; + + const updateSegment = (endTime: number) => { + if (!segmentCreated || createdSegmentIndex === -1) return; + + const minEndTime = baseSegment.start + minDuration(); + + setProject( + "timeline", + "camera3dSegments", + createdSegmentIndex, + "end", + Math.max(minEndTime, endTime), + ); + }; + + const handleMouseMove = (moveEvent: MouseEvent) => { + const deltaX = moveEvent.clientX - initialMouseX; + const deltaTime = + deltaX * secsPerPixel() - (baseSegment.end - baseSegment.start); + const newEndTime = initialEndTime + deltaTime; + + const minEndTime = baseSegment.start + minDuration(); + const maxEndTime = baseSegment.max; + + const clampedEndTime = Math.min( + Math.max(minEndTime, newEndTime), + maxEndTime, + ); + + if (!segmentCreated) { + setCreatingSegmentViaDrag(true); + createSegment(clampedEndTime); + } else { + if (deltaTime < 0) return; + updateSegment(clampedEndTime); + } + }; + + const handleMouseUp = () => { + setCreatingSegmentViaDrag(false); + dispose(); + + if (!segmentCreated) { + createSegment(initialEndTime); + } + }; + + createEventListenerMap(window, { + mousemove: handleMouseMove, + mouseup: handleMouseUp, + }); + }); + }} + > + e.stopPropagation()} + onClick={(e) => { + e.stopPropagation(); + startSetup(); + }} + > + + + + Add 3D scene + + } + > + {/* The open flow owns the whole track: the shots it would lay + down are drawn where they will land, and a click anywhere + over them leaves the flow alone rather than closing it. */} +
e.stopPropagation()} + > + + {(ghost, index) => ( + + )} + +
+
+ } + > + + {(segment, i) => { + const { setTrackState } = useTrackContext(); + + const camera3dSegments = () => + project.timeline?.camera3dSegments ?? []; + + // Double-clicking a handle expands the segment as far as it can go + // in that direction (up to the neighbouring segment / timeline edge). + const fillStart = () => { + const segs = camera3dSegments(); + let minValue = 0; + for (let j = segs.length - 1; j >= 0; j--) { + const s = segs[j]; + if (s && s.end <= segment().start) { + minValue = s.end; + break; + } + } + setProject( + "timeline", + "camera3dSegments", + produce((s) => { + const target = s[i]; + if (!target) return; + target.start = minValue; + fitCamera3DMotionToSegment(target); + s.sort((a, b) => a.start - b.start); + }), + ); + setPreviewTime(minValue); + }; + + const fillEnd = () => { + const segs = camera3dSegments(); + let maxValue = totalDuration(); + for (let j = 0; j < segs.length; j++) { + const s = segs[j]; + if (s && s.start > segment().end) { + maxValue = s.start; + break; + } + } + setProject( + "timeline", + "camera3dSegments", + produce((s) => { + const target = s[i]; + if (!target) return; + target.end = maxValue; + fitCamera3DMotionToSegment(target); + s.sort((a, b) => a.start - b.start); + }), + ); + setPreviewTime(maxValue); + }; + + function createMouseDownDrag( + setup: () => T, + _update: (e: MouseEvent, v: T, initialMouseX: number) => void, + ) { + return (downEvent: MouseEvent) => { + if (editorState.timeline.interactMode !== "seek") return; + + downEvent.stopPropagation(); + + const initial = setup(); + + let moved = false; + let initialMouseX: null | number = null; + + setTrackState("draggingSegment", true); + + const resumeHistory = projectHistory.pause(); + + props.onDragStateChanged({ type: "movePending" }); + + function finish(e: MouseEvent) { + resumeHistory(); + if (!moved) { + e.stopPropagation(); + + const currentSelection = editorState.timeline.selection; + const segmentIndex = i; + const isMultiSelect = e.ctrlKey || e.metaKey; + const isRangeSelect = e.shiftKey; + + if (isRangeSelect && currentSelection?.type === "3d") { + const existingIndices = currentSelection.indices; + const lastIndex = + existingIndices[existingIndices.length - 1]; + const start = Math.min(lastIndex, segmentIndex); + const end = Math.max(lastIndex, segmentIndex); + const rangeIndices: number[] = []; + for (let idx = start; idx <= end; idx++) { + rangeIndices.push(idx); + } + + setEditorState("timeline", "selection", { + type: "3d", + indices: rangeIndices, + }); + } else if (isMultiSelect) { + if (currentSelection?.type === "3d") { + const baseIndices = currentSelection.indices; + const exists = baseIndices.includes(segmentIndex); + const newIndices = exists + ? baseIndices.filter((idx) => idx !== segmentIndex) + : [...baseIndices, segmentIndex]; + + if (newIndices.length > 0) { + setEditorState("timeline", "selection", { + type: "3d", + indices: newIndices, + }); + } else { + setEditorState("timeline", "selection", null); + } + } else { + setEditorState("timeline", "selection", { + type: "3d", + indices: [segmentIndex], + }); + } + } else { + setEditorState("timeline", "selection", { + type: "3d", + indices: [segmentIndex], + }); + } + props.handleUpdatePlayhead(e); + } + props.onDragStateChanged({ type: "idle" }); + setTrackState("draggingSegment", false); + } + + function update(event: MouseEvent) { + if (Math.abs(event.clientX - downEvent.clientX) > 2) { + if (!moved) { + moved = true; + initialMouseX = event.clientX; + props.onDragStateChanged({ + type: "moving", + }); + } + } + + if (initialMouseX === null) return; + + _update(event, initial, initialMouseX); + } + + createRoot((dispose) => { + createEventListenerMap(window, { + mousemove: (e) => { + update(e); + }, + mouseup: (e) => { + update(e); + finish(e); + dispose(); + }, + }); + }); + }; + } + + const isSelected = createMemo(() => { + const indices = selectedCamera3DIndices(); + if (!indices) return false; + return indices.has(i); + }); + + return ( + { + e.stopPropagation(); + + if (editorState.timeline.interactMode === "split") { + const rect = e.currentTarget.getBoundingClientRect(); + const fraction = (e.clientX - rect.left) / rect.width; + + const splitTime = + fraction * (segment().end - segment().start); + + projectActions.splitCamera3DSegment(i, splitTime); + } + }} + > + { + e.stopPropagation(); + fillStart(); + }} + onMouseDown={createMouseDownDrag( + () => { + const start = segment().start; + const minDuration = Math.max( + 1, + secsPerPixel() * MIN_THREE_D_SEGMENT_PIXEL_WIDTH, + ); + + let minValue = 0; + + const maxValue = segment().end - minDuration; + + for (let j = camera3dSegments().length - 1; j >= 0; j--) { + const other = camera3dSegments()[j]; + if (!other) continue; + if (other.end <= start) { + minValue = other.end; + break; + } + } + + return { start, minValue, maxValue }; + }, + (e, value, initialMouseX) => { + const newStart = + value.start + + (e.clientX - initialMouseX) * secsPerPixel(); + const nextStart = Math.min( + value.maxValue, + Math.max(value.minValue, newStart), + ); + + setProject( + "timeline", + "camera3dSegments", + produce((s) => { + const target = s[i]; + if (!target) return; + target.start = nextStart; + fitCamera3DMotionToSegment(target); + s.sort((a, b) => a.start - b.start); + }), + ); + setPreviewTime(nextStart); + }, + )} + /> + { + const original = { ...segment() }; + + const prevSegment = camera3dSegments()[i - 1]; + const nextSegment = camera3dSegments()[i + 1]; + + const minStart = prevSegment?.end ?? 0; + const maxEnd = nextSegment?.start ?? duration(); + + return { + original, + minStart, + maxEnd, + }; + }, + (e, value, initialMouseX) => { + const rawDelta = + (e.clientX - initialMouseX) * secsPerPixel(); + + const newStart = value.original.start + rawDelta; + const newEnd = value.original.end + rawDelta; + + let delta = rawDelta; + + if (newStart < value.minStart) + delta = value.minStart - value.original.start; + else if (newEnd > value.maxEnd) + delta = value.maxEnd - value.original.end; + + setProject("timeline", "camera3dSegments", i, { + start: value.original.start + delta, + end: value.original.end + delta, + }); + }, + )} + > + {(() => { + const ctx = useSegmentContext(); + + return ( + + +
+ +
+
+ +
+ + 3D +
+
+ +
+ + {ctx.width() >= 140 ? "3D Perspective" : "3D"} + +
+ + + {hasCamera3DMotion(segment()) + ? "Motion" + : "Still"} + + {/* Presentation only: the arrow says the shot + moves from its start pose to its end pose. */} + + + +
+
+
+
+ ); + })()} +
+ { + e.stopPropagation(); + fillEnd(); + }} + onMouseDown={createMouseDownDrag( + () => { + const end = segment().end; + const minDuration = Math.max( + 1, + secsPerPixel() * MIN_THREE_D_SEGMENT_PIXEL_WIDTH, + ); + + const minValue = segment().start + minDuration; + + let maxValue = duration(); + + for (let j = 0; j < camera3dSegments().length; j++) { + const other = camera3dSegments()[j]; + if (!other) continue; + if (other.start > end) { + maxValue = other.start; + break; + } + } + + return { end, minValue, maxValue }; + }, + (e, value, initialMouseX) => { + const newEnd = + value.end + + (e.clientX - initialMouseX) * secsPerPixel(); + const nextEnd = Math.min( + value.maxValue, + Math.max(value.minValue, newEnd), + ); + + setProject( + "timeline", + "camera3dSegments", + produce((s) => { + const target = s[i]; + if (!target) return; + target.end = nextEnd; + fitCamera3DMotionToSegment(target); + s.sort((a, b) => a.start - b.start); + }), + ); + setPreviewTime(nextEnd); + }, + )} + /> +
+ ); + }} +
+ + + {(details) => ( + + +

+ + +

+
+
+ )} +
+
+ ); +} diff --git a/apps/desktop/src/routes/editor/Timeline/TrackManager.tsx b/apps/desktop/src/routes/editor/Timeline/TrackManager.tsx index 314659e5a72..08971c4f275 100644 --- a/apps/desktop/src/routes/editor/Timeline/TrackManager.tsx +++ b/apps/desktop/src/routes/editor/Timeline/TrackManager.tsx @@ -53,6 +53,10 @@ const TRACK_META: Record = { description: "Switch layouts between your screen and camera.", unavailableHint: "Record with a camera to use scenes.", }, + "3d": { + description: "Tilt the scene in 3D perspective.", + unavailableHint: "", + }, }; // Comes straight from the shared `--track-*` CSS variable defined in theme.css, diff --git a/apps/desktop/src/routes/editor/Timeline/index.tsx b/apps/desktop/src/routes/editor/Timeline/index.tsx index 4919ec4fd65..4cd4af15822 100644 --- a/apps/desktop/src/routes/editor/Timeline/index.tsx +++ b/apps/desktop/src/routes/editor/Timeline/index.tsx @@ -51,6 +51,7 @@ import { type KeyboardSegmentDragState, KeyboardTrack } from "./KeyboardTrack"; import { type MaskSegmentDragState, MaskTrack } from "./MaskTrack"; import { type SceneSegmentDragState, SceneTrack } from "./SceneTrack"; import { type TextSegmentDragState, TextTrack } from "./TextTrack"; +import { type ThreeDSegmentDragState, ThreeDTrack } from "./ThreeDTrack"; import { TrackIcon, TrackManager } from "./TrackManager"; import { type ZoomSegmentDragState, ZoomTrack } from "./ZoomTrack"; @@ -71,6 +72,7 @@ const trackIcons: Record JSX.Element> = { zoom: () => , scene: () => , audio: () => , + "3d": () => , }; type TrackDefinition = { @@ -129,6 +131,12 @@ const trackDefinitions: TrackDefinition[] = [ icon: trackIcons.scene, locked: false, }, + { + type: "3d", + label: "3D", + icon: trackIcons["3d"], + locked: false, + }, ]; function deleteTrackLane( @@ -179,6 +187,7 @@ export function Timeline(props: { const openAudioPicker = (laneIndex: number) => { batch(() => { setEditorState("timeline", "selection", null); + setEditorState("timeline", "camera3dSetup", null); setEditorState("timeline", "audioPicker", laneIndex); }); }; @@ -187,6 +196,7 @@ export function Timeline(props: { const sceneAvailable = () => meta().hasCamera && !project.camera.hide; const captionTrackVisible = () => trackState().caption; const keyboardTrackVisible = () => trackState().keyboard; + const threeDTrackVisible = () => trackState()["3d"]; const trackOptions = createMemo(() => trackDefinitions.map((definition) => ({ ...definition, @@ -197,13 +207,15 @@ export function Timeline(props: { ? trackState().keyboard : definition.type === "scene" ? trackState().scene - : definition.type === "mask" - ? trackState().mask > 0 - : definition.type === "text" - ? trackState().text > 0 - : definition.type === "audio" - ? trackState().audio > 0 - : true, + : definition.type === "3d" + ? trackState()["3d"] + : definition.type === "mask" + ? trackState().mask > 0 + : definition.type === "text" + ? trackState().text > 0 + : definition.type === "audio" + ? trackState().audio > 0 + : true, available: definition.type === "scene" ? sceneAvailable() : true, supportsMultiple: definition.type === "mask" || @@ -246,6 +258,7 @@ export function Timeline(props: { textTrackRows().length + maskTrackRows().length + audioTrackRows().length + + (threeDTrackVisible() ? 1 : 0) + (sceneTrackVisible() ? 1 : 0), ); const trackHeight = createMemo(() => @@ -316,6 +329,19 @@ export function Timeline(props: { return; } + if (type === "3d") { + batch(() => { + setEditorState("timeline", "tracks", "3d", next); + if (!next && editorState.timeline.selection?.type === "3d") { + setEditorState("timeline", "selection", null); + } + // The setup flow lives on the track it is previewing, so hiding the + // track takes its sidebar panel with it. + if (!next) setEditorState("timeline", "camera3dSetup", null); + }); + return; + } + if (type === "text") { setEditorState( "timeline", @@ -551,6 +577,7 @@ export function Timeline(props: { textSegments: [], captionSegments: [], keyboardSegments: [], + camera3dSegments: [], transitions: [], }; project.timeline.captionSegments = []; @@ -575,6 +602,7 @@ export function Timeline(props: { textSegments: [], captionSegments: [], keyboardSegments: [], + camera3dSegments: [], transitions: [], }; project.timeline.keyboardSegments = []; @@ -587,9 +615,9 @@ export function Timeline(props: { resumeHistory(); } - // Zoom and Scene are permanent tracks — deleting from them clears every - // segment on the track instead of hiding the track row itself. - function handleClearTrackSegments(type: "zoom" | "scene") { + // Zoom, Scene and 3D keep their row once shown, so deleting from them + // clears every segment on the track instead of hiding the row itself. + function handleClearTrackSegments(type: "zoom" | "scene" | "3d") { const resumeHistory = projectHistory.pause(); batch(() => { @@ -602,6 +630,7 @@ export function Timeline(props: { const timeline = project.timeline; if (!timeline) return; if (type === "zoom") timeline.zoomSegments = []; + else if (type === "3d") timeline.camera3dSegments = []; else timeline.sceneSegments = []; }), ); @@ -647,6 +676,7 @@ export function Timeline(props: { textSegments: [], captionSegments: [], keyboardSegments: [], + camera3dSegments: [], transitions: [], }); resume(); @@ -673,7 +703,8 @@ export function Timeline(props: { !project.timeline?.zoomSegments || project.timeline.zoomSegments.length < 1 || !project.timeline?.maskSegments || - !project.timeline?.textSegments + !project.timeline?.textSegments || + !project.timeline?.camera3dSegments ) { setProject( produce((project) => { @@ -691,6 +722,7 @@ export function Timeline(props: { textSegments: [], captionSegments: [], keyboardSegments: [], + camera3dSegments: [], transitions: [], }; project.timeline.sceneSegments ??= []; @@ -699,6 +731,7 @@ export function Timeline(props: { project.timeline.maskSegments ??= []; project.timeline.textSegments ??= []; project.timeline.zoomSegments ??= []; + project.timeline.camera3dSegments ??= []; }), ); } @@ -710,6 +743,7 @@ export function Timeline(props: { let audioSegmentDragState = { type: "idle" } as AudioSegmentDragState; let captionSegmentDragState = { type: "idle" } as CaptionSegmentDragState; let keyboardSegmentDragState = { type: "idle" } as KeyboardSegmentDragState; + let threeDSegmentDragState = { type: "idle" } as ThreeDSegmentDragState; let pendingZoomDelta = 0; let pendingZoomOrigin: number | null = null; @@ -787,7 +821,8 @@ export function Timeline(props: { textSegmentDragState.type !== "moving" && audioSegmentDragState.type !== "moving" && captionSegmentDragState.type !== "moving" && - keyboardSegmentDragState.type !== "moving" + keyboardSegmentDragState.type !== "moving" && + threeDSegmentDragState.type !== "moving" ) { if (!metrics) return; const rawTime = @@ -850,6 +885,8 @@ export function Timeline(props: { projectActions.deleteTextSegments(selection.indices); } else if (selection.type === "audio") { projectActions.deleteAudioSegments(selection.indices); + } else if (selection.type === "3d") { + projectActions.deleteCamera3DSegments(selection.indices); } else if (selection.type === "transition") { projectActions.deleteClipTransition(selection.index); } else if (selection.type === "clip") { @@ -877,6 +914,7 @@ export function Timeline(props: { // Deselect all selected segments setEditorState("timeline", "selection", null); setEditorState("timeline", "audioPicker", null); + setEditorState("timeline", "camera3dSetup", null); } }); @@ -990,6 +1028,7 @@ export function Timeline(props: { if (zoomSegmentDragState.type === "idle") { setEditorState("timeline", "selection", null); setEditorState("timeline", "audioPicker", null); + setEditorState("timeline", "camera3dSetup", null); } }); createEventListener(window, "mouseup", () => { @@ -1240,6 +1279,27 @@ export function Timeline(props: { handleUpdatePlayhead={handleUpdatePlayhead} /> + + 0 + ? () => handleClearTrackSegments("3d") + : undefined + } + deleteLabel="Clear all" + deleteTitle="Delete all 3D segments" + > + { + threeDSegmentDragState = v; + }} + handleUpdatePlayhead={handleUpdatePlayhead} + /> + + & { segments: EditorTimelineSegment[]; transitions: ClipTransition[]; @@ -192,6 +209,7 @@ type EditorTimelineConfiguration = Omit< maskSegments: MaskSegment[]; textSegments: TextSegment[]; audioSegments?: AudioTrackSegment[]; + camera3dSegments: Camera3DSegment[]; }; type EditorCaptionsData = NonNullable & { @@ -269,6 +287,9 @@ export function normalizeProject( } ).audioSegments ?? [], ), + camera3dSegments: normalizeCamera3DSegments( + config.timeline.camera3dSegments, + ), } : undefined; const captions = config.captions @@ -305,6 +326,7 @@ export function serializeProjectConfiguration( maskSegments: project.timeline.maskSegments ?? [], textSegments: project.timeline.textSegments ?? [], audioSegments: project.timeline.audioSegments ?? [], + camera3dSegments: project.timeline.camera3dSegments ?? [], } : project.timeline; @@ -372,6 +394,10 @@ export const [EditorContextProvider, useEditorContext] = createContextProvider( } if (shift === 0) return; + const camera3dSegments = timeline.camera3dSegments ?? []; + const previousCamera3dDurations = camera3dSegments.map( + (segment) => segment.end - segment.start, + ); const tracks = [ timeline.zoomSegments, timeline.sceneSegments ?? [], @@ -380,14 +406,69 @@ export const [EditorContextProvider, useEditorContext] = createContextProvider( timeline.captionSegments ?? [], timeline.keyboardSegments ?? [], timeline.audioSegments ?? [], + camera3dSegments, ]; for (const track of tracks) { rippleTimelineTrack(track, boundary, shift); } + for (let index = 0; index < camera3dSegments.length; index++) { + const camera3dSegment = camera3dSegments[index]; + const previousDuration = previousCamera3dDurations[index]; + // Keyframe times are relative to the segment start, so a + // segment the ripple resized (the straddling case) has to + // have them rescaled onto its new length. + const nextDuration = camera3dSegment.end - camera3dSegment.start; + if (previousDuration <= 0 || nextDuration === previousDuration) + continue; + scaleKeyframeTimes( + camera3dSegment.tracks, + nextDuration / previousDuration, + ); + } }), ); }; + // Output-time boundaries of every clip that fall strictly inside a range. + // A 3D scene lines its cuts up with these so the camera changes shot on + // the same frame the footage does. + const camera3DClipCuts = (start: number, end: number) => { + const timeline = project.timeline; + if (!timeline) return []; + const offsets = clipTimelineOffsets( + timeline.segments, + timeline.transitions ?? [], + ); + const cuts: number[] = []; + for (let index = 0; index < timeline.segments.length; index++) { + const boundaries = [ + offsets[index], + offsets[index] + clipDuration(timeline.segments[index]), + ]; + for (const boundary of boundaries) + if (boundary > start && boundary < end) cuts.push(boundary); + } + return cuts; + }; + + /** + * The chain a scene would lay over the whole timeline. The setup flow's + * ghost placeholder and the action that commits it read this same + * function, so the track previews exactly what lands. + */ + const camera3DScenePreview = (sceneId: string, shots: number) => { + const scene = CAMERA3D_SCENES.find((s) => s.id === sceneId); + if (!scene) return []; + + const end = totalDuration(); + return applySceneToRange( + sceneWithShotCount(scene, shots), + 0, + end, + camera3DClipCuts(0, end), + ); + }; + const projectActions = { setClipTransition, normalizeClipTransitions: () => { @@ -545,6 +626,124 @@ export const [EditorContextProvider, useEditorContext] = createContextProvider( setEditorState("timeline", "selection", null); }); }, + splitCamera3DSegment: (index: number, time: number) => { + setProject( + "timeline", + "camera3dSegments", + produce((segments) => { + const segment = segments?.[index]; + if (!segment) return; + + const duration = segment.end - segment.start; + const remaining = duration - time; + if (time < 1 || remaining < 1) return; + + // A split must not change what plays: both halves meet on the pose + // the segment held at the cut, so the left half moves start -> mid + // and the right half picks up mid -> end. Blur is segment-level, so + // it is simply carried onto both halves. + const startPose = getStartPose(segment); + const midPose = evaluatePose(segment, time); + const endPose = getEndPose(segment); + const easing = getMotionEasing(segment); + + const right: Camera3DSegment = { + ...segment, + start: segment.start + time, + end: segment.end, + properties: { ...segment.properties }, + blur: { ...segment.blur }, + tracks: defaultCamera3DTracks(), + }; + setMotion(right, midPose, endPose, easing); + segments.splice(index + 1, 0, right); + + const left = segments[index]; + left.end = segment.start + time; + left.tracks = defaultCamera3DTracks(); + setMotion(left, startPose, midPose, easing); + sortTrackSegments(segments); + }), + ); + }, + deleteCamera3DSegments: (segmentIndices: number[]) => { + batch(() => { + setProject( + "timeline", + "camera3dSegments", + produce((segments) => { + if (!segments) return; + const sorted = [...new Set(segmentIndices)] + .filter( + (i) => Number.isInteger(i) && i >= 0 && i < segments.length, + ) + .sort((a, b) => b - a); + if (sorted.length === 0) return; + for (const i of sorted) segments.splice(i, 1); + }), + ); + setEditorState("timeline", "selection", null); + }); + }, + applyCamera3DScene: (segmentIndex: number, sceneId: string) => { + const scene = CAMERA3D_SCENES.find((s) => s.id === sceneId); + const segment = project.timeline?.camera3dSegments?.[segmentIndex]; + if (!scene || !segment) return; + + const { start, end } = segment; + const generated = applySceneToRange( + scene, + start, + end, + camera3DClipCuts(start, end), + ); + if (generated.length === 0) return; + + batch(() => { + setProject( + "timeline", + "camera3dSegments", + produce((segments) => { + if (!segments) return; + segments.splice(segmentIndex, 1, ...generated); + sortTrackSegments(segments); + }), + ); + setEditorState("timeline", "selection", { + type: "3d", + indices: generated.map((_, offset) => segmentIndex + offset), + }); + setEditorState("playbackTime", start); + setEditorState("previewTime", null); + }); + }, + addCamera3DScene: (sceneId: string, shots: number) => { + // Only ever an empty-track offer: the scene owns the whole timeline, + // so it must not land on top of shots someone has already authored. + if (!project.timeline) return; + if ((project.timeline.camera3dSegments?.length ?? 0) > 0) return; + + const generated = camera3DScenePreview(sceneId, shots); + if (generated.length === 0) return; + + batch(() => { + setProject("timeline", "camera3dSegments", (v) => v ?? []); + setProject( + "timeline", + "camera3dSegments", + produce((segments) => { + if (!segments) return; + segments.push(...generated); + sortTrackSegments(segments); + }), + ); + setEditorState("timeline", "camera3dSetup", null); + setEditorState("timeline", "tracks", "3d", true); + setEditorState("timeline", "selection", { type: "3d", indices: [0] }); + setEditorState("playbackTime", 0); + setEditorState("previewTime", null); + }); + }, splitMaskSegment: (index: number, time: number) => { setProject( "timeline", @@ -957,6 +1156,23 @@ export const [EditorContextProvider, useEditorContext] = createContextProvider( keyboardSegment.start += diff(keyboardSegment.start); keyboardSegment.end += diff(keyboardSegment.end); } + + for (const camera3dSegment of timeline.camera3dSegments ?? []) { + const previousDuration = + camera3dSegment.end - camera3dSegment.start; + camera3dSegment.start += diff(camera3dSegment.start); + camera3dSegment.end += diff(camera3dSegment.end); + // Keyframe times are relative to the segment start, so they + // have to follow the segment's new length rather than the + // absolute shift the other tracks use. + const nextDuration = camera3dSegment.end - camera3dSegment.start; + if (previousDuration <= 0 || nextDuration === previousDuration) + continue; + scaleKeyframeTimes( + camera3dSegment.tracks, + nextDuration / previousDuration, + ); + } }), ); }, @@ -1188,6 +1404,8 @@ export const [EditorContextProvider, useEditorContext] = createContextProvider( (project.timeline?.captionSegments?.length ?? 0) > 0; const initialKeyboardTrackVisible = project.keyboard?.settings.enabled ?? false; + const initialCamera3DTrackVisible = + (project.timeline?.camera3dSegments?.length ?? 0) > 0; const [editorState, setEditorState] = createStore({ previewTime: null as number | null, @@ -1217,7 +1435,8 @@ export const [EditorContextProvider, useEditorContext] = createContextProvider( | { type: "caption"; indices: number[] } | { type: "keyboard"; indices: number[] } | { type: "text"; indices: number[] } - | { type: "audio"; indices: number[] }, + | { type: "audio"; indices: number[] } + | { type: "3d"; indices: number[] }, transform: { // visible seconds zoom: zoomOutLimit(), @@ -1260,6 +1479,7 @@ export const [EditorContextProvider, useEditorContext] = createContextProvider( keyboard: initialKeyboardTrackVisible, zoom: true, scene: true, + "3d": initialCamera3DTrackVisible, mask: initialMaskTrackCount, text: initialTextTrackCount, audio: initialAudioTrackCount, @@ -1269,6 +1489,9 @@ export const [EditorContextProvider, useEditorContext] = createContextProvider( hoveredMaskTime: null as number | null, audioPicker: null as number | null, audioReplace: null as number | null, + // The empty 3D track's setup flow: the scene and shot count the + // sidebar is currently offering, previewed live on the track. + camera3dSetup: null as null | { sceneId: string; shots: number }, // Index of a just-created text segment that should open its // inline canvas editor as soon as its overlay mounts (set by the // Add-track picker, consumed by TextOverlay). @@ -1478,6 +1701,7 @@ export const [EditorContextProvider, useEditorContext] = createContextProvider( project, setProject, projectActions, + camera3DScenePreview, projectHistory: createStoreHistory(project, setProject), editorState, setEditorState, diff --git a/apps/desktop/src/routes/editor/three-d.test.ts b/apps/desktop/src/routes/editor/three-d.test.ts new file mode 100644 index 00000000000..0146d9945b6 --- /dev/null +++ b/apps/desktop/src/routes/editor/three-d.test.ts @@ -0,0 +1,1064 @@ +import { describe, expect, it } from "vitest"; + +import { + ANGLE_PRESETS, + anglePresetMotion, + anglePresetPose, + applyMotionTemplate, + applySceneToRange, + bezierEase, + CAMERA3D_ANGLE_PRESET_KEYS, + CAMERA3D_MIN_SHOT_DURATION, + CAMERA3D_PROPERTY_KEYS, + CAMERA3D_SCENES, + CAMERA3D_TRACK_KEYS, + type Camera3DKeyframe, + type Camera3DProperties, + type Camera3DScene, + type Camera3DSegment, + camera3DPosesEqual, + DEFAULT_IN_EASING, + DEFAULT_OUT_EASING, + defaultCamera3DSegment, + defaultCamera3DTracks, + evaluatePose, + fitCamera3DMotionToSegment, + flipCamera3DSegment, + getEndPose, + getMotionEasing, + getStartPose, + hasCamera3DMotion, + LINEAR_MOTION_EASING, + MOTION_EASINGS, + MOTION_TEMPLATES, + normalizeCamera3DSegments, + sampleTrack, + sceneWithShotCount, + setMotion, + showcaseCamera3DBlur, +} from "./three-d"; + +const key = ( + time: number, + value: number, + handles?: Partial>, +): Camera3DKeyframe => ({ + time, + value, + outEasing: handles?.outEasing ?? null, + inEasing: handles?.inEasing ?? null, +}); + +const poseWith = ( + segment: Camera3DSegment, + overrides: Partial, +): Camera3DProperties => ({ ...getStartPose(segment), ...overrides }); + +// Reading a pose back off a track is a lerp landing exactly on an endpoint, so +// the recovered value can differ from the authored one in the last bit. +const expectPose = ( + actual: Camera3DProperties, + expected: Camera3DProperties, +) => { + for (const property of CAMERA3D_PROPERTY_KEYS) + expect(actual[property]).toBeCloseTo(expected[property], 9); +}; + +describe("sampleTrack", () => { + it("returns the base value for an empty track", () => { + expect(sampleTrack(7, [], 0)).toBe(7); + expect(sampleTrack(7, [], 10)).toBe(7); + }); + + it("holds before the first and after the last keyframe", () => { + const track = [key(1, 10), key(2, 20)]; + + expect(sampleTrack(0, track, -5)).toBe(10); + expect(sampleTrack(0, track, 1)).toBe(10); + expect(sampleTrack(0, track, 2)).toBe(20); + expect(sampleTrack(0, track, 99)).toBe(20); + }); + + it("sorts unsorted keyframes before sampling", () => { + const track = [key(2, 20), key(1, 10)]; + + expect(sampleTrack(0, track, 0)).toBe(10); + expect(sampleTrack(0, track, 3)).toBe(20); + }); + + it("interpolates linearly with explicit [0,0] / [1,1] handles", () => { + const track = [ + key(0, 0, { outEasing: [0, 0] }), + key(1, 100, { inEasing: [1, 1] }), + ]; + + expect(sampleTrack(0, track, 0.25)).toBeCloseTo(25, 6); + expect(sampleTrack(0, track, 0.5)).toBeCloseTo(50, 6); + expect(sampleTrack(0, track, 0.75)).toBeCloseTo(75, 6); + }); + + it("falls back to cubic ease in out when handles are absent", () => { + const track = [key(0, 0), key(1, 100)]; + + // cubic-bezier(0.65, 0, 0.35, 1) is symmetric: the midpoint is exact and + // the quarter points bracket the linear values. + expect(sampleTrack(0, track, 0.5)).toBeCloseTo(50, 4); + expect(sampleTrack(0, track, 0.25)).toBeLessThan(25); + expect(sampleTrack(0, track, 0.75)).toBeGreaterThan(75); + }); + + it("matches an explicit default-handle pair exactly", () => { + const implicit = [key(0, 0), key(1, 100)]; + const explicit = [ + key(0, 0, { outEasing: [...DEFAULT_OUT_EASING] }), + key(1, 100, { inEasing: [...DEFAULT_IN_EASING] }), + ]; + + for (const t of [0.1, 0.33, 0.5, 0.8]) { + expect(sampleTrack(0, implicit, t)).toBeCloseTo( + sampleTrack(0, explicit, t), + 10, + ); + } + }); + + it("reads the split handles from either side of the span", () => { + const track = [ + key(0, 0, { outEasing: [0, 0] }), + key(1, 100, { inEasing: [1, 1] }), + key(2, 0), + ]; + + // First span is explicitly linear, second falls back to the cubic ease. + expect(sampleTrack(0, track, 0.5)).toBeCloseTo(50, 6); + expect(sampleTrack(0, track, 1.5)).toBeCloseTo(50, 4); + }); +}); + +describe("bezierEase", () => { + it("clamps outside the unit interval", () => { + expect(bezierEase([0.65, 0], [0.35, 1], -1)).toBe(0); + expect(bezierEase([0.65, 0], [0.35, 1], 2)).toBe(1); + }); + + it("takes the linear fast path for [0,0] / [1,1]", () => { + expect(bezierEase([0, 0], [1, 1], 0.31)).toBe(0.31); + }); +}); + +describe("setMotion", () => { + it("collapses equal poses to a still shot with no keyframes", () => { + const segment = defaultCamera3DSegment(2, 6); + const pose = poseWith(segment, { zoom: 1.4, tiltY: 12 }); + + setMotion(segment, pose, { ...pose }); + + for (const property of CAMERA3D_PROPERTY_KEYS) { + expect(segment.tracks[property]).toEqual([]); + expect(segment.properties[property]).toBe(pose[property]); + } + expect(hasCamera3DMotion(segment)).toBe(false); + }); + + it("keys only the properties that actually move", () => { + const segment = defaultCamera3DSegment(2, 6); + const start = getStartPose(segment); + const end = { ...start, zoom: start.zoom + 1 }; + + setMotion(segment, start, end); + + expect(segment.tracks.zoom).toHaveLength(2); + expect(segment.tracks.zoom[0]).toMatchObject({ + time: 0, + value: start.zoom, + }); + expect(segment.tracks.zoom[1]).toMatchObject({ time: 4, value: end.zoom }); + expect(segment.tracks.tiltX).toEqual([]); + expect(hasCamera3DMotion(segment)).toBe(true); + }); + + it("round-trips through getStartPose / getEndPose", () => { + const segment = defaultCamera3DSegment(1, 5); + const start = poseWith(segment, { zoom: 0.8, panX: -0.4, fov: 30 }); + const end = poseWith(segment, { zoom: 2.2, panX: 0.5, fov: 60, roll: 8 }); + + setMotion(segment, start, end); + + expectPose(getStartPose(segment), start); + expectPose(getEndPose(segment), end); + }); + + it("recovers the poses of an old multi-keyframe segment and flattens them", () => { + const segment = defaultCamera3DSegment(0, 4); + // Three keyframes with a detour in the middle, as the old record-mode UX + // could author. + segment.tracks.zoom = [key(0, 1), key(1.5, 3), key(4, 2)]; + segment.tracks.panX = [key(0, -0.5), key(4, 0.5)]; + + const start = getStartPose(segment); + const end = getEndPose(segment); + expect(start.zoom).toBe(1); + expect(end.zoom).toBe(2); + expect(start.panX).toBe(-0.5); + expect(end.panX).toBe(0.5); + + setMotion(segment, start, end); + + expect(segment.tracks.zoom).toHaveLength(2); + expectPose(getStartPose(segment), start); + expectPose(getEndPose(segment), end); + }); + + it("never touches the blur config or its tracks", () => { + const segment = defaultCamera3DSegment(0, 4); + segment.tracks.blurStrength = [key(0, 5), key(4, 9)]; + const blur = { ...segment.blur }; + + setMotion( + segment, + getStartPose(segment), + poseWith(segment, { zoom: 3 }), + MOTION_EASINGS[1], + ); + + expect(segment.blur).toEqual(blur); + expect(segment.tracks.blurStrength).toEqual([key(0, 5), key(4, 9)]); + }); +}); + +describe("MOTION_EASINGS", () => { + it("defaults to linear", () => { + expect(LINEAR_MOTION_EASING).toBe(MOTION_EASINGS[0]); + expect(LINEAR_MOTION_EASING.out).toEqual([0, 0]); + expect(LINEAR_MOTION_EASING.in).toEqual([1, 1]); + expect(MOTION_EASINGS.map((easing) => easing.label)).toEqual([ + "Linear", + "Smooth", + "Ease in", + "Ease out", + ]); + }); + + it("writes and reads back every style", () => { + for (const easing of MOTION_EASINGS) { + const segment = defaultCamera3DSegment(0, 4); + const start = getStartPose(segment); + const end = { ...start, zoom: start.zoom + 1 }; + + setMotion(segment, start, end, easing); + + expect(segment.tracks.zoom[0].outEasing).toEqual(easing.out); + expect(segment.tracks.zoom[1].inEasing).toEqual(easing.in); + expect(getMotionEasing(segment)).toBe(easing); + } + }); + + it("reads a still shot and an unrecognised curve as linear", () => { + const still = defaultCamera3DSegment(0, 4); + expect(getMotionEasing(still)).toBe(LINEAR_MOTION_EASING); + + const custom = defaultCamera3DSegment(0, 4); + custom.tracks.zoom = [ + key(0, 1, { outEasing: [0.11, 0.22] }), + key(4, 2, { inEasing: [0.33, 0.44] }), + ]; + expect(getMotionEasing(custom)).toBe(LINEAR_MOTION_EASING); + }); +}); + +describe("camera3DPosesEqual", () => { + it("ignores differences finer than a still shot", () => { + const segment = defaultCamera3DSegment(0, 4); + const pose = getStartPose(segment); + + expect(camera3DPosesEqual(pose, { ...pose, zoom: pose.zoom + 1e-6 })).toBe( + true, + ); + expect(camera3DPosesEqual(pose, { ...pose, zoom: pose.zoom + 0.01 })).toBe( + false, + ); + }); +}); + +describe("ANGLE_PRESETS", () => { + it("ships the five angle presets", () => { + expect(ANGLE_PRESETS.map((preset) => preset.id)).toEqual([ + "spotlight", + "perspective", + "center", + "low-angle", + "close-up", + ]); + }); + + it("never writes the content plane rotations", () => { + for (const preset of ANGLE_PRESETS) { + expect(Object.keys(preset.values).sort()).toEqual( + [...CAMERA3D_ANGLE_PRESET_KEYS].sort(), + ); + expect(preset.values).not.toHaveProperty("rotateX"); + expect(preset.values).not.toHaveProperty("rotateY"); + } + }); + + it("every angle preset is a moving shot opening on its named pose", () => { + for (const preset of ANGLE_PRESETS) { + const motion = anglePresetMotion(preset); + expect(motion.from).toEqual(anglePresetPose(preset)); + // The drift only touches pose keys and actually moves something. + expect(Object.keys(preset.drift).length).toBeGreaterThan(0); + for (const key of Object.keys(preset.drift)) + expect(CAMERA3D_ANGLE_PRESET_KEYS).toContain(key); + expect(motion.to).not.toEqual(motion.from); + expect(motion.to.rotateX).toBe(0); + expect(motion.to.rotateY).toBe(0); + } + }); + + it("applying an angle preset writes keyframes like a motion template", () => { + const preset = ANGLE_PRESETS.find((p) => p.id === "perspective"); + expect(preset).toBeDefined(); + if (!preset) return; + + const segment = defaultCamera3DSegment(2, 6); + applyMotionTemplate(segment, anglePresetMotion(preset)); + + expect(segment.tracks.tiltY).toHaveLength(2); + expect(segment.tracks.tiltY[0]).toMatchObject({ time: 0, value: 26 }); + expect(segment.tracks.tiltY[1]).toMatchObject({ time: 4, value: 18 }); + expect(segment.tracks.zoom[1].value).toBeCloseTo(1.53); + expect(segment.transitionIn).toBe(0); + expect(segment.transitionOut).toBe(0); + }); +}); + +describe("MOTION_TEMPLATES", () => { + it("ships eight templates with unique ids", () => { + expect(MOTION_TEMPLATES).toHaveLength(8); + expect(new Set(MOTION_TEMPLATES.map((t) => t.id)).size).toBe(8); + }); + + it("writes two explicitly linear keyframes on every moving camera track", () => { + for (const template of MOTION_TEMPLATES) { + const segment = defaultCamera3DSegment(2, 6); + applyMotionTemplate(segment, template); + + expectPose(getStartPose(segment), template.from); + expectPose(getEndPose(segment), template.to); + expect(getMotionEasing(segment)).toBe(LINEAR_MOTION_EASING); + + for (const property of CAMERA3D_PROPERTY_KEYS) { + const track = segment.tracks[property]; + // A property the template holds still carries no keyframes at all. + if (template.from[property] === template.to[property]) { + expect(track).toEqual([]); + expect(segment.properties[property]).toBe(template.from[property]); + continue; + } + expect(track).toHaveLength(2); + expect(track[0].time).toBe(0); + expect(track[1].time).toBe(4); + expect(track[0].value).toBe(template.from[property]); + expect(track[1].value).toBe(template.to[property]); + expect(track[0].outEasing).toEqual([0, 0]); + expect(track[1].inEasing).toEqual([1, 1]); + } + + // Blur tracks are the user's, so a template must not touch them. + for (const trackKey of CAMERA3D_TRACK_KEYS) { + if ((CAMERA3D_PROPERTY_KEYS as readonly string[]).includes(trackKey)) + continue; + expect(segment.tracks[trackKey]).toEqual([]); + } + } + }); + + it("cuts into the move by clearing any transition the segment carried", () => { + const segment = defaultCamera3DSegment(0, 4); + segment.transitionIn = 0.3; + segment.transitionOut = 0.3; + + applyMotionTemplate(segment, MOTION_TEMPLATES[0]); + + expect(segment.transitionIn).toBe(0); + expect(segment.transitionOut).toBe(0); + }); + + it("samples linearly between the two poses", () => { + const template = MOTION_TEMPLATES[0]; + const segment = defaultCamera3DSegment(0, 4); + applyMotionTemplate(segment, template); + + expect(sampleTrack(0, segment.tracks.panX, 2)).toBeCloseTo( + (template.from.panX + template.to.panX) / 2, + 6, + ); + }); +}); + +describe("template blur", () => { + const cards = () => [ + ...ANGLE_PRESETS.map((preset) => anglePresetMotion(preset)), + ...MOTION_TEMPLATES, + ]; + + it("gives every card a complete radial look", () => { + for (const card of cards()) { + expect(card.blur.mode).toBe("radial"); + expect(card.blur.bokeh).toBe(true); + expect(card.blur.angle).toBe(0); + expect(card.blur.dirPosition).toBe(0.5); + expect(card.blur.strength).toBeGreaterThan(0); + } + }); + + it("uses the showcase defocus everywhere but the two authored exceptions", () => { + const detail = { + mode: "radial", + strength: 20, + falloff: 0.76, + focusX: 0.11, + focusY: 0.5, + focusSize: 0.18, + angle: 0, + dirPosition: 0.5, + bokeh: true, + }; + const overhead = { + mode: "radial", + strength: 18, + falloff: 0.72, + focusX: 0.03, + focusY: 0.36, + focusSize: 0.55, + angle: 0, + dirPosition: 0.5, + bokeh: true, + }; + + for (const card of cards()) { + if (card.id === "angle-close-up") expect(card.blur).toEqual(detail); + else if (card.id === "top-down") expect(card.blur).toEqual(overhead); + else expect(card.blur).toEqual(showcaseCamera3DBlur()); + } + }); + + it("writes the card's blur onto the segment, replacing whatever was there", () => { + const overhead = MOTION_TEMPLATES.find((t) => t.id === "top-down"); + expect(overhead).toBeDefined(); + if (!overhead) return; + + const segment = defaultCamera3DSegment(0, 4); + segment.blur = { ...segment.blur, mode: "directional", strength: 3 }; + + applyMotionTemplate(segment, overhead); + + expect(segment.blur).toEqual(overhead.blur); + // A copy, so editing the segment never mutates the template. + expect(segment.blur).not.toBe(overhead.blur); + }); +}); + +describe("CAMERA3D_SCENES", () => { + it("ships three scenes with unique ids and three shots each", () => { + expect(CAMERA3D_SCENES.map((scene) => scene.id)).toEqual([ + "showcase", + "product-tour", + "punch-in", + ]); + expect(new Set(CAMERA3D_SCENES.map((scene) => scene.id)).size).toBe(3); + for (const scene of CAMERA3D_SCENES) expect(scene.shots).toHaveLength(3); + }); + + it("gives every scene weights that fill its range and a blur per shot", () => { + for (const scene of CAMERA3D_SCENES) { + const total = scene.shots.reduce((sum, shot) => sum + shot.weight, 0); + expect(total).toBeCloseTo(1, 6); + for (const shot of scene.shots) { + expect(shot.weight).toBeGreaterThan(0); + expect(shot.blur.mode).toBe("radial"); + expect(shot.blur.bokeh).toBe(true); + } + } + }); + + it("matches the reference project's showcase values verbatim", () => { + const showcase = CAMERA3D_SCENES.find((scene) => scene.id === "showcase"); + expect(showcase).toBeDefined(); + if (!showcase) return; + + const [close, overhead, push] = showcase.shots; + + expect(close.weight).toBe(0.27); + expect(close.from).toEqual({ + tiltX: 26, + tiltY: -22, + roll: 1, + rotateX: 0, + rotateY: 0, + fov: 45, + zoom: 0.8, + panX: -0.3, + panY: -0.4, + }); + expect(close.to).toEqual({ ...close.from, tiltY: -27, panX: -0.36 }); + expect(close.blur).toEqual({ + mode: "radial", + strength: 20, + falloff: 0.76, + focusX: 0.11, + focusY: 0.5, + focusSize: 0.18, + angle: 0, + dirPosition: 0.5, + bokeh: true, + }); + + expect(overhead.weight).toBe(0.25); + expect(overhead.from).toEqual({ + tiltX: 24.8, + tiltY: 17.04, + roll: 0, + rotateX: -40, + rotateY: 18, + fov: 60, + zoom: 0.5, + panX: -0.065, + panY: -0.195, + }); + expect(overhead.to).toEqual({ + ...overhead.from, + tiltX: 34.19, + tiltY: 15.28, + rotateY: 9, + panX: -0.217, + panY: -0.476, + }); + expect(overhead.blur).toEqual({ + mode: "radial", + strength: 18, + falloff: 0.72, + focusX: 0.03, + focusY: 0.36, + focusSize: 0.55, + angle: 0, + dirPosition: 0.5, + bokeh: true, + }); + + expect(push.weight).toBe(0.48); + expect(push.from).toEqual({ + tiltX: 0, + tiltY: 0, + roll: 0, + rotateX: -14, + rotateY: 0, + fov: 45, + zoom: 0.715, + panX: 0, + panY: 0, + }); + expect(push.to).toEqual({ ...push.from, zoom: 1.6 }); + expect(push.blur).toEqual({ + mode: "radial", + strength: 19, + falloff: 0.67, + focusX: 0.37, + focusY: 0.52, + focusSize: 0.4, + angle: 0, + dirPosition: 0.5, + bokeh: true, + }); + }); + + it("builds the derived scenes out of the existing card looks", () => { + const tour = CAMERA3D_SCENES.find((scene) => scene.id === "product-tour"); + const punch = CAMERA3D_SCENES.find((scene) => scene.id === "punch-in"); + expect(tour).toBeDefined(); + expect(punch).toBeDefined(); + if (!tour || !punch) return; + + const template = (id: string) => { + const found = MOTION_TEMPLATES.find((t) => t.id === id); + expect(found).toBeDefined(); + return found; + }; + const preset = (id: string) => { + const found = ANGLE_PRESETS.find((p) => p.id === id); + expect(found).toBeDefined(); + return found ? anglePresetMotion(found) : undefined; + }; + + const expectShot = ( + shot: (typeof tour.shots)[number], + card: ReturnType, + weight: number, + ) => { + expect(card).toBeDefined(); + if (!card) return; + expect(shot.weight).toBe(weight); + expect(shot.from).toEqual(card.from); + expect(shot.to).toEqual(card.to); + expect(shot.blur).toEqual(card.blur); + }; + + expectShot(tour.shots[0], template("unfold"), 0.3); + expectShot(tour.shots[1], preset("perspective"), 0.3); + expectShot(tour.shots[2], preset("center"), 0.4); + + expectShot(punch.shots[0], preset("spotlight"), 0.3); + expectShot(punch.shots[1], preset("close-up"), 0.3); + expectShot(punch.shots[2], template("pull-back"), 0.4); + }); +}); + +describe("sceneWithShotCount", () => { + const showcase = CAMERA3D_SCENES[0]; + + it("keeps the leading shots and renormalizes their weights", () => { + const two = sceneWithShotCount(showcase, 2); + + expect(two.shots).toHaveLength(2); + expect(two.shots[0].weight).toBeCloseTo(0.27 / 0.52, 9); + expect(two.shots[1].weight).toBeCloseTo(0.25 / 0.52, 9); + expect(two.shots.reduce((sum, shot) => sum + shot.weight, 0)).toBeCloseTo( + 1, + 9, + ); + // Only the weight changes: the look and the move are the scene's. + expect(two.shots[0].from).toEqual(showcase.shots[0].from); + expect(two.shots[0].to).toEqual(showcase.shots[0].to); + expect(two.shots[1].blur).toEqual(showcase.shots[1].blur); + expect(two.id).toBe(showcase.id); + }); + + it("gives a single shot the whole range", () => { + const one = sceneWithShotCount(showcase, 1); + + expect(one.shots).toHaveLength(1); + expect(one.shots[0].weight).toBeCloseTo(1, 9); + expect(one.shots[0].from).toEqual(showcase.shots[0].from); + }); + + it("clamps a count outside the scene's shots", () => { + expect(sceneWithShotCount(showcase, 0).shots).toHaveLength(1); + expect(sceneWithShotCount(showcase, -3).shots).toHaveLength(1); + expect(sceneWithShotCount(showcase, 2.7).shots).toHaveLength(2); + expect(sceneWithShotCount(showcase, 3)).toBe(showcase); + expect(sceneWithShotCount(showcase, 99)).toBe(showcase); + }); + + it("never mutates the scene it truncates", () => { + const before = showcase.shots.map((shot) => shot.weight); + + sceneWithShotCount(showcase, 1); + sceneWithShotCount(showcase, 2); + + expect(showcase.shots.map((shot) => shot.weight)).toEqual(before); + }); +}); + +describe("applySceneToRange", () => { + const showcase = CAMERA3D_SCENES[0]; + + const boundariesOf = (segments: Camera3DSegment[]) => [ + segments[0].start, + ...segments.map((segment) => segment.end), + ]; + + it("splits a range by shot weight", () => { + const segments = applySceneToRange(showcase, 0, 10, []); + + expect(segments).toHaveLength(3); + const boundaries = boundariesOf(segments); + expect(boundaries[0]).toBeCloseTo(0, 9); + expect(boundaries[1]).toBeCloseTo(2.7, 9); + expect(boundaries[2]).toBeCloseTo(5.2, 9); + expect(boundaries[3]).toBeCloseTo(10, 9); + }); + + it("lays the shots end to end with no gaps, wherever the range starts", () => { + const segments = applySceneToRange(showcase, 4, 14, []); + + expect(segments[0].start).toBe(4); + expect(segments[segments.length - 1].end).toBe(14); + for (let index = 1; index < segments.length; index++) + expect(segments[index].start).toBe(segments[index - 1].end); + }); + + it("snaps an interior boundary onto a nearby clip cut", () => { + const segments = applySceneToRange(showcase, 0, 11.14, [3.03]); + + expect(segments).toHaveLength(3); + expect(segments[0].end).toBe(3.03); + expect(segments[1].start).toBe(3.03); + // The far boundary had no cut within reach, so it stays on its weight. + expect(segments[1].end).toBeCloseTo(11.14 * 0.52, 9); + }); + + it("ignores a cut further than the snap window away", () => { + // Window is 15% of the 10s range, and 6.9 is out of reach of both the + // 2.7 and the 5.2 boundary. + const segments = applySceneToRange(showcase, 0, 10, [6.9]); + + expect(segments[0].end).toBeCloseTo(2.7, 9); + expect(segments[1].end).toBeCloseTo(5.2, 9); + }); + + it("drops a snap that would leave a shot under the minimum", () => { + // Range fits three shots with almost nothing to spare, so the last + // boundary cannot travel to the cut even though it is within the window. + const segments = applySceneToRange(showcase, 0, 3.4, [2.45]); + + expect(segments).toHaveLength(3); + expect(segments[1].end).toBeCloseTo(2, 9); + for (const segment of segments) + expect(segment.end - segment.start).toBeGreaterThanOrEqual( + CAMERA3D_MIN_SHOT_DURATION - 1e-9, + ); + }); + + it("takes a snap that keeps every shot long enough", () => { + const segments = applySceneToRange(showcase, 0, 3.4, [2.3]); + + expect(segments[1].end).toBe(2.3); + }); + + it("drops trailing shots a short range cannot fit", () => { + const segments = applySceneToRange(showcase, 0, 2.5, []); + + expect(segments).toHaveLength(2); + expect(segments[0].start).toBe(0); + expect(segments[segments.length - 1].end).toBe(2.5); + for (const segment of segments) + expect(segment.end - segment.start).toBeGreaterThanOrEqual( + CAMERA3D_MIN_SHOT_DURATION - 1e-9, + ); + // The leading shots are kept, re-weighted onto the whole range. + expectPose(getStartPose(segments[0]), showcase.shots[0].from); + expectPose(getEndPose(segments[1]), showcase.shots[1].to); + }); + + it("never drops below one shot, and refuses an empty range", () => { + expect(applySceneToRange(showcase, 0, 0.4, [])).toHaveLength(1); + expect(applySceneToRange(showcase, 5, 5, [])).toEqual([]); + expect(applySceneToRange(showcase, 5, 1, [])).toEqual([]); + }); + + it("fills the range with a scene truncated to fewer shots", () => { + const two = applySceneToRange(sceneWithShotCount(showcase, 2), 0, 10, []); + + expect(two).toHaveLength(2); + expect(two[0].start).toBe(0); + expect(two[0].end).toBeCloseTo(10 * (0.27 / 0.52), 9); + expect(two[1].start).toBe(two[0].end); + expect(two[1].end).toBe(10); + expectPose(getStartPose(two[0]), showcase.shots[0].from); + expectPose(getEndPose(two[1]), showcase.shots[1].to); + + const one = applySceneToRange(sceneWithShotCount(showcase, 1), 0, 10, []); + + expect(one).toHaveLength(1); + expect(one[0].start).toBe(0); + expect(one[0].end).toBe(10); + expectPose(getStartPose(one[0]), showcase.shots[0].from); + expectPose(getEndPose(one[0]), showcase.shots[0].to); + }); + + it("still truncates a scene the range cannot fit", () => { + // Two shots asked for, one second of room: the fallback inside + // applySceneToRange drops the shot that would not read as a cut. + const segments = applySceneToRange( + sceneWithShotCount(showcase, 2), + 0, + 1.5, + [], + ); + + expect(segments).toHaveLength(1); + expect(segments[0].start).toBe(0); + expect(segments[0].end).toBe(1.5); + }); + + it("writes each shot's move, look and cut for every scene", () => { + for (const scene of CAMERA3D_SCENES) { + const segments = applySceneToRange(scene, 0, 30, []); + expect(segments).toHaveLength(scene.shots.length); + + segments.forEach((segment, index) => { + const shot = scene.shots[index]; + + expect(segment.enabled).toBe(true); + expect(segment.transitionIn).toBe(0); + expect(segment.transitionOut).toBe(0); + expect(segment.blur).toEqual(shot.blur); + // A copy: editing one shot must not touch the scene data. + expect(segment.blur).not.toBe(shot.blur); + + expectPose(getStartPose(segment), shot.from); + expectPose(getEndPose(segment), shot.to); + expect(hasCamera3DMotion(segment)).toBe( + !camera3DPosesEqual(shot.from, shot.to), + ); + if (hasCamera3DMotion(segment)) + expect(getMotionEasing(segment)).toBe(LINEAR_MOTION_EASING); + + // Motion always spans the shot it was written onto. + for (const property of CAMERA3D_PROPERTY_KEYS) { + const track = segment.tracks[property]; + if (track.length === 0) continue; + expect(track[0].time).toBe(0); + expect(track[1].time).toBeCloseTo(segment.end - segment.start, 9); + } + }); + } + }); + + it("holds a still shot as a plain pose", () => { + const still: Camera3DScene = { + id: "still", + name: "Still", + shots: [ + { + weight: 1, + from: anglePresetPose(ANGLE_PRESETS[0]), + to: anglePresetPose(ANGLE_PRESETS[0]), + blur: showcaseCamera3DBlur(), + }, + ], + }; + + const [segment] = applySceneToRange(still, 0, 5, []); + + expect(hasCamera3DMotion(segment)).toBe(false); + expectPose(getStartPose(segment), anglePresetPose(ANGLE_PRESETS[0])); + }); +}); + +describe("split continuity", () => { + // Mirrors projectActions.splitCamera3DSegment: both halves have to meet on + // the pose the original segment held at the cut. + const split = (segment: Camera3DSegment, time: number) => { + const startPose = getStartPose(segment); + const midPose = evaluatePose(segment, time); + const endPose = getEndPose(segment); + const easing = getMotionEasing(segment); + + const right: Camera3DSegment = { + ...segment, + start: segment.start + time, + properties: { ...segment.properties }, + blur: { ...segment.blur }, + tracks: defaultCamera3DTracks(), + }; + setMotion(right, midPose, endPose, easing); + + const left: Camera3DSegment = { + ...segment, + end: segment.start + time, + properties: { ...segment.properties }, + blur: { ...segment.blur }, + tracks: defaultCamera3DTracks(), + }; + setMotion(left, startPose, midPose, easing); + + return { left, right, midPose }; + }; + + it("keeps the pose at the cut identical on both halves", () => { + const segment = defaultCamera3DSegment(0, 4); + applyMotionTemplate(segment, MOTION_TEMPLATES[3]); + + const { left, right, midPose } = split(segment, 1.5); + + expectPose(getEndPose(left), midPose); + expectPose(getStartPose(right), midPose); + expectPose(getStartPose(left), getStartPose(segment)); + expectPose(getEndPose(right), getEndPose(segment)); + }); + + it("re-times both halves onto their own length", () => { + const segment = defaultCamera3DSegment(0, 4); + applyMotionTemplate(segment, MOTION_TEMPLATES[3]); + + const { left, right } = split(segment, 1.5); + + expect(left.tracks.zoom[1].time).toBeCloseTo(1.5, 10); + expect(right.tracks.zoom[1].time).toBeCloseTo(2.5, 10); + }); + + it("leaves a still shot still on both sides", () => { + const segment = defaultCamera3DSegment(0, 4); + + const { left, right } = split(segment, 2); + + expect(hasCamera3DMotion(left)).toBe(false); + expect(hasCamera3DMotion(right)).toBe(false); + expectPose(getStartPose(right), getStartPose(segment)); + }); +}); + +describe("normalizeCamera3DSegments", () => { + it("fills every default and gives each track an array", () => { + const [segment] = normalizeCamera3DSegments([{ start: 1, end: 3 }]); + + expect(segment.enabled).toBe(true); + expect(segment.properties.fov).toBe(45); + expect(segment.properties.zoom).toBe(2); + expect(segment.blur.mode).toBe("none"); + expect(segment.blur.focusX).toBe(0.37); + // Absent transitions mean a clean cut, matching the renderer's default. + expect(segment.transitionIn).toBe(0); + expect(segment.transitionOut).toBe(0); + for (const trackKey of CAMERA3D_TRACK_KEYS) + expect(segment.tracks[trackKey]).toEqual([]); + }); + + it("sorts keyframes and normalizes absent handles to null", () => { + const [segment] = normalizeCamera3DSegments([ + { + start: 0, + end: 2, + tracks: { + ...defaultCamera3DTracks(), + zoom: [ + { time: 1, value: 3 }, + { time: 0, value: 1 }, + ], + }, + }, + ]); + + // Sorted, and the stale 1s span is stretched onto the 2s segment by + // the fit-on-load invariant. + expect(segment.tracks.zoom.map((k) => k.time)).toEqual([0, 2]); + expect(segment.tracks.zoom[0].outEasing).toBeNull(); + expect(segment.tracks.zoom[0].inEasing).toBeNull(); + }); +}); + +describe("defaultCamera3DSegment", () => { + it("opens on the Angled pose so the 3D look is visible immediately", () => { + const segment = defaultCamera3DSegment(0, 5); + const angled = ANGLE_PRESETS.find((preset) => preset.id === "perspective"); + + expect(angled).toBeDefined(); + for (const property of CAMERA3D_ANGLE_PRESET_KEYS) + expect(segment.properties[property]).toBe(angled?.values[property]); + expect(segment.properties.rotateX).toBe(0); + expect(segment.properties.rotateY).toBe(0); + }); + + it("opens on the showcase blur so the cinematic look is on by default", () => { + expect(defaultCamera3DSegment(0, 5).blur).toEqual(showcaseCamera3DBlur()); + expect(showcaseCamera3DBlur()).toMatchObject({ + mode: "radial", + strength: 19, + falloff: 0.62, + bokeh: true, + focusX: 0.37, + focusY: 0.5, + focusSize: 0.4, + }); + }); + + it("is a still shot with no keyframes until something moves it", () => { + const segment = defaultCamera3DSegment(0, 5); + + expect(hasCamera3DMotion(segment)).toBe(false); + expectPose(getStartPose(segment), getEndPose(segment)); + }); + + it("cuts straight in and out, with no ramp either side", () => { + const segment = defaultCamera3DSegment(0, 5); + + expect(segment.transitionIn).toBe(0); + expect(segment.transitionOut).toBe(0); + }); +}); + +describe("fitCamera3DMotionToSegment", () => { + it("rescales track times onto the segment duration after a resize", () => { + const segment = defaultCamera3DSegment(0, 4); + setMotion(segment, getStartPose(segment), poseWith(segment, { zoom: 3 })); + + // A trim shortened the segment; keyframes still span the old 4s. + segment.end = 2.5; + fitCamera3DMotionToSegment(segment); + + expect(segment.tracks.zoom[1].time).toBeCloseTo(2.5, 9); + // Applying again is a no-op (safe during live drags). + fitCamera3DMotionToSegment(segment); + expect(segment.tracks.zoom[1].time).toBeCloseTo(2.5, 9); + }); + + it("repairs stale spans on load through normalize", () => { + const segment = defaultCamera3DSegment(0, 3.025); + segment.tracks.tiltY = [ + { time: 0, value: -22, outEasing: [0, 0], inEasing: null }, + { time: 4.8625, value: -27, outEasing: null, inEasing: [1, 1] }, + ]; + + const [normalized] = normalizeCamera3DSegments([segment]); + expect(normalized.tracks.tiltY[1].time).toBeCloseTo(3.025, 9); + }); + + it("leaves still segments alone", () => { + const segment = defaultCamera3DSegment(0, 4); + fitCamera3DMotionToSegment(segment); + for (const track of Object.values(segment.tracks)) + expect(track).toEqual([]); + }); +}); + +describe("flipCamera3DSegment", () => { + it("mirrors the pose family, tracks, and blur focus horizontally", () => { + const segment = defaultCamera3DSegment(0, 4); + setMotion( + segment, + poseWith(segment, { tiltY: 26, roll: 5, panX: 0.37, rotateY: 18 }), + poseWith(segment, { tiltY: 18, roll: 5, panX: 0.2, rotateY: 9 }), + ); + segment.blur.focusX = 0.11; + segment.blur.angle = 30; + + flipCamera3DSegment(segment, "horizontal"); + + expect(getStartPose(segment).tiltY).toBeCloseTo(-26, 9); + expect(getEndPose(segment).tiltY).toBeCloseTo(-18, 9); + expect(getStartPose(segment).panX).toBeCloseTo(-0.37, 9); + expect(getStartPose(segment).rotateY).toBeCloseTo(-18, 9); + expect(getStartPose(segment).roll).toBeCloseTo(-5, 9); + // Vertical family untouched (the default segment opens on Perspective). + expect(getStartPose(segment).tiltX).toBeCloseTo(-28, 9); + expect(segment.blur.focusX).toBeCloseTo(0.89, 9); + expect(segment.blur.angle).toBeCloseTo(150, 9); + + // Flipping twice restores the original. + flipCamera3DSegment(segment, "horizontal"); + expect(getStartPose(segment).tiltY).toBeCloseTo(26, 9); + expect(segment.blur.focusX).toBeCloseTo(0.11, 9); + expect(segment.blur.angle).toBeCloseTo(30, 9); + }); + + it("mirrors the vertical family and focus Y", () => { + const segment = defaultCamera3DSegment(0, 4); + setMotion( + segment, + poseWith(segment, { tiltX: -50, panY: -0.4, rotateX: -14 }), + poseWith(segment, { tiltX: -44, panY: -0.2, rotateX: -14 }), + ); + segment.blur.focusY = 0.36; + + flipCamera3DSegment(segment, "vertical"); + + expect(getStartPose(segment).tiltX).toBeCloseTo(50, 9); + expect(getEndPose(segment).tiltX).toBeCloseTo(44, 9); + expect(getStartPose(segment).panY).toBeCloseTo(0.4, 9); + expect(getStartPose(segment).rotateX).toBeCloseTo(14, 9); + // Horizontal family untouched. + expect(getStartPose(segment).tiltY).toBeCloseTo(26, 9); + expect(segment.blur.focusY).toBeCloseTo(0.64, 9); + }); +}); diff --git a/apps/desktop/src/routes/editor/three-d.ts b/apps/desktop/src/routes/editor/three-d.ts new file mode 100644 index 00000000000..e2fb93ccf44 --- /dev/null +++ b/apps/desktop/src/routes/editor/three-d.ts @@ -0,0 +1,1371 @@ +import type { + Camera3DBlurMode, + Camera3DBlur as RawCamera3DBlur, + Camera3DKeyframe as RawCamera3DKeyframe, + Camera3DProperties as RawCamera3DProperties, + Camera3DSegment as RawCamera3DSegment, + Camera3DTracks as RawCamera3DTracks, +} from "~/utils/tauri"; + +export type { Camera3DBlurMode }; + +/** + * Local, strict mirrors of the generated schema: every field is required so the + * editor never has to re-check for absent values while sampling or editing. + * `normalizeCamera3DSegments` is the one place the loose wire shape is filled in. + */ +export type Camera3DProperties = { + tiltX: number; + tiltY: number; + roll: number; + rotateX: number; + rotateY: number; + fov: number; + zoom: number; + panX: number; + panY: number; +}; + +export type Camera3DBlur = { + mode: Camera3DBlurMode; + strength: number; + falloff: number; + focusX: number; + focusY: number; + focusSize: number; + angle: number; + dirPosition: number; + bokeh: boolean; +}; + +export type Camera3DKeyframe = { + /** Seconds relative to the segment start. */ + time: number; + value: number; + /** Bezier P1 for the span leaving this keyframe. */ + outEasing: [number, number] | null; + /** Bezier P2 for the span entering this keyframe. */ + inEasing: [number, number] | null; +}; + +export type Camera3DPropertyKey = keyof Camera3DProperties; + +export type Camera3DBlurScalarKey = + | "strength" + | "falloff" + | "focusX" + | "focusY" + | "focusSize" + | "angle" + | "dirPosition"; + +export type Camera3DTrackKey = + | Camera3DPropertyKey + | "blurStrength" + | "blurFalloff" + | "blurFocusSize" + | "blurFocusX" + | "blurFocusY" + | "blurAngle" + | "blurDirPosition"; + +export type Camera3DTracks = Record; + +export type Camera3DSegment = { + start: number; + end: number; + enabled: boolean; + properties: Camera3DProperties; + blur: Camera3DBlur; + tracks: Camera3DTracks; + transitionIn: number; + transitionOut: number; +}; + +/** + * Segments cut straight into their pose: shots open already framed, and the + * renderer defaults to the same 0 (crates/rendering/src/camera3d.rs). + */ +export const DEFAULT_CAMERA3D_TRANSITION = 0; + +/** + * Handles the renderer falls back to when a keyframe carries none: cubic + * ease-in-out. Mirrors `DEFAULT_OUT_EASING`/`DEFAULT_IN_EASING` in + * crates/rendering/src/camera3d.rs. + */ +export const DEFAULT_OUT_EASING: [number, number] = [0.65, 0]; +export const DEFAULT_IN_EASING: [number, number] = [0.35, 1]; + +export const CAMERA3D_PROPERTY_KEYS = [ + "tiltX", + "tiltY", + "roll", + "rotateX", + "rotateY", + "fov", + "zoom", + "panX", + "panY", +] as const satisfies readonly Camera3DPropertyKey[]; + +export const CAMERA3D_BLUR_SCALAR_KEYS = [ + "strength", + "falloff", + "focusX", + "focusY", + "focusSize", + "angle", + "dirPosition", +] as const satisfies readonly Camera3DBlurScalarKey[]; + +export const CAMERA3D_TRACK_KEYS = [ + "tiltX", + "tiltY", + "roll", + "rotateX", + "rotateY", + "fov", + "zoom", + "panX", + "panY", + "blurStrength", + "blurFalloff", + "blurFocusSize", + "blurFocusX", + "blurFocusY", + "blurAngle", + "blurDirPosition", +] as const satisfies readonly Camera3DTrackKey[]; + +export type Camera3DLimit = { min: number; max: number; step: number }; + +export const CAMERA3D_LIMITS = { + tiltX: { min: -70, max: 70, step: 1 }, + tiltY: { min: -60, max: 60, step: 1 }, + roll: { min: -180, max: 180, step: 1 }, + rotateX: { min: -90, max: 90, step: 1 }, + rotateY: { min: -50, max: 50, step: 1 }, + fov: { min: 10, max: 100, step: 1 }, + zoom: { min: 0.5, max: 10, step: 0.05 }, + panX: { min: -3, max: 3, step: 0.01 }, + panY: { min: -3, max: 3, step: 0.01 }, +} satisfies Record; + +export const CAMERA3D_BLUR_LIMITS = { + strength: { min: 0, max: 60, step: 1 }, + falloff: { min: 0, max: 1, step: 0.01 }, + focusX: { min: 0, max: 1, step: 0.01 }, + focusY: { min: 0, max: 1, step: 0.01 }, + focusSize: { min: 0, max: 1, step: 0.01 }, + angle: { min: 0, max: 360, step: 1 }, + dirPosition: { min: 0, max: 1, step: 0.01 }, +} satisfies Record; + +export const CAMERA3D_BOKEH_MAX_STRENGTH = 20; + +/** Bokeh caps strength, and tilt shift narrows both the band and the angle. */ +export const camera3dBlurLimit = ( + key: Camera3DBlurScalarKey, + blur: Pick, +): Camera3DLimit => { + switch (key) { + case "strength": + return { + min: 0, + max: blur.bokeh ? CAMERA3D_BOKEH_MAX_STRENGTH : 60, + step: 1, + }; + case "focusSize": + return { + min: 0, + max: blur.mode === "tiltShift" ? 0.6 : 1, + step: 0.01, + }; + case "angle": + return { min: 0, max: blur.mode === "tiltShift" ? 180 : 360, step: 1 }; + default: + return CAMERA3D_BLUR_LIMITS[key]; + } +}; + +export const CAMERA3D_TRANSITION_LIMITS = { min: 0, max: 2, step: 0.05 }; + +const clamp = (value: number, min: number, max: number) => + Math.min(Math.max(value, min), max); + +export const defaultCamera3DProperties = (): Camera3DProperties => ({ + tiltX: 0, + tiltY: 0, + roll: 0, + rotateX: 0, + rotateY: 0, + fov: 45, + zoom: 2, + panX: 0, + panY: 0, +}); + +export const defaultCamera3DBlur = (): Camera3DBlur => ({ + mode: "none", + strength: 0, + falloff: 0, + focusX: 0.37, + focusY: 0.5, + focusSize: 0.5, + angle: 0, + dirPosition: 0.5, + bokeh: false, +}); + +/** + * Blur a new segment opens with: a showcase defocus, so the cinematic look is + * there before anyone opens the Blur section. + */ +export const showcaseCamera3DBlur = (): Camera3DBlur => ({ + mode: "radial", + strength: 19, + falloff: 0.62, + focusX: 0.37, + focusY: 0.5, + focusSize: 0.4, + angle: 0, + dirPosition: 0.5, + bokeh: true, +}); + +export const defaultCamera3DTracks = (): Camera3DTracks => ({ + tiltX: [], + tiltY: [], + roll: [], + rotateX: [], + rotateY: [], + fov: [], + zoom: [], + panX: [], + panY: [], + blurStrength: [], + blurFalloff: [], + blurFocusSize: [], + blurFocusX: [], + blurFocusY: [], + blurAngle: [], + blurDirPosition: [], +}); + +/** Re-seeded parameters when the blur mode changes. */ +export const CAMERA3D_BLUR_MODE_SEEDS: Record< + Camera3DBlurMode, + Partial> +> = { + none: {}, + radial: { focusX: 0.37, focusY: 0.5, focusSize: 0.5 }, + directional: { dirPosition: 0.5, angle: 0 }, + tiltShift: { focusSize: 0.1, focusY: 0.5, angle: 45 }, +}; + +// ----------------------------------------------------------------------------- +// Presets +// ----------------------------------------------------------------------------- + +/** + * The seven values an angle preset writes. Content-plane rotation + * (rotateX/rotateY) is deliberately excluded: presets reframe the camera and + * leave whatever fold the user has dialled in alone. + */ +export const CAMERA3D_ANGLE_PRESET_KEYS = [ + "tiltX", + "tiltY", + "roll", + "fov", + "zoom", + "panX", + "panY", +] as const; + +export type Camera3DAnglePresetKey = + (typeof CAMERA3D_ANGLE_PRESET_KEYS)[number]; + +export type Camera3DAnglePreset = { + id: string; + name: string; + values: Record; + /// Every card produces a moving shot: the shot opens on the named pose + /// (`values`) and drifts toward these overrides. + drift: Partial>; + /// The defocus the card writes alongside its move. A card is a complete + /// look, so clicking one never leaves the previous shot's blur behind. + blur: Camera3DBlur; +}; + +/** + * Detail's defocus: a tight, hard-edged focus spot over the close-up, taken + * from the reference project's opening shot. + */ +const detailCamera3DBlur = (): Camera3DBlur => ({ + mode: "radial", + strength: 20, + falloff: 0.76, + focusX: 0.11, + focusY: 0.5, + focusSize: 0.18, + angle: 0, + dirPosition: 0.5, + bokeh: true, +}); + +/** + * The overhead shot's defocus: a wide, soft band low and left, taken from the + * reference project's middle shot. + */ +const overheadCamera3DBlur = (): Camera3DBlur => ({ + mode: "radial", + strength: 18, + falloff: 0.72, + focusX: 0.03, + focusY: 0.36, + focusSize: 0.55, + angle: 0, + dirPosition: 0.5, + bokeh: true, +}); + +/// Cards carry the showcase defocus unless they were authored with their own. +const anglePreset = ( + preset: Omit & { blur?: Camera3DBlur }, +): Camera3DAnglePreset => ({ + ...preset, + blur: preset.blur ?? showcaseCamera3DBlur(), +}); + +export const ANGLE_PRESETS: Camera3DAnglePreset[] = [ + anglePreset({ + id: "spotlight", + name: "Spotlight", + values: { + tiltX: 0, + tiltY: 0, + roll: 0, + fov: 45, + zoom: 1.35, + panX: 0.39, + panY: -0.4, + }, + // Slow push in with a slight rise. + drift: { zoom: 1.22, panY: -0.34 }, + }), + anglePreset({ + id: "perspective", + name: "Perspective", + values: { + tiltX: -28, + tiltY: 26, + roll: 5, + fov: 45, + zoom: 1.59, + panX: 0.37, + panY: -0.15, + }, + // Orbit sweep. + drift: { tiltY: 18, zoom: 1.53 }, + }), + anglePreset({ + id: "center", + name: "Center", + values: { + tiltX: 0, + tiltY: 0, + roll: 0, + fov: 45, + zoom: 2, + panX: 0, + panY: 0, + }, + // Slow pull back. + drift: { zoom: 2.25 }, + }), + anglePreset({ + id: "low-angle", + name: "Low angle", + values: { + tiltX: -50, + tiltY: 1, + roll: 0, + fov: 45, + zoom: 1.5, + panX: 0, + panY: 0, + }, + // Low-angle rise. + drift: { tiltX: -44, panY: -0.12 }, + }), + anglePreset({ + id: "close-up", + name: "Close up", + values: { + tiltX: 26, + tiltY: -22, + roll: 1, + fov: 45, + zoom: 0.8, + panX: -0.3, + panY: -0.4, + }, + // Truck across the close-up. + drift: { tiltY: -27, panX: -0.36 }, + // The close-up is the one card that focuses tight and far left. + blur: detailCamera3DBlur(), + }), +]; + +export const anglePresetPose = ( + preset: Camera3DAnglePreset, +): Camera3DProperties => ({ + ...defaultCamera3DProperties(), + ...preset.values, +}); + +/// An angle preset as a motion template: opens on the named pose and drifts. +export const anglePresetMotion = ( + preset: Camera3DAnglePreset, +): Camera3DMotionTemplate => ({ + id: `angle-${preset.id}`, + name: preset.name, + from: anglePresetPose(preset), + to: { ...anglePresetPose(preset), ...preset.drift }, + blur: { ...preset.blur }, +}); + +// A slider cannot express anything finer than one step, so half a step is the +// tightest a pose can be "the same as" a preset and still be reachable. +const presetMatchEpsilon = (key: Camera3DAnglePresetKey) => + Math.max(CAMERA3D_LIMITS[key].step / 2, 1e-4); + +export const matchAnglePreset = ( + pose: Pick, +): string | null => + ANGLE_PRESETS.find((preset) => + CAMERA3D_ANGLE_PRESET_KEYS.every( + (key) => + Math.abs(pose[key] - preset.values[key]) <= presetMatchEpsilon(key), + ), + )?.id ?? null; + +export type Camera3DMotionTemplate = { + id: string; + name: string; + from: Camera3DProperties; + to: Camera3DProperties; + /// The defocus the template writes. Clicking a card is a complete look, so + /// the blur is part of the template rather than whatever was there before. + blur: Camera3DBlur; +}; + +const pose = (overrides: Partial): Camera3DProperties => ({ + ...defaultCamera3DProperties(), + ...overrides, +}); + +/// Templates carry the showcase defocus unless authored with their own. +const motionTemplate = ( + template: Omit & { blur?: Camera3DBlur }, +): Camera3DMotionTemplate => ({ + ...template, + blur: template.blur ?? showcaseCamera3DBlur(), +}); + +export const MOTION_TEMPLATES: Camera3DMotionTemplate[] = [ + motionTemplate({ + id: "glide-across", + name: "Glide across", + from: pose({ + tiltX: -46.65, + tiltY: 42.49, + rotateY: -20, + rotateX: -1, + zoom: 1.785, + fov: 24, + panX: 0.673, + panY: -0.133, + }), + to: pose({ + tiltX: -46.65, + tiltY: 42.49, + rotateY: -20, + rotateX: -1, + zoom: 1.785, + fov: 24, + panX: 0.054, + panY: -0.31, + }), + }), + motionTemplate({ + id: "drift-down", + name: "Drift down", + from: pose({ zoom: 0.8, fov: 45, panX: 0.536, panY: -0.452 }), + to: pose({ zoom: 0.8, fov: 45, panX: 0.544, panY: 0.5 }), + }), + motionTemplate({ + id: "rising-sweep", + name: "Rising sweep", + from: pose({ + tiltX: -57.83, + tiltY: -8.7, + rotateY: -16, + zoom: 1.51, + fov: 29, + panX: -0.634, + panY: -0.082, + }), + to: pose({ + tiltX: -46.65, + tiltY: -7.94, + rotateY: -16, + zoom: 1.51, + fov: 25, + panX: -0.613, + panY: -0.268, + }), + }), + motionTemplate({ + id: "pull-back", + name: "Pull back", + from: pose({ rotateX: -14, zoom: 0.715, fov: 45 }), + to: pose({ rotateX: -14, zoom: 2.1, fov: 45 }), + }), + motionTemplate({ + id: "top-down", + name: "Top down", + from: pose({ + tiltX: 24.8, + tiltY: 17.04, + rotateY: 18, + rotateX: -40, + zoom: 0.5, + fov: 60, + panX: -0.065, + panY: -0.195, + }), + to: pose({ + tiltX: 34.19, + tiltY: 15.28, + rotateY: 9, + rotateX: -40, + zoom: 0.5, + fov: 60, + panX: -0.217, + panY: -0.476, + }), + // Looking down at the plane wants a wider, softer band than the rest. + blur: overheadCamera3DBlur(), + }), + motionTemplate({ + id: "tilt-away", + name: "Tilt away", + from: pose({ rotateX: -5, zoom: 0.5, fov: 45 }), + to: pose({ rotateX: -21, zoom: 0.6, fov: 45 }), + }), + motionTemplate({ + id: "unfold", + name: "Unfold", + from: pose({ rotateX: -42.96, zoom: 2.05, fov: 31 }), + to: pose({ rotateX: -12.01, zoom: 2, fov: 31, panY: -0.179 }), + }), + motionTemplate({ + id: "slide-by", + name: "Slide by", + from: pose({ + tiltX: -30.29, + tiltY: 60, + rotateY: -24, + rotateX: -39, + zoom: 1.99, + fov: 13, + panX: 0.238, + panY: 0.135, + }), + to: pose({ + tiltX: -30.29, + tiltY: 60, + rotateY: -24, + rotateX: -39, + zoom: 1.99, + fov: 13, + panX: -0.204, + panY: 0.039, + }), + }), +]; + +export type Camera3DFlipAxis = "horizontal" | "vertical"; + +/** + * Mirrors the whole composition across the given axis while the content stays + * readable: negating the yaw/roll/pan family is exactly conjugating the + * camera by the mirror, so a "Close up" on the right becomes the identical + * shot on the left. Applies to both poses, every motion track, and the blur + * focus, so a moving shot keeps its drift. + */ +export const flipCamera3DSegment = ( + segment: Camera3DSegment, + axis: Camera3DFlipAxis, +) => { + // Roll negates on both axes: a mirror reverses in-plane rotation. + const negated: Camera3DPropertyKey[] = + axis === "horizontal" + ? ["tiltY", "rotateY", "roll", "panX"] + : ["tiltX", "rotateX", "roll", "panY"]; + for (const key of negated) { + segment.properties[key] = -segment.properties[key]; + for (const keyframe of segment.tracks[key]) + keyframe.value = -keyframe.value; + } + + const focusKey = axis === "horizontal" ? "focusX" : "focusY"; + const focusTrack = axis === "horizontal" ? "blurFocusX" : "blurFocusY"; + segment.blur[focusKey] = 1 - segment.blur[focusKey]; + for (const keyframe of segment.tracks[focusTrack]) + keyframe.value = 1 - keyframe.value; + + // Directional/band angles mirror too: (cos, sin) reflected across the + // axis is 180 - a horizontally, -a vertically. + const mirrorAngle = (value: number) => { + const flipped = axis === "horizontal" ? 180 - value : -value; + return ((flipped % 360) + 360) % 360; + }; + segment.blur.angle = mirrorAngle(segment.blur.angle); + for (const keyframe of segment.tracks.blurAngle) + keyframe.value = mirrorAngle(keyframe.value); +}; + +/** + * Replaces the segment's motion with the template's two poses. Linear is the + * explicit pair the template writes, so the move reads exactly as authored + * instead of picking up the default cubic ease. + * + * The template's defocus goes on with it: a card is one complete look, so what + * the preview promises is what the segment ends up holding. + */ +export const applyMotionTemplate = ( + segment: Camera3DSegment, + template: Camera3DMotionTemplate, +) => { + setMotion(segment, template.from, template.to, LINEAR_MOTION_EASING); + segment.blur = { ...template.blur }; + // A template owns the whole move, so the segment cuts in and out of it. + // Segments authored before the 0 default can still carry a ramp. + segment.transitionIn = 0; + segment.transitionOut = 0; +}; + +// ----------------------------------------------------------------------------- +// Scenes +// ----------------------------------------------------------------------------- + +/** + * One shot of a scene: a move, a look, and the share of the scene's time range + * it holds. Weights are relative, so a scene reads the same whatever length it + * is dropped onto. + */ +export type Camera3DSceneShot = { + /** Fraction of the scene's time range. */ + weight: number; + /** The card this shot was built from, when it came from one. */ + name?: string; + from: Camera3DProperties; + to: Camera3DProperties; + blur: Camera3DBlur; +}; + +/** A chained multi-shot sequence: one click, several cuts. */ +export type Camera3DScene = { + id: string; + name: string; + shots: Camera3DSceneShot[]; +}; + +const anglePresetById = (id: string) => + ANGLE_PRESETS.find((preset) => preset.id === id) ?? ANGLE_PRESETS[0]; + +const motionTemplateById = (id: string) => + MOTION_TEMPLATES.find((template) => template.id === id) ?? + MOTION_TEMPLATES[0]; + +/// A card's complete look as one shot of a scene. +const templateShot = ( + template: Camera3DMotionTemplate, + weight: number, +): Camera3DSceneShot => ({ + weight, + name: template.name, + from: { ...template.from }, + to: { ...template.to }, + blur: { ...template.blur }, +}); + +export const CAMERA3D_SCENES: Camera3DScene[] = [ + { + // Transcribed verbatim from the hand-built reference project: a tight + // close-up truck, a fold-down overhead sweep, then a long push in. + id: "showcase", + name: "Showcase", + shots: [ + { + weight: 0.27, + from: pose({ + tiltX: 26, + tiltY: -22, + roll: 1, + fov: 45, + zoom: 0.8, + panX: -0.3, + panY: -0.4, + }), + to: pose({ + tiltX: 26, + tiltY: -27, + roll: 1, + fov: 45, + zoom: 0.8, + panX: -0.36, + panY: -0.4, + }), + blur: detailCamera3DBlur(), + }, + { + weight: 0.25, + from: pose({ + tiltX: 24.8, + tiltY: 17.04, + rotateX: -40, + rotateY: 18, + fov: 60, + zoom: 0.5, + panX: -0.065, + panY: -0.195, + }), + to: pose({ + tiltX: 34.19, + tiltY: 15.28, + rotateX: -40, + rotateY: 9, + fov: 60, + zoom: 0.5, + panX: -0.217, + panY: -0.476, + }), + blur: overheadCamera3DBlur(), + }, + { + weight: 0.48, + from: pose({ rotateX: -14, fov: 45, zoom: 0.715 }), + to: pose({ rotateX: -14, fov: 45, zoom: 1.6 }), + blur: { + mode: "radial", + strength: 19, + falloff: 0.67, + focusX: 0.37, + focusY: 0.52, + focusSize: 0.4, + angle: 0, + dirPosition: 0.5, + bokeh: true, + }, + }, + ], + }, + { + // Reveal, orbit, settle: the shape a product walkthrough wants. + id: "product-tour", + name: "Product tour", + shots: [ + templateShot(motionTemplateById("unfold"), 0.3), + templateShot(anglePresetMotion(anglePresetById("perspective")), 0.3), + templateShot(anglePresetMotion(anglePresetById("center")), 0.4), + ], + }, + { + // Push in, hold on the detail, then release. + id: "punch-in", + name: "Punch in", + shots: [ + templateShot(anglePresetMotion(anglePresetById("spotlight")), 0.3), + templateShot(anglePresetMotion(anglePresetById("close-up")), 0.3), + templateShot(motionTemplateById("pull-back"), 0.4), + ], + }, +]; + +/** + * The scene's leading `count` shots with their weights renormalized, so a + * shorter sequence still fills the whole range it is laid onto. Asking for + * more shots than the scene holds simply returns the scene. + */ +export const sceneWithShotCount = ( + scene: Camera3DScene, + count: number, +): Camera3DScene => { + const kept = Math.min(Math.max(Math.floor(count), 1), scene.shots.length); + if (kept >= scene.shots.length) return scene; + + const shots = scene.shots.slice(0, kept); + const totalWeight = shots.reduce( + (sum, shot) => sum + Math.max(shot.weight, 0), + 0, + ); + return { + ...scene, + shots: shots.map((shot) => ({ + ...shot, + weight: + totalWeight > 0 + ? Math.max(shot.weight, 0) / totalWeight + : 1 / shots.length, + })), + }; +}; + +/** + * No generated shot is shorter than this: below a second a cut reads as a + * glitch rather than an edit. + */ +export const CAMERA3D_MIN_SHOT_DURATION = 1; + +/** + * How far a shot boundary will travel to land on a clip cut, as a fraction of + * the scene's range. Cutting the camera exactly where the footage cuts is what + * makes a generated sequence look authored. + */ +export const CAMERA3D_SCENE_SNAP_FRACTION = 0.15; + +const nearestValue = (values: number[], target: number) => { + let nearest: number | null = null; + let distance = Number.POSITIVE_INFINITY; + for (const value of values) { + const candidate = Math.abs(value - target); + if (candidate >= distance) continue; + distance = candidate; + nearest = value; + } + return nearest; +}; + +/** + * Lays a scene across [start, end] as a chain of segments. + * + * Boundaries come from the shot weights, then each interior one looks for a + * clip cut to sit on. A snap is dropped rather than forced when it would push a + * shot under the minimum, and a range too short for the whole scene simply gets + * its leading shots. + */ +export const applySceneToRange = ( + scene: Camera3DScene, + start: number, + end: number, + clipCuts: number[] = [], +): Camera3DSegment[] => { + const length = end - start; + if (!(length > 0) || scene.shots.length === 0) return []; + + const shots = scene.shots.slice( + 0, + Math.max( + 1, + Math.min( + scene.shots.length, + Math.floor(length / CAMERA3D_MIN_SHOT_DURATION), + ), + ), + ); + const totalWeight = shots.reduce( + (sum, shot) => sum + Math.max(shot.weight, 0), + 0, + ); + const share = (shot: Camera3DSceneShot) => + totalWeight > 0 ? Math.max(shot.weight, 0) / totalWeight : 1 / shots.length; + + const cuts = clipCuts.filter((cut) => cut > start && cut < end); + const snapWindow = length * CAMERA3D_SCENE_SNAP_FRACTION; + + const boundaries = [start]; + let cumulative = 0; + for (let index = 0; index < shots.length - 1; index++) { + cumulative += share(shots[index]); + // Every shot still to come needs its own minimum, so this boundary lives + // in whatever is left once they are reserved. + const min = boundaries[index] + CAMERA3D_MIN_SHOT_DURATION; + const max = end - (shots.length - 1 - index) * CAMERA3D_MIN_SHOT_DURATION; + const weighted = clamp(start + length * cumulative, min, max); + const cut = nearestValue(cuts, weighted); + boundaries.push( + cut !== null && + Math.abs(cut - weighted) <= snapWindow && + cut >= min && + cut <= max + ? cut + : weighted, + ); + } + boundaries.push(end); + + return shots.map((shot, index) => { + const segment: Camera3DSegment = { + start: boundaries[index], + end: boundaries[index + 1], + enabled: true, + properties: defaultCamera3DProperties(), + blur: { ...shot.blur }, + tracks: defaultCamera3DTracks(), + transitionIn: 0, + transitionOut: 0, + }; + setMotion(segment, shot.from, shot.to, LINEAR_MOTION_EASING); + return segment; + }); +}; + +/** Pose the "Reset camera" action writes: a canonical long lens. */ +export const CAMERA3D_RESET_POSE: Camera3DProperties = { + tiltX: 0, + tiltY: 0, + roll: 0, + rotateX: 0, + rotateY: 0, + fov: 24, + zoom: 4.5, + panX: 0, + panY: 0, +}; + +// ----------------------------------------------------------------------------- +// Motion easings +// ----------------------------------------------------------------------------- + +/** + * One named curve for the whole move: the pair of split bezier handles the two + * keyframes carry. There is no per-keyframe easing to pick any more, because a + * segment only ever holds one span. + */ +export type Camera3DMotionEasing = { + id: string; + label: string; + /** Bezier P1, written onto the start keyframe. */ + out: [number, number]; + /** Bezier P2, written onto the end keyframe. */ + in: [number, number]; +}; + +export const MOTION_EASINGS: Camera3DMotionEasing[] = [ + // Linear is the default: it is what every template is authored in. + { id: "linear", label: "Linear", out: [0, 0], in: [1, 1] }, + { id: "smooth", label: "Smooth", out: [0.65, 0], in: [0.35, 1] }, + { id: "easeIn", label: "Ease in", out: [0.32, 0], in: [1, 1] }, + { id: "easeOut", label: "Ease out", out: [0, 0], in: [0.68, 1] }, +]; + +export const LINEAR_MOTION_EASING = MOTION_EASINGS[0]; + +// ----------------------------------------------------------------------------- +// Normalization +// ----------------------------------------------------------------------------- + +export const normalizeCamera3DProperties = ( + properties?: RawCamera3DProperties | null, +): Camera3DProperties => { + const defaults = defaultCamera3DProperties(); + if (!properties) return defaults; + return { + tiltX: properties.tiltX ?? defaults.tiltX, + tiltY: properties.tiltY ?? defaults.tiltY, + roll: properties.roll ?? defaults.roll, + rotateX: properties.rotateX ?? defaults.rotateX, + rotateY: properties.rotateY ?? defaults.rotateY, + fov: properties.fov ?? defaults.fov, + zoom: properties.zoom ?? defaults.zoom, + panX: properties.panX ?? defaults.panX, + panY: properties.panY ?? defaults.panY, + }; +}; + +export const normalizeCamera3DBlur = ( + blur?: RawCamera3DBlur | null, +): Camera3DBlur => { + const defaults = defaultCamera3DBlur(); + if (!blur) return defaults; + return { + mode: blur.mode ?? defaults.mode, + strength: blur.strength ?? defaults.strength, + falloff: blur.falloff ?? defaults.falloff, + focusX: blur.focusX ?? defaults.focusX, + focusY: blur.focusY ?? defaults.focusY, + focusSize: blur.focusSize ?? defaults.focusSize, + angle: blur.angle ?? defaults.angle, + dirPosition: blur.dirPosition ?? defaults.dirPosition, + bokeh: blur.bokeh ?? defaults.bokeh, + }; +}; + +const normalizeKeyframes = ( + keyframes?: RawCamera3DKeyframe[] | null, +): Camera3DKeyframe[] => + (keyframes ?? []) + .map((keyframe) => ({ + time: keyframe.time, + value: keyframe.value, + outEasing: keyframe.outEasing ?? null, + inEasing: keyframe.inEasing ?? null, + })) + .sort((a, b) => a.time - b.time); + +export const normalizeCamera3DTracks = ( + tracks?: RawCamera3DTracks | null, +): Camera3DTracks => { + const normalized = defaultCamera3DTracks(); + if (!tracks) return normalized; + for (const key of CAMERA3D_TRACK_KEYS) { + normalized[key] = normalizeKeyframes(tracks[key]); + } + return normalized; +}; + +/** + * Motion always spans the whole segment in the start/end model, so after a + * resize (or when loading a project resized before this invariant existed) + * every track's keyframe times are rescaled onto the new duration. Repeated + * application is a fixpoint, so calling it during a live drag is safe. + */ +export const fitCamera3DMotionToSegment = (segment: Camera3DSegment) => { + const length = Math.max(segment.end - segment.start, 0); + let span = 0; + for (const key of CAMERA3D_TRACK_KEYS) + for (const keyframe of segment.tracks[key]) + span = Math.max(span, keyframe.time); + if (span < 1e-6 || Math.abs(span - length) < 1e-6) return; + const scale = length / span; + for (const key of CAMERA3D_TRACK_KEYS) + for (const keyframe of segment.tracks[key]) keyframe.time *= scale; +}; + +export const normalizeCamera3DSegments = ( + segments?: RawCamera3DSegment[] | null, +): Camera3DSegment[] => + (segments ?? []) + .map((segment) => { + const normalized = { + start: segment.start, + end: segment.end, + enabled: segment.enabled ?? true, + properties: normalizeCamera3DProperties(segment.properties), + blur: normalizeCamera3DBlur(segment.blur), + tracks: normalizeCamera3DTracks(segment.tracks), + transitionIn: segment.transitionIn ?? DEFAULT_CAMERA3D_TRANSITION, + transitionOut: segment.transitionOut ?? DEFAULT_CAMERA3D_TRANSITION, + }; + fitCamera3DMotionToSegment(normalized); + return normalized; + }) + .sort((a, b) => a.start - b.start || a.end - b.end); + +export const defaultCamera3DSegment = ( + start: number, + end: number, +): Camera3DSegment => { + const angled = + ANGLE_PRESETS.find((preset) => preset.id === "perspective") ?? + ANGLE_PRESETS[0]; + return { + start, + end, + enabled: true, + // New segments open on a real pose so the 3D look is visible immediately. + properties: anglePresetPose(angled), + blur: showcaseCamera3DBlur(), + tracks: defaultCamera3DTracks(), + transitionIn: DEFAULT_CAMERA3D_TRANSITION, + transitionOut: DEFAULT_CAMERA3D_TRANSITION, + }; +}; + +// ----------------------------------------------------------------------------- +// Sampling (mirrors crates/rendering/src/camera3d.rs) +// ----------------------------------------------------------------------------- + +const bezierComponent = (p1: number, p2: number, t: number) => { + const inv = 1 - t; + return 3 * inv * inv * t * p1 + 3 * inv * t * t * p2 + t * t * t; +}; + +const bezierDerivative = (p1: number, p2: number, t: number) => { + const a = 1 - 3 * p2 + 3 * p1; + const b = 3 * p2 - 6 * p1; + const c = 3 * p1; + return (3 * a * t + 2 * b) * t + c; +}; + +/** + * Cubic-bezier timing with control points P1/P2 (P0 = (0,0), P3 = (1,1)): + * 8 Newton iterations, then bisection. + */ +export const bezierEase = ( + p1: readonly [number, number], + p2: readonly [number, number], + t: number, +) => { + if (t <= 0) return 0; + if (t >= 1) return 1; + + const [x1, y1] = p1; + const [x2, y2] = p2; + // Linear fast path ([0,0]/[1,1] handles). + if (x1 === y1 && x2 === y2 && x1 === 0 && x2 === 1) return t; + + let guess = t; + for (let i = 0; i < 8; i++) { + const error = bezierComponent(x1, x2, guess) - t; + if (Math.abs(error) < 1e-5) return bezierComponent(y1, y2, guess); + const slope = bezierDerivative(x1, x2, guess); + if (Math.abs(slope) < 1e-6) break; + guess -= error / slope; + } + + let lo = 0; + let hi = 1; + guess = t; + for (let i = 0; i < 30; i++) { + const error = bezierComponent(x1, x2, guess) - t; + if (Math.abs(error) < 1e-5) break; + if (error > 0) hi = guess; + else lo = guess; + guess = (lo + hi) / 2; + } + return bezierComponent(y1, y2, guess); +}; + +const sortedByTime = (keys: Camera3DKeyframe[]) => { + for (let i = 1; i < keys.length; i++) { + if (keys[i].time < keys[i - 1].time) + return [...keys].sort((a, b) => a.time - b.time); + } + return keys; +}; + +/** + * Samples one per-property track: base value when empty, hold outside the keyed + * range, split-handle bezier lerp between neighbours. + */ +export const sampleTrack = ( + base: number, + keys: Camera3DKeyframe[] | null | undefined, + time: number, +) => { + const list = keys ?? []; + if (list.length === 0) return base; + + const sorted = sortedByTime(list); + + if (time <= sorted[0].time) return sorted[0].value; + + for (let i = 1; i < sorted.length; i++) { + const prev = sorted[i - 1]; + const next = sorted[i]; + if (time > next.time) continue; + const span = Math.max(next.time - prev.time, 1e-6); + const progress = clamp((time - prev.time) / span, 0, 1); + const eased = bezierEase( + prev.outEasing ?? DEFAULT_OUT_EASING, + next.inEasing ?? DEFAULT_IN_EASING, + progress, + ); + return prev.value + (next.value - prev.value) * eased; + } + + return sorted[sorted.length - 1].value; +}; + +export const evaluatePose = ( + segment: Camera3DSegment, + relativeTime: number, +): Camera3DProperties => { + const base = segment.properties; + const tracks = segment.tracks; + return { + tiltX: sampleTrack(base.tiltX, tracks.tiltX, relativeTime), + tiltY: sampleTrack(base.tiltY, tracks.tiltY, relativeTime), + roll: sampleTrack(base.roll, tracks.roll, relativeTime), + rotateX: sampleTrack(base.rotateX, tracks.rotateX, relativeTime), + rotateY: sampleTrack(base.rotateY, tracks.rotateY, relativeTime), + fov: sampleTrack(base.fov, tracks.fov, relativeTime), + zoom: sampleTrack(base.zoom, tracks.zoom, relativeTime), + panX: sampleTrack(base.panX, tracks.panX, relativeTime), + panY: sampleTrack(base.panY, tracks.panY, relativeTime), + }; +}; + +export const evaluateBlur = ( + segment: Camera3DSegment, + relativeTime: number, +): Camera3DBlur => { + const base = segment.blur; + const tracks = segment.tracks; + return { + mode: base.mode, + bokeh: base.bokeh, + strength: sampleTrack(base.strength, tracks.blurStrength, relativeTime), + falloff: sampleTrack(base.falloff, tracks.blurFalloff, relativeTime), + focusX: sampleTrack(base.focusX, tracks.blurFocusX, relativeTime), + focusY: sampleTrack(base.focusY, tracks.blurFocusY, relativeTime), + focusSize: sampleTrack(base.focusSize, tracks.blurFocusSize, relativeTime), + angle: sampleTrack(base.angle, tracks.blurAngle, relativeTime), + dirPosition: sampleTrack( + base.dirPosition, + tracks.blurDirPosition, + relativeTime, + ), + }; +}; + +// ----------------------------------------------------------------------------- +// Start pose / end pose +// ----------------------------------------------------------------------------- + +/** + * Below this a move is a hold: nothing a slider can express is finer, so two + * poses this close are stored as a still shot with no keyframes at all. + */ +export const MOTION_STILL_EPSILON = 1e-4; + +/** The pose the segment opens on, whatever form its tracks are stored in. */ +export const getStartPose = (segment: Camera3DSegment): Camera3DProperties => + evaluatePose(segment, 0); + +/** The pose the segment lands on, whatever form its tracks are stored in. */ +export const getEndPose = (segment: Camera3DSegment): Camera3DProperties => + evaluatePose(segment, Math.max(segment.end - segment.start, 0)); + +export const camera3DPosesEqual = ( + a: Camera3DProperties, + b: Camera3DProperties, +) => + CAMERA3D_PROPERTY_KEYS.every( + (key) => Math.abs(a[key] - b[key]) < MOTION_STILL_EPSILON, + ); + +/** Whether any camera property is animated. Blur is never keyed. */ +export const hasCamera3DMotion = (segment: Camera3DSegment) => + CAMERA3D_PROPERTY_KEYS.some((key) => segment.tracks[key].length > 0); + +/** + * The one way the editor authors camera animation: a segment is a start pose + * and an end pose, and this writes that pair into the per-property tracks the + * renderer reads. A property that does not move keeps no keyframes at all, so + * a still shot stores as a plain base pose. + * + * Blur is never touched here: it is segment-level and static. + * + * An older segment carrying a richer track flattens to this form on its first + * edit, which is intended: it flattens onto the poses it already opened and + * closed on. + */ +export const setMotion = ( + segment: Camera3DSegment, + start: Camera3DProperties, + end: Camera3DProperties, + easing: Camera3DMotionEasing = LINEAR_MOTION_EASING, +) => { + const length = Math.max(segment.end - segment.start, 0); + for (const key of CAMERA3D_PROPERTY_KEYS) { + const from = start[key]; + const to = end[key]; + // The base pose always carries the start: it is what shows when the track + // is empty, and what the renderer holds outside the keyed range. + segment.properties[key] = from; + if (Math.abs(from - to) < MOTION_STILL_EPSILON) { + segment.tracks[key] = []; + continue; + } + segment.tracks[key] = [ + { time: 0, value: from, outEasing: [...easing.out], inEasing: null }, + { time: length, value: to, outEasing: null, inEasing: [...easing.in] }, + ]; + } +}; + +const EASING_MATCH_EPSILON = 1e-3; + +const handlesMatch = ( + a: readonly [number, number], + b: readonly [number, number], +) => + Math.abs(a[0] - b[0]) <= EASING_MATCH_EPSILON && + Math.abs(a[1] - b[1]) <= EASING_MATCH_EPSILON; + +/** + * The curve the segment's move runs on, read back off its first animated camera + * track. Anything unrecognised (a hand-keyed segment from before this model) + * reads as Linear, and picking a style rewrites both handles anyway. + */ +export const getMotionEasing = ( + segment: Camera3DSegment, +): Camera3DMotionEasing => { + for (const key of CAMERA3D_PROPERTY_KEYS) { + const track = segment.tracks[key]; + if (track.length < 2) continue; + const out = track[0].outEasing ?? DEFAULT_OUT_EASING; + const into = track[track.length - 1].inEasing ?? DEFAULT_IN_EASING; + return ( + MOTION_EASINGS.find( + (easing) => + handlesMatch(easing.out, out) && handlesMatch(easing.in, into), + ) ?? LINEAR_MOTION_EASING + ); + } + return LINEAR_MOTION_EASING; +}; + +/** Retimes every track when the segment's length changes. */ +export const scaleKeyframeTimes = (tracks: Camera3DTracks, scale: number) => { + for (const key of CAMERA3D_TRACK_KEYS) { + for (const keyframe of tracks[key]) keyframe.time *= scale; + } +}; + +// ----------------------------------------------------------------------------- +// CSS 3D preview +// ----------------------------------------------------------------------------- + +/** + * Scale constant: the Flat preset (zoom 2, fov 45) lands at roughly 60% of the + * preview card, which leaves room for the poses that push in closer. + */ +export const CAMERA3D_PREVIEW_SCALE_K = 0.5; + +/** + * A CSS-3D preview of a pose. The perspective distance reproduces the field + * of view at this card height, the rotations run camera-then-plane in the + * renderer's order, and apparent size follows 1 / (zoom * tan(fov/2)) exactly + * like the real projection. Pan uses half the projected offset so the extreme + * poses stay legible inside a thumbnail. + */ +export const cssPreviewTransform = ( + props: Camera3DProperties, + containerHeightPx: number, +) => { + const tan = Math.tan((clamp(props.fov, 1, 179) * Math.PI) / 360); + const zoom = Math.max(props.zoom, 0.05); + const scale = CAMERA3D_PREVIEW_SCALE_K / (zoom * tan); + const offset = (containerHeightPx / 2) * scale; + const tx = props.panX * offset; + const ty = -props.panY * offset; + + return { + perspective: containerHeightPx / (2 * tan), + transform: [ + `translate(${tx.toFixed(2)}px, ${ty.toFixed(2)}px)`, + `rotateY(${props.tiltY}deg)`, + `rotateX(${-props.tiltX}deg)`, + `rotateZ(${props.roll}deg)`, + `rotateY(${props.rotateY}deg)`, + `rotateX(${-props.rotateX}deg)`, + `scale(${scale.toFixed(4)})`, + ].join(" "), + }; +}; diff --git a/apps/desktop/src/styles/theme.css b/apps/desktop/src/styles/theme.css index 11f488cd1b3..330b4fd1284 100644 --- a/apps/desktop/src/styles/theme.css +++ b/apps/desktop/src/styles/theme.css @@ -30,6 +30,7 @@ --track-mask: #d2444b; /* red */ --track-scene: #975cfa; /* purple */ --track-audio: var(--jade-9); /* green */ + --track-3d: #7c6ff0; /* violet */ } :root { diff --git a/apps/desktop/src/utils/tauri.ts b/apps/desktop/src/utils/tauri.ts index 923a834e7eb..7ad31e0951f 100644 --- a/apps/desktop/src/utils/tauri.ts +++ b/apps/desktop/src/utils/tauri.ts @@ -652,6 +652,114 @@ export type Camera = { hide: boolean; mirror: boolean; position: CameraPosition; * Overrides `position` when set. */ manualPosition: XY | null; size: number; zoomSize: number | null; rounding: number; shadow: number; advancedShadow: ShadowConfiguration | null; shape: CameraShape; roundingType: CornerStyle; scaleDuringZoom?: number; backgroundBlur?: BackgroundBlurConfig } +/** + * Screen-space focus blur applied over the composed frame while a 3d segment + * is active (a UV-mask variable blur, not a depth-of-field). `strength` is a + * blur radius in pixels at 1080p output height; the renderer scales it with + * output size so preview and export match. + */ +export type Camera3DBlur = { mode?: Camera3DBlurMode; strength?: number; +/** + * Widens and flattens the sharp-to-blurred transition (0..1). + */ +falloff?: number; +/** + * Focus center in screen UV (radial mode). + */ +focusX?: number; focusY?: number; +/** + * Sharp region size: radius (radial) or band width (tilt-shift). + */ +focusSize?: number; +/** + * Degrees; blur direction (directional) or band angle (tilt-shift). + */ +angle?: number; +/** + * Where the directional blur begins along its axis (0..1). + */ +dirPosition?: number; +/** + * Swaps the gaussian kernel for a ring-disc bokeh kernel with highlight + * gain. Strength is capped at 20 while enabled. + */ +bokeh?: boolean } +export type Camera3DBlurMode = "none" | "radial" | "directional" | "tiltShift" +/** + * One scalar keyframe on a per-property track. Interpolation between two + * keyframes is a linear value lerp with time remapped by a cubic bezier whose + * P1 comes from the left keyframe's `out_easing` and P2 from the right one's + * `in_easing` (a split-handle model). Absent handles default to + * cubic ease-in-out: P1 [0.65, 0], P2 [0.35, 1]. + */ +export type Camera3DKeyframe = { +/** + * Seconds relative to the segment start. + */ +time: number; value: number; +/** + * Bezier P1 for the track segment leaving this keyframe. + */ +outEasing?: [number, number] | null; +/** + * Bezier P2 for the track segment entering this keyframe. + */ +inEasing?: [number, number] | null } +/** + * A 3D camera pose for the composed content plane. Angles are degrees. + * + * Geometry: the content plane's longest side spans 2 world units, centered at + * the origin facing +Z. `tilt_x`/`tilt_y`/`roll` orbit the CAMERA (Euler YXZ, + * roll innermost); `rotate_x`/`rotate_y` rotate the CONTENT plane itself. + * `zoom` is the camera DISTANCE in world units (larger = further = smaller on + * screen); `fov` is the vertical field of view with no size compensation, so + * apparent size ∝ 1 / (zoom · tan(fov/2)). `pan_x`/`pan_y` truck the camera in + * its own plane (world units; +x moves the subject right, +y up). + */ +export type Camera3DProperties = { +/** + * Camera orbit pitch. + */ +tiltX?: number; +/** + * Camera orbit yaw. + */ +tiltY?: number; +/** + * Camera roll (innermost camera rotation). + */ +roll?: number; +/** + * Content plane pitch. + */ +rotateX?: number; +/** + * Content plane yaw. + */ +rotateY?: number; fov?: number; +/** + * Camera distance in world units. + */ +zoom?: number; panX?: number; panY?: number } +export type Camera3DSegment = { start: number; end: number; enabled?: boolean; +/** + * Base pose; per-property tracks override individual values. + */ +properties?: Camera3DProperties; blur?: Camera3DBlur; tracks?: Camera3DTracks; +/** + * Seconds to ease from the flat frame into the pose at the segment start. + */ +transitionIn?: number; +/** + * Seconds to ease back to the flat frame before the segment end. + */ +transitionOut?: number } +/** + * Per-property keyframe tracks. A property with an empty track holds the + * segment's base value; each track holds independently before its first and + * after its last keyframe. + */ +export type Camera3DTracks = { tiltX: Camera3DKeyframe[]; tiltY: Camera3DKeyframe[]; roll: Camera3DKeyframe[]; rotateX: Camera3DKeyframe[]; rotateY: Camera3DKeyframe[]; fov: Camera3DKeyframe[]; zoom: Camera3DKeyframe[]; panX: Camera3DKeyframe[]; panY: Camera3DKeyframe[]; blurStrength: Camera3DKeyframe[]; blurFalloff: Camera3DKeyframe[]; blurFocusSize: Camera3DKeyframe[]; blurFocusX: Camera3DKeyframe[]; blurFocusY: Camera3DKeyframe[]; blurAngle: Camera3DKeyframe[]; blurDirPosition: Camera3DKeyframe[] } export type CameraDeviceSettings = { width: number | null; height: number | null; frameRate: number | null } export type CameraFormatInfo = { width: number; height: number; frameRate: number } export type CameraInfo = { device_id: string; model_id: ModelIDType | null; display_name: string } @@ -959,7 +1067,7 @@ export type StudioRecordingStatus = { status: "InProgress" } | { status: "NeedsR export type SystemDiagnostics = { macosVersion: MacOSVersionInfo | null; availableEncoders: string[]; screenCaptureSupported: boolean; metalSupported: boolean; gpuName: string | null } export type TargetUnderCursor = { display_id: DisplayId | null; window: WindowUnderCursor | null } export type TextSegment = { start: number; end: number; track?: number; enabled?: boolean; content?: string; center?: XY; size?: XY; fontFamily?: string; fontSize?: number; fontWeight?: number; italic?: boolean; color?: string; fadeDuration?: number } -export type TimelineConfiguration = { segments: TimelineSegment[]; transitions: ClipTransition[]; zoomSegments: ZoomSegment[]; sceneSegments?: SceneSegment[]; maskSegments?: MaskSegment[]; textSegments?: TextSegment[]; captionSegments?: CaptionTrackSegment[]; keyboardSegments?: KeyboardTrackSegment[]; audioSegments?: AudioTrackSegment[] } +export type TimelineConfiguration = { segments: TimelineSegment[]; transitions: ClipTransition[]; zoomSegments: ZoomSegment[]; sceneSegments?: SceneSegment[]; maskSegments?: MaskSegment[]; textSegments?: TextSegment[]; captionSegments?: CaptionTrackSegment[]; keyboardSegments?: KeyboardTrackSegment[]; audioSegments?: AudioTrackSegment[]; camera3dSegments?: Camera3DSegment[] } export type TimelineSegment = { recordingSegment?: number; timescale: number; start: number; end: number; name?: string | null; speedAudioMode?: ClipSpeedAudioMode | null } export type TranscriptionEngine = "Whisper" | "Parakeet" export type Trigger = "screenshotTaken" | "studioRecordingFinished" | "instantRecordingFinished" | "recordingStarted" | "uploadCompleted" | "videoImported" | "recordingDeleted" diff --git a/crates/editor/examples/editor-playback-benchmark.rs b/crates/editor/examples/editor-playback-benchmark.rs index bc26b774e0a..0d65a297384 100644 --- a/crates/editor/examples/editor-playback-benchmark.rs +++ b/crates/editor/examples/editor-playback-benchmark.rs @@ -388,6 +388,7 @@ async fn load_recording( caption_segments: Vec::new(), keyboard_segments: Vec::new(), audio_segments: Vec::new(), + camera3d_segments: Vec::new(), }); } } diff --git a/crates/editor/examples/playback-pipeline-benchmark.rs b/crates/editor/examples/playback-pipeline-benchmark.rs index d966287ebea..5601ac3231a 100644 --- a/crates/editor/examples/playback-pipeline-benchmark.rs +++ b/crates/editor/examples/playback-pipeline-benchmark.rs @@ -314,6 +314,7 @@ async fn load_recording( caption_segments: Vec::new(), keyboard_segments: Vec::new(), audio_segments: Vec::new(), + camera3d_segments: Vec::new(), }); } } diff --git a/crates/editor/src/audio.rs b/crates/editor/src/audio.rs index 5ff83811ab7..a655886d66f 100644 --- a/crates/editor/src/audio.rs +++ b/crates/editor/src/audio.rs @@ -1941,6 +1941,7 @@ mod tests { caption_segments: Vec::new(), keyboard_segments: Vec::new(), audio_segments: Vec::new(), + camera3d_segments: Vec::new(), }), clips: vec![ ClipConfiguration { @@ -2074,6 +2075,7 @@ mod tests { caption_segments: Vec::new(), keyboard_segments: Vec::new(), audio_segments: Vec::new(), + camera3d_segments: Vec::new(), }), clips: vec![ClipConfiguration { index: 0, @@ -2159,6 +2161,7 @@ mod tests { caption_segments: Vec::new(), keyboard_segments: Vec::new(), audio_segments: Vec::new(), + camera3d_segments: Vec::new(), }), clips: vec![ClipConfiguration { index: 0, @@ -2270,6 +2273,7 @@ mod tests { caption_segments: Vec::new(), keyboard_segments: Vec::new(), audio_segments: Vec::new(), + camera3d_segments: Vec::new(), }), clips: vec![ ClipConfiguration { @@ -2366,6 +2370,7 @@ mod tests { caption_segments: Vec::new(), keyboard_segments: Vec::new(), audio_segments: Vec::new(), + camera3d_segments: Vec::new(), }), clips: vec![ClipConfiguration { index: 0, @@ -2405,6 +2410,7 @@ mod tests { caption_segments: Vec::new(), keyboard_segments: Vec::new(), audio_segments: Vec::new(), + camera3d_segments: Vec::new(), }), clips: vec![ClipConfiguration { index: 0, @@ -2545,6 +2551,7 @@ mod tests { caption_segments: Vec::new(), keyboard_segments: Vec::new(), audio_segments, + camera3d_segments: Vec::new(), }), clips: vec![ClipConfiguration { index: 0, diff --git a/crates/editor/src/editor_instance.rs b/crates/editor/src/editor_instance.rs index 9898c5cbe34..5214531101a 100644 --- a/crates/editor/src/editor_instance.rs +++ b/crates/editor/src/editor_instance.rs @@ -221,6 +221,7 @@ impl EditorInstance { caption_segments: Vec::new(), keyboard_segments: Vec::new(), audio_segments: Vec::new(), + camera3d_segments: Vec::new(), }); if let Err(e) = project.write(&recording_meta.project_path) { diff --git a/crates/export/src/lib.rs b/crates/export/src/lib.rs index 4706ee89c39..3ced3f0ca15 100644 --- a/crates/export/src/lib.rs +++ b/crates/export/src/lib.rs @@ -132,6 +132,7 @@ impl ExporterBuilder { caption_segments: Vec::new(), keyboard_segments: Vec::new(), audio_segments: Vec::new(), + camera3d_segments: Vec::new(), }); } } diff --git a/crates/project/src/configuration.rs b/crates/project/src/configuration.rs index 71e22b1ec22..1e5234c82db 100644 --- a/crates/project/src/configuration.rs +++ b/crates/project/src/configuration.rs @@ -1033,6 +1033,264 @@ pub struct SceneSegment { pub transition_out: f64, } +// Shots cut straight into the pose; easing in and out is opt-in. +fn default_camera3d_transition() -> f64 { + 0.0 +} + +/// A 3D camera pose for the composed content plane. Angles are degrees. +/// +/// Geometry: the content plane's longest side spans 2 world units, centered at +/// the origin facing +Z. `tilt_x`/`tilt_y`/`roll` orbit the CAMERA (Euler YXZ, +/// roll innermost); `rotate_x`/`rotate_y` rotate the CONTENT plane itself. +/// `zoom` is the camera DISTANCE in world units (larger = further = smaller on +/// screen); `fov` is the vertical field of view with no size compensation, so +/// apparent size ∝ 1 / (zoom · tan(fov/2)). `pan_x`/`pan_y` truck the camera in +/// its own plane (world units; +x moves the subject right, +y up). +#[derive(Type, Serialize, Deserialize, Clone, Copy, Debug, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct Camera3DProperties { + /// Camera orbit pitch. + #[serde(default)] + pub tilt_x: f64, + /// Camera orbit yaw. + #[serde(default)] + pub tilt_y: f64, + /// Camera roll (innermost camera rotation). + #[serde(default)] + pub roll: f64, + /// Content plane pitch. + #[serde(default)] + pub rotate_x: f64, + /// Content plane yaw. + #[serde(default)] + pub rotate_y: f64, + #[serde(default = "Camera3DProperties::default_fov")] + pub fov: f64, + /// Camera distance in world units. + #[serde(default = "Camera3DProperties::default_zoom")] + pub zoom: f64, + #[serde(default)] + pub pan_x: f64, + #[serde(default)] + pub pan_y: f64, +} + +impl Camera3DProperties { + fn default_fov() -> f64 { + 45.0 + } + + fn default_zoom() -> f64 { + 2.0 + } +} + +impl Default for Camera3DProperties { + fn default() -> Self { + Self { + tilt_x: 0.0, + tilt_y: 0.0, + roll: 0.0, + rotate_x: 0.0, + rotate_y: 0.0, + fov: Self::default_fov(), + zoom: Self::default_zoom(), + pan_x: 0.0, + pan_y: 0.0, + } + } +} + +#[derive(Type, Serialize, Deserialize, Clone, Copy, Debug, Default, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub enum Camera3DBlurMode { + #[default] + None, + Radial, + Directional, + TiltShift, +} + +/// Screen-space focus blur applied over the composed frame while a 3d segment +/// is active (a UV-mask variable blur, not a depth-of-field). `strength` is a +/// blur radius in pixels at 1080p output height; the renderer scales it with +/// output size so preview and export match. +#[derive(Type, Serialize, Deserialize, Clone, Copy, Debug, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct Camera3DBlur { + #[serde(default)] + pub mode: Camera3DBlurMode, + #[serde(default)] + pub strength: f64, + /// Widens and flattens the sharp-to-blurred transition (0..1). + #[serde(default)] + pub falloff: f64, + /// Focus center in screen UV (radial mode). + #[serde(default = "Camera3DBlur::default_focus_x")] + pub focus_x: f64, + #[serde(default = "Camera3DBlur::default_focus_y")] + pub focus_y: f64, + /// Sharp region size: radius (radial) or band width (tilt-shift). + #[serde(default = "Camera3DBlur::default_focus_size")] + pub focus_size: f64, + /// Degrees; blur direction (directional) or band angle (tilt-shift). + #[serde(default)] + pub angle: f64, + /// Where the directional blur begins along its axis (0..1). + #[serde(default = "Camera3DBlur::default_dir_position")] + pub dir_position: f64, + /// Swaps the gaussian kernel for a ring-disc bokeh kernel with highlight + /// gain. Strength is capped at 20 while enabled. + #[serde(default)] + pub bokeh: bool, +} + +impl Camera3DBlur { + fn default_focus_x() -> f64 { + 0.37 + } + + fn default_focus_y() -> f64 { + 0.5 + } + + fn default_focus_size() -> f64 { + 0.5 + } + + fn default_dir_position() -> f64 { + 0.5 + } +} + +impl Default for Camera3DBlur { + fn default() -> Self { + Self { + mode: Camera3DBlurMode::None, + strength: 0.0, + falloff: 0.0, + focus_x: Self::default_focus_x(), + focus_y: Self::default_focus_y(), + focus_size: Self::default_focus_size(), + angle: 0.0, + dir_position: Self::default_dir_position(), + bokeh: false, + } + } +} + +/// One scalar keyframe on a per-property track. Interpolation between two +/// keyframes is a linear value lerp with time remapped by a cubic bezier whose +/// P1 comes from the left keyframe's `out_easing` and P2 from the right one's +/// `in_easing` (a split-handle model). Absent handles default to cubic +/// ease-in-out: P1 [0.65, 0], P2 [0.35, 1]. +#[derive(Type, Serialize, Deserialize, Clone, Copy, Debug, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct Camera3DKeyframe { + /// Seconds relative to the segment start. + pub time: f64, + pub value: f64, + /// Bezier P1 for the track segment leaving this keyframe. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub out_easing: Option<[f64; 2]>, + /// Bezier P2 for the track segment entering this keyframe. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub in_easing: Option<[f64; 2]>, +} + +/// Per-property keyframe tracks. A property with an empty track holds the +/// segment's base value; each track holds independently before its first and +/// after its last keyframe. +#[derive(Type, Serialize, Deserialize, Clone, Debug, Default)] +#[serde(rename_all = "camelCase")] +pub struct Camera3DTracks { + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub tilt_x: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub tilt_y: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub roll: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub rotate_x: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub rotate_y: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub fov: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub zoom: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub pan_x: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub pan_y: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub blur_strength: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub blur_falloff: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub blur_focus_size: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub blur_focus_x: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub blur_focus_y: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub blur_angle: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub blur_dir_position: Vec, +} + +impl Camera3DTracks { + /// Every track, for whole-timeline operations like time remapping. + pub fn all_tracks_mut(&mut self) -> [&mut Vec; 16] { + [ + &mut self.tilt_x, + &mut self.tilt_y, + &mut self.roll, + &mut self.rotate_x, + &mut self.rotate_y, + &mut self.fov, + &mut self.zoom, + &mut self.pan_x, + &mut self.pan_y, + &mut self.blur_strength, + &mut self.blur_falloff, + &mut self.blur_focus_size, + &mut self.blur_focus_x, + &mut self.blur_focus_y, + &mut self.blur_angle, + &mut self.blur_dir_position, + ] + } +} + +#[derive(Type, Serialize, Deserialize, Clone, Debug)] +#[serde(rename_all = "camelCase")] +pub struct Camera3DSegment { + pub start: f64, + pub end: f64, + #[serde(default = "Camera3DSegment::default_enabled")] + pub enabled: bool, + /// Base pose; per-property tracks override individual values. + #[serde(default)] + pub properties: Camera3DProperties, + #[serde(default)] + pub blur: Camera3DBlur, + #[serde(default)] + pub tracks: Camera3DTracks, + /// Seconds to ease from the flat frame into the pose at the segment start. + #[serde(default = "default_camera3d_transition")] + pub transition_in: f64, + /// Seconds to ease back to the flat frame before the segment end. + #[serde(default = "default_camera3d_transition")] + pub transition_out: f64, +} + +impl Camera3DSegment { + fn default_enabled() -> bool { + true + } +} + /// A timeline-positioned audio clip (background music or imported audio). /// /// Unlike the recording's mic/system audio (which is keyed to recording clips), @@ -1126,6 +1384,10 @@ pub struct TimelineConfiguration { pub keyboard_segments: Vec, #[serde(default)] pub audio_segments: Vec, + // Explicit rename: the digit boundary makes rename_all's camelCase output + // easy to second-guess, and the editor TypeScript hardcodes this name. + #[serde(default, rename = "camera3dSegments")] + pub camera3d_segments: Vec, } #[derive(Clone, Copy, Debug)] @@ -1995,6 +2257,7 @@ mod tests { caption_segments: Vec::new(), keyboard_segments: Vec::new(), audio_segments: Vec::new(), + camera3d_segments: Vec::new(), } } @@ -2104,6 +2367,100 @@ mod tests { assert_eq!(timeline.duration(), 4.0); } + /// The editor TypeScript hardcodes these field names; the serialized + /// form is a compatibility contract, not an implementation detail. + #[test] + fn camera3d_segments_serialize_with_stable_field_names() { + let mut timeline = timeline_with_transitions(Vec::new()); + timeline.camera3d_segments = vec![Camera3DSegment { + start: 1.0, + end: 3.0, + enabled: true, + properties: Camera3DProperties { + tilt_x: -28.0, + tilt_y: 26.0, + roll: 5.0, + zoom: 1.59, + pan_x: 0.37, + pan_y: -0.15, + ..Default::default() + }, + blur: Camera3DBlur { + mode: Camera3DBlurMode::Radial, + strength: 19.0, + falloff: 0.62, + bokeh: true, + ..Default::default() + }, + tracks: Camera3DTracks { + zoom: vec![ + Camera3DKeyframe { + time: 0.0, + value: 0.715, + out_easing: Some([0.0, 0.0]), + in_easing: None, + }, + Camera3DKeyframe { + time: 2.0, + value: 2.1, + out_easing: None, + in_easing: Some([1.0, 1.0]), + }, + ], + ..Default::default() + }, + transition_in: 0.3, + transition_out: 0.3, + }]; + + let json = serde_json::to_value(&timeline).unwrap(); + let segment = &json["camera3dSegments"][0]; + assert_eq!(segment["start"], 1.0); + assert_eq!(segment["properties"]["tiltX"], -28.0); + assert_eq!(segment["properties"]["tiltY"], 26.0); + assert_eq!(segment["properties"]["panX"], 0.37); + assert_eq!(segment["properties"]["fov"], 45.0); + assert_eq!(segment["properties"]["rotateX"], 0.0); + assert_eq!(segment["blur"]["mode"], "radial"); + assert_eq!(segment["blur"]["strength"], 19.0); + assert_eq!(segment["blur"]["focusX"], 0.37); + assert_eq!(segment["blur"]["dirPosition"], 0.5); + assert_eq!(segment["blur"]["bokeh"], true); + assert_eq!(segment["tracks"]["zoom"][0]["time"], 0.0); + assert_eq!(segment["tracks"]["zoom"][0]["value"], 0.715); + assert_eq!(segment["tracks"]["zoom"][0]["outEasing"][1], 0.0); + assert!(segment["tracks"]["zoom"][0].get("inEasing").is_none()); + assert!(segment["tracks"].get("tiltX").is_none()); + assert_eq!(segment["transitionIn"], 0.3); + assert_eq!( + serde_json::to_value(Camera3DBlurMode::TiltShift).unwrap(), + serde_json::json!("tiltShift") + ); + + // Old configs without the field still load, and the loaded form + // round-trips. + let legacy: TimelineConfiguration = serde_json::from_value(serde_json::json!({ + "segments": [ + { "recordingSegment": 0, "timescale": 1.0, "start": 0.0, "end": 4.0 } + ], + "zoomSegments": [] + })) + .unwrap(); + assert!(legacy.camera3d_segments.is_empty()); + + let reloaded: TimelineConfiguration = serde_json::from_value(json).unwrap(); + assert_eq!(reloaded.camera3d_segments.len(), 1); + assert_eq!(reloaded.camera3d_segments[0].tracks.zoom.len(), 2); + assert_eq!( + reloaded.camera3d_segments[0].tracks.zoom[0].out_easing, + Some([0.0, 0.0]) + ); + assert_eq!( + reloaded.camera3d_segments[0].blur.mode, + Camera3DBlurMode::Radial + ); + } + #[test] fn transition_json_is_normalized_by_segment_index() { let timeline: TimelineConfiguration = serde_json::from_value(serde_json::json!({ @@ -2297,6 +2654,7 @@ mod tests { caption_segments: Vec::new(), keyboard_segments: Vec::new(), audio_segments: Vec::new(), + camera3d_segments: Vec::new(), }), ..Default::default() }; diff --git a/crates/recording/src/recovery.rs b/crates/recording/src/recovery.rs index a9bc3cf32a2..b754bac0188 100644 --- a/crates/recording/src/recovery.rs +++ b/crates/recording/src/recovery.rs @@ -1519,6 +1519,7 @@ impl RecoveryManager { caption_segments: Vec::new(), keyboard_segments: Vec::new(), audio_segments: Vec::new(), + camera3d_segments: Vec::new(), }); config diff --git a/crates/recording/src/studio_recording.rs b/crates/recording/src/studio_recording.rs index cacc071996f..1130576913d 100644 --- a/crates/recording/src/studio_recording.rs +++ b/crates/recording/src/studio_recording.rs @@ -1198,6 +1198,7 @@ async fn stop_recording( caption_segments: Vec::new(), keyboard_segments: Vec::new(), audio_segments: Vec::new(), + camera3d_segments: Vec::new(), }); } if let Some(clips) = clip_configs { diff --git a/crates/recording/src/track_heal.rs b/crates/recording/src/track_heal.rs index 00e3e889557..2534ec7cdaa 100644 --- a/crates/recording/src/track_heal.rs +++ b/crates/recording/src/track_heal.rs @@ -607,6 +607,17 @@ fn rescale_project_config(project_path: &Path, scales: &HashMap) { s.start *= scale; s.end *= scale; } + for s in &mut timeline.camera3d_segments { + s.start *= scale; + s.end *= scale; + s.transition_in *= scale; + s.transition_out *= scale; + for track in s.tracks.all_tracks_mut() { + for k in track.iter_mut() { + k.time *= scale; + } + } + } changed = true; } @@ -1059,6 +1070,7 @@ mod tests { caption_segments: vec![], keyboard_segments: vec![], audio_segments: vec![], + camera3d_segments: Vec::new(), }; let config = ProjectConfiguration { timeline: Some(timeline), @@ -1078,6 +1090,80 @@ mod tests { assert!((timeline.zoom_segments[0].end - 150.0 * scale).abs() < 1e-6); } + #[test] + fn config_rescale_scales_camera3d_segments_exactly() { + use cap_project::{ + Camera3DKeyframe, Camera3DSegment, Camera3DTracks, ProjectConfiguration, + TimelineConfiguration, TimelineSegment, + }; + + let dir = tempfile::tempdir().unwrap(); + let timeline = TimelineConfiguration { + segments: vec![TimelineSegment { + recording_clip: 0, + timescale: 1.0, + start: 0.0, + end: 202.220711, + name: None, + speed_audio_mode: None, + }], + transitions: Vec::new(), + zoom_segments: vec![], + scene_segments: vec![], + mask_segments: vec![], + text_segments: vec![], + caption_segments: vec![], + keyboard_segments: vec![], + audio_segments: vec![], + camera3d_segments: vec![Camera3DSegment { + start: 100.0, + end: 150.0, + enabled: true, + properties: Default::default(), + blur: Default::default(), + tracks: Camera3DTracks { + tilt_x: vec![ + Camera3DKeyframe { + time: 0.0, + value: 0.0, + out_easing: None, + in_easing: None, + }, + Camera3DKeyframe { + time: 10.0, + value: 15.0, + out_easing: None, + in_easing: None, + }, + ], + ..Default::default() + }, + transition_in: 0.5, + transition_out: 0.8, + }], + }; + let config = ProjectConfiguration { + timeline: Some(timeline), + ..Default::default() + }; + config.write(dir.path()).unwrap(); + + let scale = 0.4468; + let mut scales = HashMap::new(); + scales.insert(0u32, scale); + rescale_project_config(dir.path(), &scales); + + let reloaded = ProjectConfiguration::load(dir.path()).unwrap(); + let timeline = reloaded.timeline.unwrap(); + let segment = &timeline.camera3d_segments[0]; + assert!((segment.start - 100.0 * scale).abs() < 1e-6); + assert!((segment.end - 150.0 * scale).abs() < 1e-6); + assert!((segment.transition_in - 0.5 * scale).abs() < 1e-6); + assert!((segment.transition_out - 0.8 * scale).abs() < 1e-6); + assert!((segment.tracks.tilt_x[0].time - 0.0 * scale).abs() < 1e-6); + assert!((segment.tracks.tilt_x[1].time - 10.0 * scale).abs() < 1e-6); + } + #[test] fn config_rescale_leaves_overlays_alone_with_unhealed_clips() { use cap_project::{ @@ -1121,6 +1207,7 @@ mod tests { caption_segments: vec![], keyboard_segments: vec![], audio_segments: vec![], + camera3d_segments: Vec::new(), }; let config = ProjectConfiguration { timeline: Some(timeline), diff --git a/crates/rendering/src/camera3d.rs b/crates/rendering/src/camera3d.rs new file mode 100644 index 00000000000..080f41f606f --- /dev/null +++ b/crates/rendering/src/camera3d.rs @@ -0,0 +1,772 @@ +//! 3D camera transform for the composed content plane. +//! +//! Geometry: the content plane's longest side spans 2 world units, centered +//! at the origin facing +Z. The camera orbits via Euler YXZ (`tilt_y`, +//! `tilt_x`, `roll`), the plane itself rotates via `rotate_y`·`rotate_x`, +//! `zoom` is the camera distance, `pan_x`/`pan_y` truck the camera in its own +//! plane, and `fov` is the vertical field of view with no size compensation: +//! apparent size ∝ 1 / (zoom · tan(fov/2)). +//! +//! view(p) = T(pan_x, pan_y, -zoom) · R_cam · R_obj · p +//! R_cam = Ry(tilt_y)·Rx(tilt_x)·Rz(roll) R_obj = Ry(rotate_y)·Rx(rotate_x) +//! +//! Keyframes are per-property tracks with a split-handle bezier easing: the +//! segment between two keyframes lerps the value linearly with time remapped +//! by cubic-bezier(P1 = left.out_easing ?? [0.65, 0], +//! P2 = right.in_easing ?? [0.35, 1]). +//! +//! Cap addition: `transition_in`/`transition_out` ease the whole effect from +//! the flat frame (zoom at the exact fill distance, all angles zero) into the +//! pose, so a segment always enters and leaves gracefully. + +use cap_project::{Camera3DBlurMode, Camera3DKeyframe, Camera3DProperties, Camera3DSegment}; + +/// Default split handles: cubic ease-in-out. +const DEFAULT_OUT_EASING: [f64; 2] = [0.65, 0.0]; +const DEFAULT_IN_EASING: [f64; 2] = [0.35, 1.0]; + +/// Blur strength is authored in pixels at this output height and scaled by the +/// renderer so preview and export match (same convention as mask effects). +pub const CAMERA3D_BLUR_BASE_HEIGHT: f64 = 1080.0; + +/// Effective, fully-sampled 3D state for one frame. +#[derive(Clone, Copy, Debug)] +pub struct Camera3DFrame { + /// `None` when the frame is exactly flat (warp pass can be skipped). + pub pose: Option, + /// `None` when no blur should render this frame. + pub blur: Option, + /// Boundary ramp 0..1. Decorations that read wrong in 3D (the flat drop + /// shadow baked into the content) fade out with this. + pub activity: f64, +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Camera3DBlurFrame { + pub mode: Camera3DBlurMode, + /// Pixels at [`CAMERA3D_BLUR_BASE_HEIGHT`]; already boundary-ramped and + /// bokeh-capped. + pub strength: f64, + pub falloff: f64, + pub focus_x: f64, + pub focus_y: f64, + pub focus_size: f64, + /// Radians. + pub angle: f64, + pub dir_position: f64, + pub bokeh: bool, +} + +/// Samples the active 3d segment at `frame_time`, or `None` when no segment +/// covers the frame (or the sampled state is visually flat with no blur). +pub fn interpolate_camera3d( + frame_time: f64, + segments: &[Camera3DSegment], + aspect: f64, +) -> Option { + let segment = segments + .iter() + .filter(|s| s.enabled) + .find(|s| frame_time >= s.start && frame_time <= s.end)?; + + let rel = frame_time - segment.start; + let tracks = &segment.tracks; + let base = &segment.properties; + let blur_base = &segment.blur; + + let pose = Camera3DProperties { + tilt_x: sample_track(base.tilt_x, &tracks.tilt_x, rel), + tilt_y: sample_track(base.tilt_y, &tracks.tilt_y, rel), + roll: sample_track(base.roll, &tracks.roll, rel), + rotate_x: sample_track(base.rotate_x, &tracks.rotate_x, rel), + rotate_y: sample_track(base.rotate_y, &tracks.rotate_y, rel), + fov: sample_track(base.fov, &tracks.fov, rel), + zoom: sample_track(base.zoom, &tracks.zoom, rel), + pan_x: sample_track(base.pan_x, &tracks.pan_x, rel), + pan_y: sample_track(base.pan_y, &tracks.pan_y, rel), + }; + + let blur = Camera3DBlurFrame { + mode: blur_base.mode, + strength: sample_track(blur_base.strength, &tracks.blur_strength, rel), + falloff: sample_track(blur_base.falloff, &tracks.blur_falloff, rel), + focus_x: sample_track(blur_base.focus_x, &tracks.blur_focus_x, rel), + focus_y: sample_track(blur_base.focus_y, &tracks.blur_focus_y, rel), + focus_size: sample_track(blur_base.focus_size, &tracks.blur_focus_size, rel), + angle: sample_track(blur_base.angle, &tracks.blur_angle, rel).to_radians(), + dir_position: sample_track(blur_base.dir_position, &tracks.blur_dir_position, rel), + bokeh: blur_base.bokeh, + }; + + let activity = boundary_activity(segment, frame_time); + Some(effective_frame(&pose, &blur, activity, aspect)) +} + +fn effective_frame( + pose: &Camera3DProperties, + blur: &Camera3DBlurFrame, + activity: f64, + aspect: f64, +) -> Camera3DFrame { + let fov = pose.fov.clamp(1.0, 179.0); + let (_, hy) = plane_half_extents(aspect); + // The camera distance at which the plane exactly fills the frame — the + // "flat" end of the boundary ramp. + let fill_zoom = hy / (fov.to_radians() / 2.0).tan(); + + let eased = pose_from_activity(pose, activity, fill_zoom); + + let flat = eased.tilt_x.abs() < 1e-3 + && eased.tilt_y.abs() < 1e-3 + && eased.roll.abs() < 1e-3 + && eased.rotate_x.abs() < 1e-3 + && eased.rotate_y.abs() < 1e-3 + && eased.pan_x.abs() < 1e-4 + && eased.pan_y.abs() < 1e-4 + && (eased.zoom - fill_zoom).abs() < 1e-4; + + let mut strength = blur.strength.max(0.0) * activity; + if blur.bokeh { + strength = strength.min(20.0); + } else { + strength = strength.min(60.0); + } + let blur_active = + blur.mode != Camera3DBlurMode::None && strength >= 0.5 && strength.is_finite(); + + Camera3DFrame { + pose: (!flat).then_some(eased), + blur: blur_active.then_some(Camera3DBlurFrame { strength, ..*blur }), + activity: activity.clamp(0.0, 1.0), + } +} + +fn pose_from_activity( + pose: &Camera3DProperties, + activity: f64, + fill_zoom: f64, +) -> Camera3DProperties { + let a = activity.clamp(0.0, 1.0); + let zoom = pose.zoom.clamp(0.05, 100.0); + Camera3DProperties { + tilt_x: pose.tilt_x * a, + tilt_y: pose.tilt_y * a, + roll: pose.roll * a, + rotate_x: pose.rotate_x * a, + rotate_y: pose.rotate_y * a, + fov: pose.fov, + zoom: fill_zoom + (zoom - fill_zoom) * a, + pan_x: pose.pan_x * a, + pan_y: pose.pan_y * a, + } +} + +/// 0 at the segment edges, ramping to 1 over `transition_in`/`transition_out`. +fn boundary_activity(segment: &Camera3DSegment, frame_time: f64) -> f64 { + let len = (segment.end - segment.start).max(0.0); + let half = len / 2.0; + let t_in = segment.transition_in.clamp(0.0, half); + let t_out = segment.transition_out.clamp(0.0, half); + + let a_in = if t_in > 0.0 { + ((frame_time - segment.start) / t_in).clamp(0.0, 1.0) + } else { + 1.0 + }; + let a_out = if t_out > 0.0 { + ((segment.end - frame_time) / t_out).clamp(0.0, 1.0) + } else { + 1.0 + }; + + bezier_ease(DEFAULT_OUT_EASING, DEFAULT_IN_EASING, a_in) + * bezier_ease(DEFAULT_OUT_EASING, DEFAULT_IN_EASING, a_out) +} + +/// Samples one per-property track: base value when empty, hold outside the +/// keyed range, split-handle bezier lerp between neighbours. +pub fn sample_track(base: f64, keys: &[Camera3DKeyframe], time: f64) -> f64 { + if keys.is_empty() { + return base; + } + + let mut sorted = keys.to_vec(); + sorted.sort_by(|a, b| { + a.time + .partial_cmp(&b.time) + .unwrap_or(std::cmp::Ordering::Equal) + }); + + if time <= sorted[0].time { + return sorted[0].value; + } + + for window in sorted.windows(2) { + let prev = &window[0]; + let next = &window[1]; + if time <= next.time { + let span = (next.time - prev.time).max(1e-6); + let progress = ((time - prev.time) / span).clamp(0.0, 1.0); + let p1 = prev.out_easing.unwrap_or(DEFAULT_OUT_EASING); + let p2 = next.in_easing.unwrap_or(DEFAULT_IN_EASING); + let eased = bezier_ease(p1, p2, progress); + return prev.value + (next.value - prev.value) * eased; + } + } + + sorted.last().map(|k| k.value).unwrap_or(base) +} + +fn bezier_component(p1: f64, p2: f64, t: f64) -> f64 { + let inv = 1.0 - t; + 3.0 * inv * inv * t * p1 + 3.0 * inv * t * t * p2 + t * t * t +} + +fn bezier_derivative(p1: f64, p2: f64, t: f64) -> f64 { + let a = 1.0 - 3.0 * p2 + 3.0 * p1; + let b = 3.0 * p2 - 6.0 * p1; + let c = 3.0 * p1; + (3.0 * a * t + 2.0 * b) * t + c +} + +/// Cubic-bezier timing with control points P1/P2 (P0=(0,0), P3=(1,1)): +/// 8 Newton iterations then bisection. +pub fn bezier_ease(p1: [f64; 2], p2: [f64; 2], t: f64) -> f64 { + if t <= 0.0 { + return 0.0; + } + if t >= 1.0 { + return 1.0; + } + let [x1, y1] = p1; + let [x2, y2] = p2; + // Linear fast path ([0,0]/[1,1] handles). + if x1 == y1 && x2 == y2 && x1 == 0.0 && x2 == 1.0 { + return t; + } + + let mut guess = t; + for _ in 0..8 { + let error = bezier_component(x1, x2, guess) - t; + if error.abs() < 1e-5 { + return bezier_component(y1, y2, guess); + } + let slope = bezier_derivative(x1, x2, guess); + if slope.abs() < 1e-6 { + break; + } + guess -= error / slope; + } + + let mut lo = 0.0; + let mut hi = 1.0; + guess = t; + for _ in 0..30 { + let error = bezier_component(x1, x2, guess) - t; + if error.abs() < 1e-5 { + break; + } + if error > 0.0 { + hi = guess; + } else { + lo = guess; + } + guess = (lo + hi) / 2.0; + } + bezier_component(y1, y2, guess) +} + +/// Half-extents of the content plane: the longest side spans 2 world units. +pub fn plane_half_extents(aspect: f64) -> (f64, f64) { + let aspect = if aspect.is_finite() && aspect > 0.0 { + aspect + } else { + 1.0 + }; + if aspect >= 1.0 { + (1.0, 1.0 / aspect) + } else { + (aspect, 1.0) + } +} + +/// The 2D zoom while a 3D pose is active: magnify the card on screen by +/// `amount` about the zoom target (given as content-frame UV, y-down). The +/// target eases toward frame center as the magnification grows, matching the +/// flat zoom's framing. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Camera3DScreenZoom { + pub content_uv: cap_project::XY, + pub amount: f64, +} + +/// Rows of the 3×3 inverse homography mapping y-up NDC (x, y, 1) back to +/// plane coordinates (X·w, Y·w, w), plus the plane half-extents for UV +/// mapping in the shader. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Camera3DHomography { + pub inverse_rows: [[f32; 3]; 3], + pub half_extents: (f32, f32), +} + +type Mat3 = [[f64; 3]; 3]; + +fn mat3_mul(a: &Mat3, b: &Mat3) -> Mat3 { + let mut out = [[0.0; 3]; 3]; + for (i, out_row) in out.iter_mut().enumerate() { + for (j, cell) in out_row.iter_mut().enumerate() { + *cell = (0..3).map(|k| a[i][k] * b[k][j]).sum(); + } + } + out +} + +fn rot_x3(theta: f64) -> Mat3 { + let (s, c) = theta.sin_cos(); + [[1.0, 0.0, 0.0], [0.0, c, -s], [0.0, s, c]] +} + +fn rot_y3(theta: f64) -> Mat3 { + let (s, c) = theta.sin_cos(); + [[c, 0.0, s], [0.0, 1.0, 0.0], [-s, 0.0, c]] +} + +fn rot_z3(theta: f64) -> Mat3 { + let (s, c) = theta.sin_cos(); + [[c, -s, 0.0], [s, c, 0.0], [0.0, 0.0, 1.0]] +} + +/// Builds the inverse homography for a pose, or `None` when degenerate. +/// `screen_zoom` composes the 2D zoom as an NDC magnification of the card. +pub fn camera3d_inverse_homography( + props: &Camera3DProperties, + aspect: f64, + screen_zoom: Option<&Camera3DScreenZoom>, +) -> Option { + let aspect = if aspect.is_finite() && aspect > 0.0 { + aspect + } else { + 1.0 + }; + let (hx, hy) = plane_half_extents(aspect); + + let fov = props.fov.clamp(1.0, 179.0).to_radians(); + let f = 1.0 / (fov / 2.0).tan(); + let zoom = props.zoom.clamp(0.05, 100.0); + + let r_cam = mat3_mul( + &rot_y3(props.tilt_y.to_radians()), + &mat3_mul( + &rot_x3(props.tilt_x.to_radians()), + &rot_z3(props.roll.to_radians()), + ), + ); + let r_obj = mat3_mul( + &rot_y3(props.rotate_y.to_radians()), + &rot_x3(props.rotate_x.to_radians()), + ); + let r = mat3_mul(&r_cam, &r_obj); + + // view = R·(X, Y, 0) + (pan_x, pan_y, -zoom); homography on [X, Y, 1]: + let fx = f / aspect; + let mut forward = [ + [fx * r[0][0], fx * r[0][1], fx * props.pan_x], + [f * r[1][0], f * r[1][1], f * props.pan_y], + [-r[2][0], -r[2][1], zoom], + ]; + + if let Some(screen_zoom) = screen_zoom + && screen_zoom.amount > 1.001 + && screen_zoom.amount.is_finite() + { + // Project the zoom target (content UV, y-down) to NDC through the + // pose, then scale about it. The fixed point slides toward frame + // center as `q / amount`, mirroring the flat zoom's centering. + let plane_x = (2.0 * screen_zoom.content_uv.x - 1.0) * hx; + let plane_y = (1.0 - 2.0 * screen_zoom.content_uv.y) * hy; + let w = forward[2][0] * plane_x + forward[2][1] * plane_y + forward[2][2]; + if w > 1e-6 { + let qx = (forward[0][0] * plane_x + forward[0][1] * plane_y + forward[0][2]) / w; + let qy = (forward[1][0] * plane_x + forward[1][1] * plane_y + forward[1][2]) / w; + let amount = screen_zoom.amount; + let recenter = amount - 1.0 / amount; + let magnify = [ + [amount, 0.0, -qx * recenter], + [0.0, amount, -qy * recenter], + [0.0, 0.0, 1.0], + ]; + forward = mat3_mul(&magnify, &forward); + } + } + + invert3(&forward).map(|inv| Camera3DHomography { + inverse_rows: inv.map(|row| row.map(|v| v as f32)), + half_extents: (hx as f32, hy as f32), + }) +} + +fn invert3(m: &[[f64; 3]; 3]) -> Option<[[f64; 3]; 3]> { + let c00 = m[1][1] * m[2][2] - m[1][2] * m[2][1]; + let c01 = m[1][2] * m[2][0] - m[1][0] * m[2][2]; + let c02 = m[1][0] * m[2][1] - m[1][1] * m[2][0]; + + let det = m[0][0] * c00 + m[0][1] * c01 + m[0][2] * c02; + if !det.is_finite() || det.abs() < 1e-12 { + return None; + } + let inv_det = 1.0 / det; + + Some([ + [ + c00 * inv_det, + (m[0][2] * m[2][1] - m[0][1] * m[2][2]) * inv_det, + (m[0][1] * m[1][2] - m[0][2] * m[1][1]) * inv_det, + ], + [ + c01 * inv_det, + (m[0][0] * m[2][2] - m[0][2] * m[2][0]) * inv_det, + (m[0][2] * m[1][0] - m[0][0] * m[1][2]) * inv_det, + ], + [ + c02 * inv_det, + (m[0][1] * m[2][0] - m[0][0] * m[2][1]) * inv_det, + (m[0][0] * m[1][1] - m[0][1] * m[1][0]) * inv_det, + ], + ]) +} + +#[cfg(test)] +mod tests { + use super::*; + use cap_project::{Camera3DBlur, Camera3DTracks}; + + const ASPECT: f64 = 16.0 / 9.0; + const HY: f64 = 9.0 / 16.0; + + fn apply3(rows: &[[f32; 3]; 3], v: [f64; 3]) -> [f64; 3] { + let mut out = [0.0; 3]; + for (i, row) in rows.iter().enumerate() { + out[i] = row.iter().zip(v.iter()).map(|(a, b)| *a as f64 * b).sum(); + } + out + } + + fn ndc_to_plane(h: &Camera3DHomography, ndc: [f64; 2]) -> [f64; 2] { + let q = apply3(&h.inverse_rows, [ndc[0], ndc[1], 1.0]); + assert!(q[2] > 0.0, "point must be in front of the camera"); + [q[0] / q[2], q[1] / q[2]] + } + + fn forward_ndc(h: &Camera3DHomography, plane: [f64; 2]) -> [f64; 2] { + let inv = h.inverse_rows.map(|r| r.map(|v| v as f64)); + let fwd = invert3(&inv).unwrap(); + let s = apply3(&fwd.map(|r| r.map(|v| v as f32)), [plane[0], plane[1], 1.0]); + [s[0] / s[2], s[1] / s[2]] + } + + fn assert_close(a: f64, b: f64, eps: f64, label: &str) { + assert!((a - b).abs() < eps, "{label}: {a} vs {b}"); + } + + fn fill_zoom(fov_deg: f64) -> f64 { + HY / (fov_deg.to_radians() / 2.0).tan() + } + + #[test] + fn fill_distance_maps_frame_corners_to_plane_corners() { + for fov in [24.0, 45.0, 60.0] { + let props = Camera3DProperties { + fov, + zoom: fill_zoom(fov), + ..Default::default() + }; + let h = camera3d_inverse_homography(&props, ASPECT, None).unwrap(); + let p = ndc_to_plane(&h, [1.0, 1.0]); + assert_close(p[0], 1.0, 1e-4, "corner x"); + assert_close(p[1], HY, 1e-4, "corner y"); + } + } + + #[test] + fn default_framing_matches_expected_size() { + // fov 45 / zoom 2.25 puts the 16:9 card at ~60% of frame height + // (size law: size ∝ 1/(zoom·tan(fov/2))). + let props = Camera3DProperties { + zoom: 2.25, + ..Default::default() + }; + let h = camera3d_inverse_homography(&props, ASPECT, None).unwrap(); + let top = forward_ndc(&h, [0.0, HY]); + assert_close(top[1], 0.6035, 1e-3, "subject half-height in NDC"); + + // fov 24 / zoom 4.5 (the reset pose) frames almost identically. + let props = Camera3DProperties { + fov: 24.0, + zoom: 4.5, + ..Default::default() + }; + let h = camera3d_inverse_homography(&props, ASPECT, None).unwrap(); + let top = forward_ndc(&h, [0.0, HY]); + assert_close(top[1], 0.5876, 1e-3, "long-lens half-height"); + } + + #[test] + fn fov_is_not_size_compensated() { + let at = |fov: f64| { + let props = Camera3DProperties { + fov, + zoom: 2.0, + ..Default::default() + }; + let h = camera3d_inverse_homography(&props, ASPECT, None).unwrap(); + forward_ndc(&h, [0.0, HY])[1] + }; + // Wider fov at the same distance makes the subject smaller. + assert!(at(100.0) < at(45.0)); + assert!(at(45.0) < at(10.0)); + } + + #[test] + fn zoom_is_distance() { + let at = |zoom: f64| { + let props = Camera3DProperties { + zoom, + ..Default::default() + }; + let h = camera3d_inverse_homography(&props, ASPECT, None).unwrap(); + forward_ndc(&h, [0.0, HY])[1] + }; + assert!(at(4.0) < at(2.0), "further away is smaller"); + assert_close(at(4.0), at(2.0) / 2.0, 1e-6, "size ∝ 1/zoom"); + } + + #[test] + fn pan_moves_subject_in_screen_direction() { + let props = Camera3DProperties { + pan_x: 0.5, + pan_y: 0.25, + ..Default::default() + }; + let h = camera3d_inverse_homography(&props, ASPECT, None).unwrap(); + let center = forward_ndc(&h, [0.0, 0.0]); + assert!(center[0] > 0.0, "+pan_x moves subject right"); + assert!(center[1] > 0.0, "+pan_y moves subject up"); + } + + #[test] + fn screen_zoom_magnifies_about_target() { + // Flat pose at the fill distance: the card exactly fills the frame, + // so a 2x screen zoom at content center doubles every NDC coordinate. + let props = Camera3DProperties { + zoom: fill_zoom(45.0), + ..Default::default() + }; + let zoom = Camera3DScreenZoom { + content_uv: cap_project::XY::new(0.5, 0.5), + amount: 2.0, + }; + let h = camera3d_inverse_homography(&props, ASPECT, Some(&zoom)).unwrap(); + let p = ndc_to_plane(&h, [1.0, 1.0]); + assert_close(p[0], 0.5, 1e-4, "half plane x visible"); + assert_close(p[1], HY / 2.0, 1e-4, "half plane y visible"); + + // An off-center target slides toward frame center as q / amount. + let zoom = Camera3DScreenZoom { + content_uv: cap_project::XY::new(0.75, 0.5), + amount: 2.0, + }; + let h = camera3d_inverse_homography(&props, ASPECT, Some(&zoom)).unwrap(); + // Content u 0.75 sits at plane x 0.5·hx, NDC 0.5 unzoomed. + let target = forward_ndc(&h, [0.5, 0.0]); + assert_close(target[0], 0.25, 1e-4, "target recenters as q/amount"); + assert_close(target[1], 0.0, 1e-4, "target y"); + + // Amount 1 (or None) leaves the homography untouched. + let identity_zoom = Camera3DScreenZoom { + content_uv: cap_project::XY::new(0.2, 0.9), + amount: 1.0, + }; + let with_zoom = camera3d_inverse_homography(&props, ASPECT, Some(&identity_zoom)).unwrap(); + let without = camera3d_inverse_homography(&props, ASPECT, None).unwrap(); + assert_eq!(with_zoom.inverse_rows, without.inverse_rows); + } + + #[test] + fn tilt_y_positive_recedes_right_edge() { + let props = Camera3DProperties { + tilt_y: 30.0, + ..Default::default() + }; + let h = camera3d_inverse_homography(&props, ASPECT, None).unwrap(); + let inv = h.inverse_rows.map(|r| r.map(|v| v as f64)); + let fwd = invert3(&inv).unwrap(); + let w_right = apply3(&fwd.map(|r| r.map(|v| v as f32)), [1.0, 0.0, 1.0])[2]; + let w_left = apply3(&fwd.map(|r| r.map(|v| v as f32)), [-1.0, 0.0, 1.0])[2]; + assert!(w_right > w_left, "right edge should be further away"); + } + + #[test] + fn rotate_x_positive_brings_top_closer() { + // Matches the CSS-3D preview (rotateX(-rotateX) in y-down CSS): + // positive rotate_x tips the top toward the viewer. + let props = Camera3DProperties { + rotate_x: 30.0, + ..Default::default() + }; + let h = camera3d_inverse_homography(&props, ASPECT, None).unwrap(); + let inv = h.inverse_rows.map(|r| r.map(|v| v as f64)); + let fwd = invert3(&inv).unwrap(); + let w_top = apply3(&fwd.map(|r| r.map(|v| v as f32)), [0.0, HY, 1.0])[2]; + let w_bottom = apply3(&fwd.map(|r| r.map(|v| v as f32)), [0.0, -HY, 1.0])[2]; + assert!(w_top < w_bottom, "top should be closer"); + } + + #[test] + fn track_sampling_holds_and_lerps() { + let keys = vec![ + Camera3DKeyframe { + time: 1.0, + value: 10.0, + out_easing: Some([0.0, 0.0]), + in_easing: None, + }, + Camera3DKeyframe { + time: 3.0, + value: 20.0, + out_easing: None, + in_easing: Some([1.0, 1.0]), + }, + ]; + assert_eq!(sample_track(5.0, &[], 0.0), 5.0); + assert_eq!(sample_track(5.0, &keys, 0.0), 10.0,); + // Linear handles ([0,0]/[1,1]) give an exact linear midpoint. + assert_close(sample_track(5.0, &keys, 2.0), 15.0, 1e-9, "linear mid"); + assert_eq!(sample_track(5.0, &keys, 9.0), 20.0); + } + + #[test] + fn default_handles_are_cubic_ease_in_out() { + let keys = vec![ + Camera3DKeyframe { + time: 0.0, + value: 0.0, + out_easing: None, + in_easing: None, + }, + Camera3DKeyframe { + time: 1.0, + value: 1.0, + out_easing: None, + in_easing: None, + }, + ]; + // cubic-bezier(0.65, 0, 0.35, 1): slow start, midpoint exactly 0.5. + let quarter = sample_track(0.0, &keys, 0.25); + assert!( + quarter < 0.15, + "eased quarter should lag linear, got {quarter}" + ); + assert_close(sample_track(0.0, &keys, 0.5), 0.5, 1e-4, "eased mid"); + } + + fn segment() -> Camera3DSegment { + Camera3DSegment { + start: 2.0, + end: 8.0, + enabled: true, + properties: Camera3DProperties { + tilt_y: 26.0, + tilt_x: -28.0, + zoom: 1.59, + ..Default::default() + }, + blur: Camera3DBlur::default(), + tracks: Camera3DTracks::default(), + transition_in: 0.5, + transition_out: 0.5, + } + } + + #[test] + fn outside_segments_is_none() { + assert!(interpolate_camera3d(1.0, &[segment()], ASPECT).is_none()); + assert!(interpolate_camera3d(9.0, &[segment()], ASPECT).is_none()); + assert!(interpolate_camera3d(5.0, &[], ASPECT).is_none()); + } + + #[test] + fn segment_edges_ramp_from_flat() { + let seg = segment(); + // At the exact edge the pose is flat (skipped) and blur is off. + let frame = interpolate_camera3d(2.0, std::slice::from_ref(&seg), ASPECT).unwrap(); + assert!(frame.pose.is_none()); + assert!(frame.blur.is_none()); + + let mid_ramp = interpolate_camera3d(2.25, std::slice::from_ref(&seg), ASPECT) + .unwrap() + .pose + .unwrap(); + assert!(mid_ramp.tilt_y > 0.0 && mid_ramp.tilt_y < 26.0); + // The ramp dollies from the fill distance toward the pose zoom. + let fill = fill_zoom(45.0); + let (lo, hi) = (fill.min(1.59), fill.max(1.59)); + assert!( + mid_ramp.zoom > lo && mid_ramp.zoom < hi, + "mid-ramp zoom {} should sit between {lo} and {hi}", + mid_ramp.zoom + ); + + let settled = interpolate_camera3d(5.0, std::slice::from_ref(&seg), ASPECT) + .unwrap() + .pose + .unwrap(); + assert_close(settled.tilt_y, 26.0, 1e-9, "settled tilt_y"); + assert_close(settled.zoom, 1.59, 1e-9, "settled zoom"); + } + + #[test] + fn blur_requires_mode_and_strength() { + let mut seg = segment(); + seg.transition_in = 0.0; + seg.transition_out = 0.0; + seg.blur.strength = 19.0; + // Mode none: no blur even with strength. + let frame = interpolate_camera3d(5.0, std::slice::from_ref(&seg), ASPECT).unwrap(); + assert!(frame.blur.is_none()); + + seg.blur.mode = Camera3DBlurMode::Radial; + let frame = interpolate_camera3d(5.0, std::slice::from_ref(&seg), ASPECT).unwrap(); + let blur = frame.blur.unwrap(); + assert_close(blur.strength, 19.0, 1e-9, "blur strength"); + + // Bokeh caps strength at 20. + seg.blur.bokeh = true; + seg.blur.strength = 45.0; + let frame = interpolate_camera3d(5.0, std::slice::from_ref(&seg), ASPECT).unwrap(); + assert_close(frame.blur.unwrap().strength, 20.0, 1e-9, "bokeh cap"); + } + + #[test] + fn blur_is_keyframable() { + let mut seg = segment(); + seg.transition_in = 0.0; + seg.transition_out = 0.0; + seg.blur.mode = Camera3DBlurMode::TiltShift; + seg.tracks.blur_strength = vec![ + Camera3DKeyframe { + time: 0.0, + value: 0.0, + out_easing: Some([0.0, 0.0]), + in_easing: None, + }, + Camera3DKeyframe { + time: 6.0, + value: 30.0, + out_easing: None, + in_easing: Some([1.0, 1.0]), + }, + ]; + let frame = interpolate_camera3d(5.0, std::slice::from_ref(&seg), ASPECT).unwrap(); + assert_close(frame.blur.unwrap().strength, 15.0, 1e-6, "keyed strength"); + } +} diff --git a/crates/rendering/src/layers/camera3d.rs b/crates/rendering/src/layers/camera3d.rs new file mode 100644 index 00000000000..bcb9e811072 --- /dev/null +++ b/crates/rendering/src/layers/camera3d.rs @@ -0,0 +1,433 @@ +use bytemuck::{Pod, Zeroable}; +use wgpu::util::DeviceExt; + +use crate::ProjectUniforms; +use crate::camera3d::{CAMERA3D_BLUR_BASE_HEIGHT, camera3d_inverse_homography}; +use cap_project::Camera3DBlurMode; + +/// Which blur kernel runs this frame. +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum Camera3DBlurKind { + /// Separable gaussian: horizontal pass then vertical pass. + Gaussian, + /// Single-pass ring-disc bokeh. + Bokeh, +} + +/// Warps the composed content texture through the timeline's 3D camera pose +/// and applies a screen-space focus blur. +/// See `shaders/camera3d.wgsl` and `shaders/camera3d-blur.wgsl`. +pub struct Camera3DLayer { + warp_active: bool, + blur_kind: Option, + sampler: wgpu::Sampler, + warp_uniforms_buffer: wgpu::Buffer, + warp_pipeline: Camera3DPipeline, + cached_warp_uniforms: Option, + // Separate H/V buffers: both passes are encoded before submission, so a + // single buffer would make the H pass read the V parameters. + blur_h_buffer: wgpu::Buffer, + blur_v_buffer: wgpu::Buffer, + blur_pipeline: Camera3DBlurPipeline, + cached_blur_uniforms: Option<(Camera3DBlurUniforms, Camera3DBlurUniforms)>, +} + +impl Camera3DLayer { + pub fn new(device: &wgpu::Device) -> Self { + let make_blur_buffer = |label: &str| { + device.create_buffer_init(&wgpu::util::BufferInitDescriptor { + label: Some(label), + contents: bytemuck::cast_slice(&[Camera3DBlurUniforms::default()]), + usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST, + }) + }; + Self { + warp_active: false, + blur_kind: None, + sampler: device.create_sampler(&wgpu::SamplerDescriptor { + address_mode_u: wgpu::AddressMode::ClampToEdge, + address_mode_v: wgpu::AddressMode::ClampToEdge, + address_mode_w: wgpu::AddressMode::ClampToEdge, + mag_filter: wgpu::FilterMode::Linear, + min_filter: wgpu::FilterMode::Linear, + mipmap_filter: wgpu::FilterMode::Nearest, + ..Default::default() + }), + warp_uniforms_buffer: device.create_buffer_init(&wgpu::util::BufferInitDescriptor { + label: Some("Camera3D Uniform Buffer"), + contents: bytemuck::cast_slice(&[Camera3DUniforms::default()]), + usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST, + }), + warp_pipeline: Camera3DPipeline::new(device), + cached_warp_uniforms: None, + blur_h_buffer: make_blur_buffer("Camera3D Blur H Uniform Buffer"), + blur_v_buffer: make_blur_buffer("Camera3D Blur V Uniform Buffer"), + blur_pipeline: Camera3DBlurPipeline::new(device), + cached_blur_uniforms: None, + } + } + + /// Whether the warp pass should run this frame. + pub fn is_active(&self) -> bool { + self.warp_active + } + + pub fn blur_kind(&self) -> Option { + self.blur_kind + } + + pub fn prepare(&mut self, queue: &wgpu::Queue, uniforms: &ProjectUniforms) { + self.warp_active = false; + self.blur_kind = None; + + let Some(frame) = uniforms.camera3d else { + return; + }; + let (out_w, out_h) = (uniforms.output_size.0.max(1), uniforms.output_size.1.max(1)); + let aspect = out_w as f64 / out_h as f64; + + if let Some(pose) = frame.pose + && let Some(homography) = + camera3d_inverse_homography(&pose, aspect, uniforms.camera3d_zoom.as_ref()) + { + let rows = homography.inverse_rows; + let (hx, hy) = homography.half_extents; + let warp_uniforms = Camera3DUniforms { + inv_row0: [rows[0][0], rows[0][1], rows[0][2], 0.0], + inv_row1: [rows[1][0], rows[1][1], rows[1][2], hx], + inv_row2: [rows[2][0], rows[2][1], rows[2][2], hy], + }; + + if self.cached_warp_uniforms.as_ref() != Some(&warp_uniforms) { + queue.write_buffer( + &self.warp_uniforms_buffer, + 0, + bytemuck::cast_slice(&[warp_uniforms]), + ); + self.cached_warp_uniforms = Some(warp_uniforms); + } + self.warp_active = true; + } + + if let Some(blur) = frame.blur { + // Strength is authored at 1080p output height; scale so preview + // and export produce the same look. + let strength_px = (blur.strength * out_h as f64 / CAMERA3D_BLUR_BASE_HEIGHT) as f32; + let mode = match blur.mode { + Camera3DBlurMode::None => return, + Camera3DBlurMode::Radial => 1.0, + Camera3DBlurMode::Directional => 2.0, + Camera3DBlurMode::TiltShift => 3.0, + }; + let params = |pass: f32| Camera3DBlurUniforms { + params0: [pass, mode, strength_px, blur.falloff as f32], + params1: [ + blur.focus_x as f32, + blur.focus_y as f32, + blur.focus_size as f32, + blur.angle as f32, + ], + params2: [blur.dir_position as f32, out_w as f32, out_h as f32, 0.0], + }; + let (h, v) = if blur.bokeh { + // Bokeh is single-pass; only the "V" slot is used. + (params(2.0), params(2.0)) + } else { + (params(0.0), params(1.0)) + }; + + if self.cached_blur_uniforms.as_ref() != Some(&(h, v)) { + if !blur.bokeh { + // Bokeh never reads the H buffer (lib.rs only runs the V + // pass for it), so skip the redundant upload. + queue.write_buffer(&self.blur_h_buffer, 0, bytemuck::cast_slice(&[h])); + } + queue.write_buffer(&self.blur_v_buffer, 0, bytemuck::cast_slice(&[v])); + self.cached_blur_uniforms = Some((h, v)); + } + self.blur_kind = Some(if blur.bokeh { + Camera3DBlurKind::Bokeh + } else { + Camera3DBlurKind::Gaussian + }); + } + } + + pub fn render( + &self, + pass: &mut wgpu::RenderPass<'_>, + device: &wgpu::Device, + content_texture: &wgpu::TextureView, + ) { + pass.set_pipeline(&self.warp_pipeline.render_pipeline); + pass.set_bind_group( + 0, + &self.warp_pipeline.bind_group( + device, + &self.warp_uniforms_buffer, + content_texture, + &self.sampler, + ), + &[], + ); + pass.draw(0..3, 0..1); + } + + /// Horizontal gaussian pass (or the single bokeh pass, which uses the V + /// buffer via [`Self::render_blur_v`] instead). + pub fn render_blur_h( + &self, + pass: &mut wgpu::RenderPass<'_>, + device: &wgpu::Device, + source_texture: &wgpu::TextureView, + ) { + self.render_blur(pass, device, &self.blur_h_buffer, source_texture); + } + + pub fn render_blur_v( + &self, + pass: &mut wgpu::RenderPass<'_>, + device: &wgpu::Device, + source_texture: &wgpu::TextureView, + ) { + self.render_blur(pass, device, &self.blur_v_buffer, source_texture); + } + + fn render_blur( + &self, + pass: &mut wgpu::RenderPass<'_>, + device: &wgpu::Device, + buffer: &wgpu::Buffer, + source_texture: &wgpu::TextureView, + ) { + pass.set_pipeline(&self.blur_pipeline.render_pipeline); + pass.set_bind_group( + 0, + &self + .blur_pipeline + .bind_group(device, buffer, source_texture, &self.sampler), + &[], + ); + pass.draw(0..3, 0..1); + } +} + +#[repr(C)] +#[derive(Debug, Clone, Copy, Pod, Zeroable, Default, PartialEq)] +struct Camera3DUniforms { + inv_row0: [f32; 4], + inv_row1: [f32; 4], + inv_row2: [f32; 4], +} + +#[repr(C)] +#[derive(Debug, Clone, Copy, Pod, Zeroable, Default, PartialEq)] +struct Camera3DBlurUniforms { + params0: [f32; 4], + params1: [f32; 4], + params2: [f32; 4], +} + +fn make_bind_group_layout(device: &wgpu::Device, label: &str) -> wgpu::BindGroupLayout { + device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some(label), + entries: &[ + wgpu::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu::ShaderStages::VERTEX_FRAGMENT, + ty: wgpu::BindingType::Buffer { + ty: wgpu::BufferBindingType::Uniform, + has_dynamic_offset: false, + min_binding_size: None, + }, + count: None, + }, + wgpu::BindGroupLayoutEntry { + binding: 1, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Texture { + sample_type: wgpu::TextureSampleType::Float { filterable: true }, + view_dimension: wgpu::TextureViewDimension::D2, + multisampled: false, + }, + count: None, + }, + wgpu::BindGroupLayoutEntry { + binding: 2, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering), + count: None, + }, + ], + }) +} + +fn make_pipeline( + device: &wgpu::Device, + label: &str, + shader: wgpu::ShaderModuleDescriptor<'_>, + layout: &wgpu::BindGroupLayout, + blend: wgpu::BlendState, +) -> wgpu::RenderPipeline { + let shader = device.create_shader_module(shader); + let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some(label), + bind_group_layouts: &[layout], + push_constant_ranges: &[], + }); + device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some(label), + layout: Some(&pipeline_layout), + vertex: wgpu::VertexState { + module: &shader, + entry_point: Some("vs_main"), + buffers: &[], + compilation_options: wgpu::PipelineCompilationOptions { + constants: &[], + zero_initialize_workgroup_memory: false, + }, + }, + fragment: Some(wgpu::FragmentState { + module: &shader, + entry_point: Some("fs_main"), + targets: &[Some(wgpu::ColorTargetState { + format: wgpu::TextureFormat::Rgba8Unorm, + blend: Some(blend), + write_mask: wgpu::ColorWrites::ALL, + })], + compilation_options: wgpu::PipelineCompilationOptions { + constants: &[], + zero_initialize_workgroup_memory: false, + }, + }), + primitive: wgpu::PrimitiveState { + topology: wgpu::PrimitiveTopology::TriangleList, + strip_index_format: None, + front_face: wgpu::FrontFace::Ccw, + cull_mode: None, + polygon_mode: wgpu::PolygonMode::Fill, + unclipped_depth: false, + conservative: false, + }, + depth_stencil: None, + multisample: wgpu::MultisampleState::default(), + multiview: None, + cache: None, + }) +} + +fn make_bind_group( + device: &wgpu::Device, + label: &str, + layout: &wgpu::BindGroupLayout, + uniform_buffer: &wgpu::Buffer, + texture_view: &wgpu::TextureView, + sampler: &wgpu::Sampler, +) -> wgpu::BindGroup { + device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some(label), + layout, + entries: &[ + wgpu::BindGroupEntry { + binding: 0, + resource: uniform_buffer.as_entire_binding(), + }, + wgpu::BindGroupEntry { + binding: 1, + resource: wgpu::BindingResource::TextureView(texture_view), + }, + wgpu::BindGroupEntry { + binding: 2, + resource: wgpu::BindingResource::Sampler(sampler), + }, + ], + }) +} + +struct Camera3DPipeline { + bind_group_layout: wgpu::BindGroupLayout, + render_pipeline: wgpu::RenderPipeline, +} + +impl Camera3DPipeline { + fn new(device: &wgpu::Device) -> Self { + let bind_group_layout = make_bind_group_layout(device, "camera3d Bind Group Layout"); + let render_pipeline = make_pipeline( + device, + "Camera3D Pipeline", + wgpu::ShaderModuleDescriptor { + label: Some("Camera3D Shader"), + source: wgpu::ShaderSource::Wgsl(include_str!("../shaders/camera3d.wgsl").into()), + }, + &bind_group_layout, + // The content texture is premultiplied; composite it over the + // background that's already in the target. + wgpu::BlendState::PREMULTIPLIED_ALPHA_BLENDING, + ); + Self { + bind_group_layout, + render_pipeline, + } + } + + fn bind_group( + &self, + device: &wgpu::Device, + uniform_buffer: &wgpu::Buffer, + texture_view: &wgpu::TextureView, + sampler: &wgpu::Sampler, + ) -> wgpu::BindGroup { + make_bind_group( + device, + "Camera3D Bind Group", + &self.bind_group_layout, + uniform_buffer, + texture_view, + sampler, + ) + } +} + +struct Camera3DBlurPipeline { + bind_group_layout: wgpu::BindGroupLayout, + render_pipeline: wgpu::RenderPipeline, +} + +impl Camera3DBlurPipeline { + fn new(device: &wgpu::Device) -> Self { + let bind_group_layout = make_bind_group_layout(device, "camera3d-blur Bind Group Layout"); + let render_pipeline = make_pipeline( + device, + "Camera3D Blur Pipeline", + wgpu::ShaderModuleDescriptor { + label: Some("Camera3D Blur Shader"), + source: wgpu::ShaderSource::Wgsl( + include_str!("../shaders/camera3d-blur.wgsl").into(), + ), + }, + &bind_group_layout, + wgpu::BlendState::REPLACE, + ); + Self { + bind_group_layout, + render_pipeline, + } + } + + fn bind_group( + &self, + device: &wgpu::Device, + uniform_buffer: &wgpu::Buffer, + texture_view: &wgpu::TextureView, + sampler: &wgpu::Sampler, + ) -> wgpu::BindGroup { + make_bind_group( + device, + "Camera3D Blur Bind Group", + &self.bind_group_layout, + uniform_buffer, + texture_view, + sampler, + ) + } +} diff --git a/crates/rendering/src/layers/mod.rs b/crates/rendering/src/layers/mod.rs index 258af84ede6..cb96d8b4d5e 100644 --- a/crates/rendering/src/layers/mod.rs +++ b/crates/rendering/src/layers/mod.rs @@ -1,6 +1,7 @@ mod background; mod blur; mod camera; +mod camera3d; mod captions; mod cursor; mod display; @@ -64,6 +65,7 @@ pub(crate) fn new_font_system() -> glyphon::FontSystem { pub use background::*; pub use blur::*; pub use camera::*; +pub use camera3d::*; pub use captions::*; pub use cursor::*; pub use display::*; diff --git a/crates/rendering/src/lib.rs b/crates/rendering/src/lib.rs index eb41f1116e1..c3167f012f1 100644 --- a/crates/rendering/src/lib.rs +++ b/crates/rendering/src/lib.rs @@ -17,8 +17,9 @@ use frame_pipeline::{ }; use futures::future::OptionFuture; use layers::{ - Background, BackgroundLayer, BlurLayer, CameraLayer, CaptionsLayer, CursorLayer, DisplayLayer, - FrameLayer, KeyboardLayer, MaskLayer, NotchLayer, NotchUniforms, TextLayer, + Background, BackgroundLayer, BlurLayer, Camera3DBlurKind, Camera3DLayer, CameraLayer, + CaptionsLayer, CursorLayer, DisplayLayer, FrameLayer, KeyboardLayer, MaskLayer, NotchLayer, + NotchUniforms, TextLayer, }; use specta::Type; use spring_mass_damper::SpringMassDamperSimulationConfig; @@ -29,6 +30,7 @@ use std::sync::{ use std::{path::PathBuf, time::Instant}; use tokio::sync::mpsc; +pub mod camera3d; pub mod composite_frame; mod coord; pub mod cpu_yuv; @@ -68,6 +70,7 @@ pub fn prewarm_fonts() { drop(layers::new_font_system()); } +use camera3d::{Camera3DFrame, interpolate_camera3d}; pub use cursor_interpolation::PrecomputedCursorTimeline; use mask::interpolate_masks; use scene::*; @@ -2262,6 +2265,12 @@ pub struct ProjectUniforms { pub motion_blur_amount: f32, pub masks: Vec, pub texts: Vec, + /// Effective 3D camera state for this frame; `None` when the frame is + /// outside every 3d segment (the warp and blur passes are skipped). + pub camera3d: Option, + /// The 2D zoom re-expressed as an on-screen card magnification while a 3D + /// pose is active (the display itself renders unzoomed into the card). + pub camera3d_zoom: Option, } #[derive(Debug, Clone)] @@ -3348,6 +3357,35 @@ impl ProjectUniforms { scene_segments, )); + let camera3d = project.timeline.as_ref().and_then(|timeline| { + interpolate_camera3d( + frame_time as f64, + &timeline.camera3d_segments, + output_size.0 as f64 / output_size.1.max(1) as f64, + ) + }); + // The card's flat drop shadow is baked into the warped texture, so in + // 3D it would rotate with the plane and clip at the texture edge. + // A floating card has no baked shadow, so fade ours out. + let camera3d_shadow_fade = 1.0 - camera3d.map_or(0.0, |c| c.activity) as f32; + + // While a 3D pose is active the 2D zoom must not crop the display + // inside the card texture (the crop edge reads as the card arbitrarily + // cutting content off). The content renders unzoomed and the sampled + // zoom becomes a screen-space magnification of the whole card about + // the zoom target instead (see `camera3d_zoom` below). + let camera3d_pose_active = camera3d.as_ref().is_some_and(|c| c.pose.is_some()); + let raw_zoom = zoom; + let (zoom, prev_zoom, motion_prev_zoom) = if camera3d_pose_active { + ( + InterpolatedZoom::default(), + InterpolatedZoom::default(), + InterpolatedZoom::default(), + ) + } else { + (zoom, prev_zoom, motion_prev_zoom) + }; + // Resolve the side-by-side layout once and share it with the display, // camera and cursor layers. Only engages when a camera actually exists; // otherwise the layers render normally (graceful full-screen fallback). @@ -3467,6 +3505,7 @@ impl ProjectUniforms { None }; + let mut camera3d_zoom: Option = None; let (display, display_motion_parent, frame_chrome, display_outer_bounds, notch) = { let output_size = XY::new(output_size.0 as f64, output_size.1 as f64); let size = [options.screen_size.x as f32, options.screen_size.y as f32]; @@ -3485,6 +3524,27 @@ impl ProjectUniforms { let display_size = Coord::::new(layout.content_size); let frame_config = project.background.frame.clone().filter(|f| f.is_active()); + if camera3d_pose_active { + // Re-express the (neutralized) 2D zoom as a magnification of + // the whole card about the zoom target's on-card position. + let bounds = raw_zoom.bounds; + let span = bounds.bottom_right - bounds.top_left; + let amount = ((span.x + span.y) / 2.0).max(1.0); + if amount > 1.001 { + let center_u = (0.5 - bounds.top_left.x) / span.x.max(1e-6); + let center_v = (0.5 - bounds.top_left.y) / span.y.max(1e-6); + let frame_x = display_offset.coord.x + center_u * display_size.coord.x; + let frame_y = display_offset.coord.y + center_v * display_size.coord.y; + camera3d_zoom = Some(camera3d::Camera3DScreenZoom { + content_uv: XY::new( + frame_x / output_size.x.max(1.0), + frame_y / output_size.y.max(1.0), + ), + amount, + }); + } + } + let (start, end) = Self::display_bounds(&zoom, display_offset, display_size); let (prev_start, prev_end) = Self::display_bounds(&motion_prev_zoom, display_offset, display_size); @@ -3631,7 +3691,7 @@ impl ProjectUniforms { 0.0, ], shadow: if decorated { - project.background.shadow * split_fade + project.background.shadow * split_fade * camera3d_shadow_fade } else { 0.0 }, @@ -3645,7 +3705,8 @@ impl ProjectUniforms { .advanced_shadow .as_ref() .map_or(18.0, |s| s.opacity) - * split_fade, + * split_fade + * camera3d_shadow_fade, shadow_blur: project .background .advanced_shadow @@ -3744,7 +3805,9 @@ impl ProjectUniforms { descriptor.zoom_amount, 0.0, ], - shadow: project.background.shadow * display_decoration_fade, + shadow: project.background.shadow + * display_decoration_fade + * camera3d_shadow_fade, shadow_size: project .background .advanced_shadow @@ -3755,7 +3818,8 @@ impl ProjectUniforms { .advanced_shadow .as_ref() .map_or(18.0, |s| s.opacity) - * display_decoration_fade, + * display_decoration_fade + * camera3d_shadow_fade, shadow_blur: project .background .advanced_shadow @@ -3949,7 +4013,7 @@ impl ProjectUniforms { camera_descriptor.zoom_amount, 0.0, ], - shadow: project.camera.shadow * chrome_fade, + shadow: project.camera.shadow * chrome_fade * camera3d_shadow_fade, shadow_size: project .camera .advanced_shadow @@ -3960,7 +4024,8 @@ impl ProjectUniforms { .advanced_shadow .as_ref() .map_or(18.0, |s| s.opacity) - * chrome_fade, + * chrome_fade + * camera3d_shadow_fade, shadow_blur: project .camera .advanced_shadow @@ -4122,6 +4187,8 @@ impl ProjectUniforms { motion_blur_amount: cursor_motion_blur, masks, texts, + camera3d, + camera3d_zoom, } } } @@ -5191,6 +5258,7 @@ pub struct RendererLayers { text: TextLayer, captions: CaptionsLayer, keyboard: KeyboardLayer, + camera3d: Camera3DLayer, camera_blur_processor: Option, camera_blur_init_failed: bool, } @@ -5235,6 +5303,7 @@ impl RendererLayers { text: TextLayer::new(device, queue), captions: CaptionsLayer::new(device, queue), keyboard: KeyboardLayer::new(device, queue), + camera3d: Camera3DLayer::new(device), camera_blur_processor: None, camera_blur_init_failed: false, } @@ -5637,6 +5706,8 @@ impl RendererLayers { ); timings.keyboard_prepare_duration = start.elapsed(); + self.camera3d.prepare(&constants.queue, uniforms); + Ok(timings) } @@ -5699,31 +5770,53 @@ impl RendererLayers { true }; + // When a 3D camera pose is active, the content group (frame chrome, + // display, cursor, notch, camera) renders into the spare ping-pong + // texture, and the camera3d pass then warps it over the untouched + // background. Overlay annotations (masks, text, keyboard, captions) + // stay flat on top. When inactive this is exactly the flat path. + let camera3d_active = self.camera3d.is_active(); + if camera3d_active { + let _pass = render_pass!( + session.other_texture_view(), + wgpu::LoadOp::Clear(wgpu::Color::TRANSPARENT) + ); + } + macro_rules! content_view { + () => { + if camera3d_active { + session.other_texture_view() + } else { + session.current_texture_view() + } + }; + } + if should_render_screen && self.frame.has_content() { - let mut pass = render_pass!(session.current_texture_view(), wgpu::LoadOp::Load); + let mut pass = render_pass!(content_view!(), wgpu::LoadOp::Load); self.frame.render(&mut pass); } if should_render_screen { - let mut pass = render_pass!(session.current_texture_view(), wgpu::LoadOp::Load); + let mut pass = render_pass!(content_view!(), wgpu::LoadOp::Load); self.display.render(&mut pass); } if should_render_cursor { - let mut pass = render_pass!(session.current_texture_view(), wgpu::LoadOp::Load); + let mut pass = render_pass!(content_view!(), wgpu::LoadOp::Load); self.cursor.render(&mut pass); } // After the cursor, which really does disappear behind the notch on a // Mac, but before masks and text, which are editor annotations. if should_render_screen && self.notch.has_content() { - let mut pass = render_pass!(session.current_texture_view(), wgpu::LoadOp::Load); + let mut pass = render_pass!(content_view!(), wgpu::LoadOp::Load); self.notch.render(&mut pass); } // Render camera-only layer when transitioning with CameraOnly mode if uniforms.scene.is_transitioning_camera_only() { - let mut pass = render_pass!(session.current_texture_view(), wgpu::LoadOp::Load); + let mut pass = render_pass!(content_view!(), wgpu::LoadOp::Load); self.camera_only.render(&mut pass); } @@ -5731,10 +5824,40 @@ impl RendererLayers { if uniforms.scene.should_render_camera() && uniforms.scene.regular_camera_transition_opacity() > 0.01 { - let mut pass = render_pass!(session.current_texture_view(), wgpu::LoadOp::Load); + let mut pass = render_pass!(content_view!(), wgpu::LoadOp::Load); self.camera.render(&mut pass); } + if camera3d_active { + let mut pass = render_pass!(session.current_texture_view(), wgpu::LoadOp::Load); + self.camera3d + .render(&mut pass, device, session.other_texture_view()); + } + + // Focus blur over the composed frame (background and warped content + // together), before the flat annotations. + match self.camera3d.blur_kind() { + Some(Camera3DBlurKind::Gaussian) => { + { + let mut pass = render_pass!(session.other_texture_view(), wgpu::LoadOp::Load); + self.camera3d + .render_blur_h(&mut pass, device, session.current_texture_view()); + } + let mut pass = render_pass!(session.current_texture_view(), wgpu::LoadOp::Load); + self.camera3d + .render_blur_v(&mut pass, device, session.other_texture_view()); + } + Some(Camera3DBlurKind::Bokeh) => { + { + let mut pass = render_pass!(session.other_texture_view(), wgpu::LoadOp::Load); + self.camera3d + .render_blur_v(&mut pass, device, session.current_texture_view()); + } + session.swap_textures(); + } + None => {} + } + if !uniforms.masks.is_empty() { for mask in &uniforms.masks { self.mask.render(device, queue, session, encoder, mask); diff --git a/crates/rendering/src/shaders/camera3d-blur.wgsl b/crates/rendering/src/shaders/camera3d-blur.wgsl new file mode 100644 index 00000000000..4ab74b69743 --- /dev/null +++ b/crates/rendering/src/shaders/camera3d-blur.wgsl @@ -0,0 +1,174 @@ +// Screen-space focus blur system. +// +// The "depth of field" is a UV-mask variable-radius blur: a per-pixel CoC +// (blur radius in output pixels) computed from the focus configuration, fed to +// either a separable gaussian (two passes, pass 0 = horizontal, pass 1 = +// vertical) or a single-pass ring-disc bokeh kernel with highlight gain +// (pass 2). Runs on the fully composed frame; the sharp region is wherever +// the focus mask says it is, exactly like the original. +// +// params0: (pass, mode, strength_px, falloff) +// pass: 0 = gaussian H, 1 = gaussian V, 2 = bokeh +// mode: 1 = radial, 2 = directional, 3 = tilt-shift +// strength_px: blur radius, already scaled to this render resolution +// params1: (focus_x, focus_y, focus_size, angle_rad) +// params2: (dir_position, resolution_x, resolution_y, unused) +// +// Focus coordinates and angles work in GL-style UV space (origin bottom-left, +// y up), matching the original shaders; screen_uv is converted on entry. + +struct Camera3DBlurUniforms { + params0: vec4, + params1: vec4, + params2: vec4, +}; + +@group(0) @binding(0) var u: Camera3DBlurUniforms; +@group(0) @binding(1) var t_source: texture_2d; +@group(0) @binding(2) var s_source: sampler; + +struct VertexOutput { + @builtin(position) position: vec4, + @location(0) screen_uv: vec2, +}; + +@vertex +fn vs_main(@builtin(vertex_index) vertex_index: u32) -> VertexOutput { + let uv = vec2( + f32((vertex_index << 1u) & 2u), + f32(vertex_index & 2u), + ); + var out: VertexOutput; + out.position = vec4(uv * 2.0 - 1.0, 0.0, 1.0); + out.screen_uv = vec2(uv.x, 1.0 - uv.y); + return out; +} + +// CoC (circle of confusion) function. +fn blur_amount(gl_uv: vec2) -> f32 { + let mode = u.params0.y; + let strength = u.params0.z; + let falloff = u.params0.w; + let resolution = u.params2.yz; + let aspect = resolution.x / max(resolution.y, 1.0); + + let widen = 1.0 + falloff * 3.0; + let curve = mix(2.0, 0.7, falloff); + + if (mode == 2.0) { + // Directional: one-sided blur past a line through frame center. + let angle = u.params1.w; + let dir = vec2(cos(angle), sin(angle)); + var diff = gl_uv - vec2(0.5); + diff.x = diff.x * aspect; + let projected = dot(diff, dir); + let reach = aspect * 0.5 + 0.5; + let threshold = mix(-reach, reach, u.params2.x); + let d = max(projected - threshold, 0.0); + let t = smoothstep(0.0, 0.7 * widen, d); + return pow(t, curve) * strength; + } + + var diff = gl_uv - u.params1.xy; + diff.x = diff.x * aspect; + + var dist = 0.0; + if (mode == 3.0) { + // Tilt-shift: sharp band around the focus line. + let angle = u.params1.w; + let perp = vec2(sin(angle), cos(angle)); + let d = abs(dot(diff, perp)); + let band = u.params1.z * 0.5; + dist = max(d - band, 0.0); + } else { + // Radial: sharp circle around the focus point. + dist = length(diff); + } + + let edge = u.params1.z * 0.5; + let t = smoothstep(0.0, (edge + 0.35) * widen, dist - edge); + return pow(t, curve) * strength; +} + +fn gaussian(gl_uv: vec2, screen_uv: vec2, horizontal: bool) -> vec4 { + let coc = blur_amount(gl_uv); + let center = textureSampleLevel(t_source, s_source, screen_uv, 0.0); + if (coc < 0.5) { + return center; + } + + let resolution = u.params2.yz; + let texel = 1.0 / max(resolution, vec2(1.0)); + // The radius cap scales with output height, exactly like strength_px does + // on the CPU side (camera3d.rs), so the kernel covers the same fraction of + // the gaussian curve at every resolution and preview matches export. The + // flat ceiling (the scaled cap at 8K) bounds per-fragment cost. + let radius_cap = min(resolution.y * (40.0 / 1080.0), 160.0); + let radius = i32(min(coc, radius_cap)); + let sigma = coc * 0.5; + let inv_sigma2 = 1.0 / (2.0 * sigma * sigma); + + var color = vec4(0.0); + var total = 0.0; + for (var i = -radius; i <= radius; i++) { + let fi = f32(i); + let weight = exp(-fi * fi * inv_sigma2); + var offset: vec2; + if (horizontal) { + offset = vec2(texel.x * fi, 0.0); + } else { + offset = vec2(0.0, texel.y * fi); + } + color += textureSampleLevel(t_source, s_source, screen_uv + offset, 0.0) * weight; + total += weight; + } + return color / max(total, 1e-6); +} + +// Ring-disc bokeh: 3 rings × (5·ring) samples with highlight gain, so bright +// spots read as discs. Offsets are aspect-corrected to circles (fixing the +// original's inverted correction). +fn bokeh(gl_uv: vec2, screen_uv: vec2) -> vec4 { + let coc = blur_amount(gl_uv); + let center = textureSampleLevel(t_source, s_source, screen_uv, 0.0); + if (coc < 0.5) { + return center; + } + + let resolution = u.params2.yz; + let texel = 1.0 / max(resolution, vec2(1.0)); + let aspect = resolution.x / max(resolution.y, 1.0); + let ring_step = coc / 3.0; + + var acc = center; + var wsum = 1.0; + for (var ring = 1; ring <= 3; ring++) { + let ring_samples = ring * 5; + let r = f32(ring) * ring_step; + let ring_weight = mix(1.0, f32(ring) / 3.0, 0.3); + for (var j = 0; j < 15; j++) { + if (j >= ring_samples) { + break; + } + let a = 6.28318530718 * f32(j) / f32(ring_samples); + let offset = vec2(cos(a) / aspect, sin(a)) * r * texel.y; + // texel.y scale + /aspect on x keeps the disc circular on screen. + let s = textureSampleLevel(t_source, s_source, screen_uv + vec2(offset.x, -offset.y), 0.0); + let luma = dot(s.rgb, vec3(0.299, 0.587, 0.114)); + let gain = 1.0 + smoothstep(0.7, 1.0, luma) * 1.5; + acc += s * ring_weight * gain; + wsum += ring_weight * gain; + } + } + return acc / wsum; +} + +@fragment +fn fs_main(in: VertexOutput) -> @location(0) vec4 { + let gl_uv = vec2(in.screen_uv.x, 1.0 - in.screen_uv.y); + let pass_kind = u.params0.x; + if (pass_kind == 2.0) { + return bokeh(gl_uv, in.screen_uv); + } + return gaussian(gl_uv, in.screen_uv, pass_kind == 0.0); +} diff --git a/crates/rendering/src/shaders/camera3d.wgsl b/crates/rendering/src/shaders/camera3d.wgsl new file mode 100644 index 00000000000..dc2c9a3f9de --- /dev/null +++ b/crates/rendering/src/shaders/camera3d.wgsl @@ -0,0 +1,79 @@ +// 3D perspective warp for the composed content plane. +// +// The content group (frame chrome, display, cursor, notch, camera) is rendered +// into a transparent texture; this pass maps each output pixel back onto that +// texture through the inverse planar homography and composites the result over +// the untouched background with premultiplied alpha. +// +// Row layouts: inv_row0.w is unused; inv_row1.w and inv_row2.w carry the +// plane half-extents (hx, hy) — the content plane spans [-hx, hx] × [-hy, hy] +// in world units with its longest side equal to 2. + +struct Camera3DUniforms { + inv_row0: vec4, + inv_row1: vec4, + inv_row2: vec4, +}; + +@group(0) @binding(0) var u: Camera3DUniforms; +@group(0) @binding(1) var t_content: texture_2d; +@group(0) @binding(2) var s_content: sampler; + +struct VertexOutput { + @builtin(position) position: vec4, + @location(0) screen_uv: vec2, +}; + +@vertex +fn vs_main(@builtin(vertex_index) vertex_index: u32) -> VertexOutput { + // Fullscreen triangle. + let uv = vec2( + f32((vertex_index << 1u) & 2u), + f32(vertex_index & 2u), + ); + var out: VertexOutput; + out.position = vec4(uv * 2.0 - 1.0, 0.0, 1.0); + out.screen_uv = vec2(uv.x, 1.0 - uv.y); + return out; +} + +@fragment +fn fs_main(in: VertexOutput) -> @location(0) vec4 { + let half_extents = vec2(u.inv_row1.w, u.inv_row2.w); + // y-up NDC of this output pixel. + let ndc = vec3( + in.screen_uv.x * 2.0 - 1.0, + 1.0 - in.screen_uv.y * 2.0, + 1.0, + ); + + let q = vec3( + dot(u.inv_row0.xyz, ndc), + dot(u.inv_row1.xyz, ndc), + dot(u.inv_row2.xyz, ndc), + ); + + // q.z <= 0 means the ray hits the plane behind the camera (past the + // horizon of an extreme rotation) — those pixels stay background. + let in_front = f32(q.z > 1e-6); + + // Plane coords: X in [-hx, hx], Y in [-hy, hy] inside the content. + let denom = select(q.z, 1e-6, abs(q.z) <= 1e-6); + let plane = q.xy / denom; + let content_uv = vec2( + (plane.x / half_extents.x + 1.0) * 0.5, + (1.0 - plane.y / half_extents.y) * 0.5, + ); + + // Analytic edge anti-aliasing: fade coverage over one output pixel. + let width = max(fwidth(content_uv), vec2(1e-6)); + let edge = smoothstep(vec2(0.0), width, content_uv) + * smoothstep(vec2(0.0), width, vec2(1.0) - content_uv); + let coverage = edge.x * edge.y * in_front; + + let clamped = clamp(content_uv, vec2(0.0), vec2(1.0)); + // The content texture is premultiplied (straight-alpha layers composited + // onto transparent black), so coverage scales the whole texel. + let color = textureSampleLevel(t_content, s_content, clamped, 0.0); + return color * coverage; +} diff --git a/crates/rendering/src/zoom.rs b/crates/rendering/src/zoom.rs index 9623dca9bd3..be66af9e9a9 100644 --- a/crates/rendering/src/zoom.rs +++ b/crates/rendering/src/zoom.rs @@ -106,6 +106,16 @@ impl InterpolatedZoom { } } +impl Default for InterpolatedZoom { + /// The resting transform: no zoom engaged. + fn default() -> Self { + Self { + t: 0.0, + bounds: SegmentBounds::default(), + } + } +} + #[cfg(test)] mod test { use super::*; diff --git a/crates/rendering/src/zoom_spring.rs b/crates/rendering/src/zoom_spring.rs index 0ae20444077..540e71c7de2 100644 --- a/crates/rendering/src/zoom_spring.rs +++ b/crates/rendering/src/zoom_spring.rs @@ -1501,6 +1501,7 @@ mod tests { caption_segments: vec![], keyboard_segments: vec![], audio_segments: vec![], + camera3d_segments: Vec::new(), }; let cursor = CursorEvents { moves: vec![ @@ -1570,6 +1571,7 @@ mod tests { caption_segments: Vec::new(), keyboard_segments: Vec::new(), audio_segments: Vec::new(), + camera3d_segments: Vec::new(), }; let map = build_time_map(Some(&timeline)); diff --git a/packages/ui-solid/src/auto-imports.d.ts b/packages/ui-solid/src/auto-imports.d.ts index b2277e16cf6..151b31391b5 100644 --- a/packages/ui-solid/src/auto-imports.d.ts +++ b/packages/ui-solid/src/auto-imports.d.ts @@ -75,6 +75,7 @@ declare global { const IconLucideCaptions: typeof import('~icons/lucide/captions.jsx')['default'] const IconLucideCheck: typeof import('~icons/lucide/check.jsx')['default'] const IconLucideChevronDown: typeof import('~icons/lucide/chevron-down.jsx')['default'] + const IconLucideChevronRight: typeof import('~icons/lucide/chevron-right.jsx')['default'] const IconLucideChevronUp: typeof import('~icons/lucide/chevron-up.jsx')['default'] const IconLucideCircleOff: typeof import('~icons/lucide/circle-off.jsx')['default'] const IconLucideClapperboard: typeof import('~icons/lucide/clapperboard.jsx')['default']