diff --git a/.claude/skills/rustmotion/rules/captions-workflow.md b/.claude/skills/rustmotion/rules/captions-workflow.md new file mode 100644 index 0000000..853c4bb --- /dev/null +++ b/.claude/skills/rustmotion/rules/captions-workflow.md @@ -0,0 +1,85 @@ +# Rule: Captions Workflow (audio → word timings → caption component) + +Word-synced captions are a two-step pipeline: generate word timings with +`rustmotion captions`, then inject them into a scenario through the variables +system. Never hand-write word timings for real voiceovers. + +## Step 1 — Generate word timings + +```bash +# From audio (requires whisper.cpp: brew install whisper-cpp) +rustmotion captions voice.mp3 -o words.json + +# Pick a model / force the language +rustmotion captions voice.mp3 -o words.json --model small --lang fr + +# Offline import from existing subtitles (no whisper needed) +rustmotion captions --from-srt subs.srt -o words.json +rustmotion captions --from-vtt subs.vtt -o words.json +``` + +Output shape (also valid as a `--props` file, on purpose): + +```json +{ + "words": [ + { "text": "Hello", "start": 0.0, "end": 0.32 }, + { "text": "world", "start": 0.32, "end": 0.7 } + ] +} +``` + +- Transcription uses a whisper.cpp binary found in PATH (`whisper-cli`, + `whisper-cpp`, `main`). Models are resolved from + `~/.cache/whisper/ggml-.bin` or a direct `.bin` path. +- **SRT/VTT approximation**: subtitle cues carry sentence-level timing only, + so each cue's duration is spread **uniformly** across its words. Good + enough for karaoke modes; for precise word pops prefer real transcription. + +## Step 2 — Inject into the scenario + +Declare a `words` variable in `config` and reference it from the caption +component: + +```json +{ + "config": { "words": { "type": "array", "default": [] } }, + "scenes": [ + { + "duration": 5, + "children": [ + { + "type": "caption", + "mode": "word_pop", + "words": "$words", + "active_color": "#FFFFFF", + "pill_color": "#FF3366", + "position": { "x": 0, "y": 1600 }, + "style": { "width": "100%", "font-size": 72 } + } + ] + } + ] +} +``` + +```bash +rustmotion render -f scenario.json --props words.json +``` + +## Caption modes + +| Mode | Behavior | +|---|---| +| `highlight` (default) | All words visible, active word takes `active_color` | +| `karaoke` | Same as highlight (karaoke-style progression) | +| `word_by_word` | Only the active word, centered, no pill | +| `word_pop` | TikTok style: only the active word, spring scale-in, pill background | +| `karaoke_pop` | Full line visible + active word scales 1.15x with `active_color` and pill | + +- `pill_color` (word_pop / karaoke_pop): pill background behind the active + word. Defaults to black at 70% (`#000000B3`). +- `max_width`: wraps the line in highlight/karaoke/karaoke_pop modes — always + set it on vertical formats to avoid viewport overflow. +- The caption paints from its baseline: position it with enough headroom + (font-size + pill padding above the anchor point). diff --git a/crates/rustmotion-cli/src/commands/captions.rs b/crates/rustmotion-cli/src/commands/captions.rs new file mode 100644 index 0000000..1be5249 --- /dev/null +++ b/crates/rustmotion-cli/src/commands/captions.rs @@ -0,0 +1,667 @@ +//! `rustmotion captions` — generate word-level caption timings. +//! +//! Two modes: +//! - **Transcription** (default): shells out to a whisper.cpp binary +//! (`whisper-cli`, `whisper-cpp` or `main` in PATH), following the same +//! philosophy as ffmpeg: auto-detection, actionable error when absent, +//! no compiled-in C++ dependency. +//! - **Import**: `--from-srt` / `--from-vtt` parse subtitle files offline +//! with a hand-rolled parser (no dependency). Word timing inside a cue is +//! spread uniformly over the cue duration (approximation). +//! +//! Output shape (stdout or `-o`): `{"words":[{"text","start","end"}]}` — +//! directly usable as a `--props` file with a `"words": "$words"` variable +//! reference in the scenario. + +use rustmotion::error::{Result, RustmotionError}; +use serde::{Deserialize, Serialize}; +use std::path::{Path, PathBuf}; +use std::process::Command; + +/// A single timed word, as emitted in the output JSON. +#[derive(Debug, Clone, Serialize, PartialEq)] +pub struct TimedWord { + pub text: String, + pub start: f64, + pub end: f64, +} + +/// Output file shape: `{"words": [...]}`. +#[derive(Debug, Serialize)] +struct WordsFile<'a> { + words: &'a [TimedWord], +} + +/// A subtitle cue (SRT/VTT): a time window and its (already cleaned) text. +#[derive(Debug, Clone, PartialEq)] +struct Cue { + start: f64, + end: f64, + text: String, +} + +// --------------------------------------------------------------------------- +// Subtitle parsing (SRT / VTT) +// --------------------------------------------------------------------------- + +fn parse_srt(input: &str) -> Result> { + parse_cues(input, false) +} + +fn parse_vtt(input: &str) -> Result> { + parse_cues(input, true) +} + +/// Shared SRT/VTT cue parser: blocks separated by blank lines, each with a +/// `start --> end` timing line followed by (possibly multi-line) text. +fn parse_cues(input: &str, vtt: bool) -> Result> { + let input = input + .trim_start_matches('\u{feff}') + .replace("\r\n", "\n") + .replace('\r', "\n"); + + let mut cues = Vec::new(); + for block in input.split("\n\n") { + let lines: Vec<&str> = block + .lines() + .map(str::trim) + .filter(|l| !l.is_empty()) + .collect(); + let Some(first) = lines.first() else { continue }; + if vtt + && (first.starts_with("WEBVTT") + || first.starts_with("NOTE") + || first.starts_with("STYLE") + || first.starts_with("REGION")) + { + continue; + } + // The timing line is usually line 0 (VTT) or line 1 (SRT index / + // VTT cue identifier before it). + let Some(timing_idx) = lines.iter().position(|l| l.contains("-->")) else { + continue; + }; + let Some((start, end)) = parse_timing_line(lines[timing_idx]) else { + continue; + }; + let text = strip_tags(&lines[timing_idx + 1..].join(" ")); + let text = text.split_whitespace().collect::>().join(" "); + if text.is_empty() { + continue; + } + cues.push(Cue { start, end, text }); + } + + if cues.is_empty() { + return Err(RustmotionError::Generic(format!( + "no cues found in {} input — expected \"HH:MM:SS{}mmm --> HH:MM:SS{}mmm\" timing lines followed by text", + if vtt { "VTT" } else { "SRT" }, + if vtt { "." } else { "," }, + if vtt { "." } else { "," }, + ))); + } + Ok(cues) +} + +/// Parses `start --> end [cue settings...]` into a `(start, end)` pair. +fn parse_timing_line(line: &str) -> Option<(f64, f64)> { + let (left, right) = line.split_once("-->")?; + let start = parse_timestamp(left)?; + // The end timestamp may be followed by VTT cue settings. + let end = parse_timestamp(right.split_whitespace().next()?)?; + Some((start, end)) +} + +/// Parses `HH:MM:SS,mmm` (SRT), `HH:MM:SS.mmm` (VTT) or `MM:SS.mmm` +/// (VTT short form) into seconds. +fn parse_timestamp(s: &str) -> Option { + let s = s.trim(); + let (clock, frac) = match s.rsplit_once([',', '.']) { + Some((clock, millis)) => (clock, format!("0.{millis}").parse::().ok()?), + None => (s, 0.0), + }; + let parts: Vec<&str> = clock.split(':').collect(); + let (h, m, sec): (u64, u64, u64) = match parts.as_slice() { + [h, m, s] => (h.parse().ok()?, m.parse().ok()?, s.parse().ok()?), + [m, s] => (0, m.parse().ok()?, s.parse().ok()?), + _ => return None, + }; + Some(h as f64 * 3600.0 + m as f64 * 60.0 + sec as f64 + frac) +} + +/// Removes ``, ``, `` and any other angle-bracket tags. +fn strip_tags(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + let mut in_tag = false; + for c in text.chars() { + match c { + '<' => in_tag = true, + '>' if in_tag => in_tag = false, + c if !in_tag => out.push(c), + _ => {} + } + } + out +} + +/// Spreads each cue's duration uniformly across its words (approximation: +/// every word in a cue gets `duration / word_count` seconds). +fn distribute_words(cues: &[Cue]) -> Vec { + let mut words = Vec::new(); + for cue in cues { + let tokens: Vec<&str> = cue.text.split_whitespace().collect(); + if tokens.is_empty() { + continue; + } + let per_word = (cue.end - cue.start).max(0.0) / tokens.len() as f64; + for (i, token) in tokens.iter().enumerate() { + words.push(TimedWord { + text: (*token).to_string(), + start: round_ms(cue.start + i as f64 * per_word), + end: round_ms(cue.start + (i + 1) as f64 * per_word), + }); + } + } + words +} + +fn round_ms(t: f64) -> f64 { + (t * 1000.0).round() / 1000.0 +} + +/// Serializes the words to the `{"words": [...]}` JSON shape. +fn words_to_json(words: &[TimedWord]) -> String { + serde_json::to_string_pretty(&WordsFile { words }).expect("words serialize to JSON") +} + +// --------------------------------------------------------------------------- +// whisper.cpp subprocess +// --------------------------------------------------------------------------- + +/// Actionable error when no whisper.cpp binary is found in PATH. +fn missing_binary_error() -> RustmotionError { + RustmotionError::Generic( + "No whisper.cpp binary found in PATH (tried `whisper-cli`, `whisper-cpp`, `main`).\n\ + Install it with: brew install whisper-cpp\n\ + Or build it from source: https://github.com/ggml-org/whisper.cpp\n\ + Alternatively, import existing subtitles with --from-srt / --from-vtt." + .to_string(), + ) +} + +/// Actionable error when the requested model cannot be resolved. +fn missing_model_error(name: &str, searched: &[PathBuf]) -> RustmotionError { + let searched_list = searched + .iter() + .map(|p| format!(" - {}", p.display())) + .collect::>() + .join("\n"); + RustmotionError::Generic(format!( + "Whisper model '{name}' not found (looked for ggml-{name}.bin in):\n{searched_list}\n\ + Download it with:\n \ + mkdir -p ~/.cache/whisper && curl -L -o ~/.cache/whisper/ggml-{name}.bin \\\n \ + https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-{name}.bin" + )) +} + +/// Searches PATH for an executable file with the given name. +fn find_in_path(name: &str) -> Option { + let path_var = std::env::var_os("PATH")?; + std::env::split_paths(&path_var) + .map(|dir| dir.join(name)) + .find(|candidate| candidate.is_file()) +} + +/// Looks for a whisper.cpp binary in PATH (`whisper-cli`, `whisper-cpp`, +/// `main`), probing `--help` and requiring "whisper" in the help text (this +/// guards against an unrelated binary that happens to be called `main`). +fn detect_whisper_binary() -> Option { + ["whisper-cli", "whisper-cpp", "main"] + .iter() + .filter_map(|name| find_in_path(name)) + .find(|path| { + Command::new(path) + .arg("--help") + .output() + .map(|out| { + let help = format!( + "{}{}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + help.to_lowercase().contains("whisper") + }) + .unwrap_or(false) + }) +} + +/// Resolves `--model`: a direct `.bin` path, or a name searched as +/// `ggml-.bin` in `~/.cache/whisper` and next to the binary. +fn resolve_model(model: &str, binary: &Path) -> Result { + if model.ends_with(".bin") || model.contains('/') { + let as_path = Path::new(model); + if as_path.is_file() { + return Ok(as_path.to_path_buf()); + } + return Err(RustmotionError::Generic(format!( + "Whisper model file not found: {model}" + ))); + } + let file = format!("ggml-{model}.bin"); + let mut searched = Vec::new(); + if let Some(home) = std::env::var_os("HOME") { + searched.push(Path::new(&home).join(".cache").join("whisper").join(&file)); + } + if let Some(dir) = binary.parent() { + if !dir.as_os_str().is_empty() { + searched.push(dir.join(&file)); + } + } + searched + .iter() + .find(|p| p.is_file()) + .cloned() + .ok_or_else(|| missing_model_error(model, &searched)) +} + +/// whisper.cpp `-oj` output (documented format): segments under +/// `transcription`, with `offsets.{from,to}` in milliseconds. +#[derive(Deserialize)] +struct WhisperOutput { + #[serde(default)] + transcription: Vec, +} + +#[derive(Deserialize)] +struct WhisperSegment { + offsets: WhisperOffsets, + text: String, +} + +#[derive(Deserialize)] +struct WhisperOffsets { + from: u64, + to: u64, +} + +/// Parses whisper.cpp `-oj` JSON output (`transcription[].offsets.{from,to}` +/// in milliseconds, `.text`) into timed words. Punctuation-only segments +/// (produced by `-ml 1 -sow`) are glued to the previous word. +fn parse_whisper_json(json: &str) -> Result> { + let parsed: WhisperOutput = serde_json::from_str(json)?; + let mut words: Vec = Vec::new(); + for seg in parsed.transcription { + let text = seg.text.trim().to_string(); + if text.is_empty() { + continue; + } + let end = round_ms(seg.offsets.to as f64 / 1000.0); + if !text.chars().any(|c| c.is_alphanumeric()) { + // Punctuation-only segment: attach to the previous word + // (drop it when there is no previous word). + if let Some(prev) = words.last_mut() { + prev.text.push_str(&text); + prev.end = end; + } + continue; + } + words.push(TimedWord { + text, + start: round_ms(seg.offsets.from as f64 / 1000.0), + end, + }); + } + Ok(words) +} + +/// Runs the whisper.cpp binary on the audio file and returns timed words. +/// +/// Flags used (whisper.cpp CLI): +/// - `-m ` / `-f