diff --git a/.claude/skills/rustmotion/rules/audio-reactive.md b/.claude/skills/rustmotion/rules/audio-reactive.md new file mode 100644 index 0000000..d14a13b --- /dev/null +++ b/.claude/skills/rustmotion/rules/audio-reactive.md @@ -0,0 +1,131 @@ +# Audio-Reactive Components + +Rustmotion supports two built-in audio-reactive painters and a general-purpose `audio_reactive` CSS binding for any component. + +## How it works + +Before rendering frames, `analyze_scenario_audio` decodes each audio track (via symphonia), computes per-frame RMS amplitude and 16 log-spaced frequency bands (20 Hz–16 kHz) using a Hann-windowed FFT, normalizes the values to 0..1 across the whole track, and stores the result in a global `AudioAnalysis` cache keyed by `src`. Graceful degradation: if a track cannot be decoded (file not found, corrupt), it is silently skipped and all audio bindings fall back to their `min` value. + +## audio_spectrum component + +A frequency spectrum visualizer using the 16 internal bands. + +```json +{ + "type": "audio_spectrum", + "track": "music.mp3", + "bars": 32, + "mode": "bars", + "color": "#38bdf8", + "bar_gap": 3, + "min_height": 2, + "style": { "width": "600px", "height": "160px" } +} +``` + +Fields: +- `track` (optional): src path of the audio track. When absent, the first cached track is used. +- `bars` (default 16): number of bars displayed. Bands are resampled by linear interpolation from the 16 internal bands. +- `mode`: `"bars"` (vertical bars from bottom) or `"radial"` (circular spoke pattern). +- `color`: hex color string. +- `bar_gap` (default 2.0): gap in pixels between bars. +- `min_height` (default 2.0): minimum bar height in pixels (so bars are always visible). +- Default size: 400×120 px (override via `style.width`/`style.height`). + +## waveform component + +A time-domain waveform of the amplitude signal, centered on the current playhead. + +```json +{ + "type": "waveform", + "track": "music.mp3", + "color": "#38bdf8", + "draw_style": "filled", + "window": 3.0, + "style": { "width": "800px", "height": "100px" } +} +``` + +Fields: +- `track` (optional): src path. Falls back to first cached track. +- `color`: hex color. +- `draw_style`: `"line"` (single stroke path) or `"filled"` (semi-transparent fill + outline). +- `window` (default 2.0): time window in seconds, centered on the current frame time. +- Default size: 400×80 px. + +When no cached analysis exists (graceful degradation), both components render a flat line / empty bars at minimum height rather than panicking. + +## audio_reactive CSS binding + +Any component's `style` block can include an `audio-reactive` binding (kebab-case key — `audio_reactive` is rejected by the schema) that lerps a CSS property between `min` and `max` based on audio data: + +```json +{ + "type": "shape", + "shape": "rect", + "fill": "#ff3366", + "style": { + "width": "100px", + "height": "100px", + "audio-reactive": { + "track": "music.mp3", + "source": "amplitude", + "property": "opacity", + "min": 0.2, + "max": 1.0, + "smoothing_frames": 3 + } + } +} +``` + +### Fields + +| Field | Type | Description | +|---|---|---| +| `track` | string? | Audio track src key. Falls back to first cached track when absent. | +| `source` | `"amplitude"` or `{"band": N}` | `"amplitude"` → overall RMS; `{"band": N}` → N-th frequency band (0..15). | +| `property` | string | CSS property to drive: `"opacity"`, `"scale"`, `"translate_y"`, `"rotation"`. | +| `min` | f64 | Value at audio=0. | +| `max` | f64 | Value at audio=1. | +| `smoothing_frames` | u32 (default 0) | Number of past frames to average (temporal smoothing). | + +### Supported properties + +- **`opacity`**: multiplied with the base opacity (0..1 range, clamped). +- **`scale`**: uniform 2D scale applied via CSS transform. `min: 0.5, max: 1.5` pulses the element. +- **`translate_y`**: vertical translation in px (negative = up). Useful for bouncing elements. +- **`rotation`**: rotation in degrees. + +### Notes + +- The binding is applied at box-tree build time (before taffy layout), so `opacity` and `transform` flow through the standard CSS paint pass. +- `scale` and `transform`-based properties are appended to the existing transform list (they compose). +- `smoothing_frames: 3` at 30 fps ≈ 100ms smoothing, which removes most percussive jitter while keeping reactivity. +- Cache miss (no audio or track not found) → all reactive values fall back to `min` (deterministic, no flicker). + +## band indices + +The 16 bands span 20 Hz to 16 kHz, log-spaced. Approximate centers: + +| Index | ~Center Hz | +|---|---| +| 0 | 24 | +| 1 | 36 | +| 2 | 55 | +| 3 | 82 | +| 4 | 124 | +| 5 | 186 | +| 6 | 279 | +| 7 | 419 | +| 8 | 629 | +| 9 | 944 | +| 10 | 1416 | +| 11 | 2125 | +| 12 | 3190 | +| 13 | 4790 | +| 14 | 7190 | +| 15 | 10795 | + +Use bands 0–3 for bass, 4–7 for low-mid, 8–11 for mid, 12–15 for high. diff --git a/Cargo.lock b/Cargo.lock index ea2d879..274c8da 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4917,6 +4917,7 @@ dependencies = [ "rayon", "resvg", "rubato", + "rustfft", "rustmotion-components", "rustmotion-core", "rustmotion-html", diff --git a/crates/rustmotion-cli/src/commands/geometry.rs b/crates/rustmotion-cli/src/commands/geometry.rs index 0f40eab..5f515dd 100644 --- a/crates/rustmotion-cli/src/commands/geometry.rs +++ b/crates/rustmotion-cli/src/commands/geometry.rs @@ -428,6 +428,8 @@ fn component_kind(c: &Component) -> &'static str { Component::Grid(_) => "grid", Component::Card(_) => "card", Component::Container(_) => "container", + Component::AudioSpectrum(_) => "audio_spectrum", + Component::Waveform(_) => "waveform", } } diff --git a/crates/rustmotion-components/src/audio_spectrum.rs b/crates/rustmotion-components/src/audio_spectrum.rs new file mode 100644 index 0000000..cfc90a7 --- /dev/null +++ b/crates/rustmotion-components/src/audio_spectrum.rs @@ -0,0 +1,158 @@ +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use skia_safe::{Canvas, Color, Paint, PaintStyle, Point, Rect}; + +use rustmotion_core::css::CssStyle; +use rustmotion_core::engine::animator::AnimatedProperties; +use rustmotion_core::engine::layout_pass::BoxLayout; +use rustmotion_core::engine::renderer::audio_analysis::audio_analysis_cache; +use rustmotion_core::engine::renderer::parse_hex_color; +use rustmotion_core::schema::TimelineStep; +use rustmotion_core::traits::{PaintCtx, Painter, TimingConfig}; + +fn default_bars() -> u32 { + 16 +} +fn default_color() -> String { + "#38bdf8".to_string() +} +fn default_bar_gap() -> f32 { + 2.0 +} +fn default_min_height() -> f32 { + 2.0 +} + +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum SpectrumMode { + #[default] + Bars, + Radial, +} + +#[derive(Debug, Serialize, Deserialize, JsonSchema)] +pub struct AudioSpectrum { + /// Source audio track (src path). If None, uses the first track in the cache. + #[serde(default)] + pub track: Option, + /// Number of frequency bars to display (resampled from 16 internal bands). + #[serde(default = "default_bars")] + pub bars: u32, + /// Display mode: bars (vertical) or radial. + #[serde(default)] + pub mode: SpectrumMode, + /// Bar color as hex string. + #[serde(default = "default_color")] + pub color: String, + /// Gap between bars in pixels. + #[serde(default = "default_bar_gap")] + pub bar_gap: f32, + /// Minimum bar height in pixels. + #[serde(default = "default_min_height")] + pub min_height: f32, + #[serde(flatten)] + pub timing: TimingConfig, + #[serde(default)] + pub style: CssStyle, + #[serde(default)] + pub timeline: Vec, + #[serde(default)] + pub stagger: Option, +} + +rustmotion_core::impl_traits!(AudioSpectrum { + Animatable => animation, + Timed => timing, + Styled => style, +}); + +impl AudioSpectrum { + fn get_band_values(&self, time: f64) -> Vec { + let cache = audio_analysis_cache(); + let analysis = if let Some(ref src) = self.track { + cache.get(src).map(|r| r.clone()) + } else { + cache.iter().next().map(|r| r.value().clone()) + }; + + let n = self.bars.max(1) as usize; + + // Empty cache / missing track → all-zero values; the painter clamps + // each bar to `min_height`, so this degrades to a flat baseline. + let Some(analysis) = analysis else { + return vec![0.0; n]; + }; + + // Resample from 16 bands to n bars + let num_bands = 16usize; + (0..n) + .map(|i| { + let band_f = i as f32 * (num_bands as f32 - 1.0) / (n as f32 - 1.0).max(1.0); + let band_lo = band_f as usize; + let band_hi = (band_lo + 1).min(num_bands - 1); + let frac = band_f - band_lo as f32; + let v_lo = analysis.band_at(time, band_lo as u8); + let v_hi = analysis.band_at(time, band_hi as u8); + v_lo * (1.0 - frac) + v_hi * frac + }) + .collect() + } +} + +impl Painter for AudioSpectrum { + fn paint_content( + &self, + canvas: &Canvas, + layout: &BoxLayout, + _props: &AnimatedProperties, + ctx: &PaintCtx, + ) { + let w = layout.width; + let h = layout.height; + let n = self.bars.max(1) as usize; + let values = self.get_band_values(ctx.time); + let (r, g, b, a) = parse_hex_color(&self.color); + let color = Color::from_argb(a, r, g, b); + + match self.mode { + SpectrumMode::Bars => { + let total_gap = self.bar_gap * (n as f32 - 1.0); + let bar_w = ((w - total_gap) / n as f32).max(1.0); + let mut paint = Paint::default(); + paint.set_color(color); + paint.set_style(PaintStyle::Fill); + paint.set_anti_alias(true); + + for (i, &v) in values.iter().enumerate() { + let bar_h = (v * h).max(self.min_height); + let x = i as f32 * (bar_w + self.bar_gap); + let y = h - bar_h; + canvas.draw_rect(Rect::from_xywh(x, y, bar_w, bar_h), &paint); + } + } + SpectrumMode::Radial => { + let cx = w / 2.0; + let cy = h / 2.0; + let max_r = cx.min(cy); + let inner_r = max_r * 0.3; + + let mut paint = Paint::default(); + paint.set_color(color); + paint.set_style(PaintStyle::Stroke); + paint.set_stroke_width(2.0); + paint.set_anti_alias(true); + + for (i, &v) in values.iter().enumerate() { + let angle = (i as f32 / n as f32) * 2.0 * std::f32::consts::PI; + let bar_len = (v * (max_r - inner_r)).max(self.min_height); + let x0 = cx + inner_r * angle.cos(); + let y0 = cy + inner_r * angle.sin(); + let x1 = cx + (inner_r + bar_len) * angle.cos(); + let y1 = cy + (inner_r + bar_len) * angle.sin(); + canvas.draw_line(Point::new(x0, y0), Point::new(x1, y1), &paint); + } + } + } + } +} diff --git a/crates/rustmotion-components/src/box_builder.rs b/crates/rustmotion-components/src/box_builder.rs index 9f9d0b6..2c15856 100644 --- a/crates/rustmotion-components/src/box_builder.rs +++ b/crates/rustmotion-components/src/box_builder.rs @@ -215,6 +215,61 @@ fn build_child<'a>( } } + // ── Audio-reactive binding ──────────────────────────────────────────────── + // Reads the audio analysis cache and lerps the target CSS property between + // min and max. Cache-miss → value = min (deterministic fallback). + if let Some(ar) = css.audio_reactive.take() { + use rustmotion_core::css::style::{AudioReactiveProperty, AudioSource, AudioSourceTag}; + use rustmotion_core::engine::renderer::audio_analysis::audio_analysis_cache; + + if let Some(actx) = anim { + let cache = audio_analysis_cache(); + let analysis_opt = if let Some(ref src) = ar.track { + cache.get(src).map(|r| r.clone()) + } else { + cache.iter().next().map(|r| r.value().clone()) + }; + + let raw = if let Some(analysis) = analysis_opt { + match &ar.source { + AudioSource::Amplitude(AudioSourceTag::Amplitude) => { + analysis.amplitude_smoothed(actx.time, ar.smoothing_frames) + } + AudioSource::Band { band } => { + analysis.band_smoothed(actx.time, *band, ar.smoothing_frames) + } + } + } else { + 0.0 // cache empty → use min + }; + + let lerped = ar.min as f32 + raw * (ar.max - ar.min) as f32; + + match ar.property { + AudioReactiveProperty::Opacity => { + let base = css.opacity.unwrap_or(1.0); + css.opacity = Some(base * lerped.clamp(0.0, 1.0)); + } + AudioReactiveProperty::Scale => { + let s = lerped.max(0.0); + let tx = css.transform.get_or_insert_with(Vec::new); + tx.push(rustmotion_core::css::style::TransformFn::Scale { x: s, y: s }); + } + AudioReactiveProperty::TranslateY => { + use rustmotion_core::css::units::LengthPercentage; + let tx = css.transform.get_or_insert_with(Vec::new); + tx.push(rustmotion_core::css::style::TransformFn::TranslateY { + y: LengthPercentage::Px(lerped), + }); + } + AudioReactiveProperty::Rotation => { + let tx = css.transform.get_or_insert_with(Vec::new); + tx.push(rustmotion_core::css::style::TransformFn::Rotate { deg: lerped }); + } + } + } + } + // Visibility window (start_at/end_at) — enforced by the paint pass. // The stagger delay shifts the window too, so a hard-cut child appears // in step with its staggered siblings. @@ -768,6 +823,22 @@ fn apply_intrinsic_overrides(component: &Component, css: &mut CssStyle) { css.height = Some(CSize::Length(CLP::Px(box_h))); } } + AudioSpectrum(_) => { + if css.width.is_none() { + css.width = Some(CSize::Length(CLP::Px(400.0))); + } + if css.height.is_none() { + css.height = Some(CSize::Length(CLP::Px(120.0))); + } + } + Waveform(_) => { + if css.width.is_none() { + css.width = Some(CSize::Length(CLP::Px(400.0))); + } + if css.height.is_none() { + css.height = Some(CSize::Length(CLP::Px(80.0))); + } + } _ => {} } } @@ -831,6 +902,8 @@ fn component_style(c: &Component) -> &CssStyle { Grid(c) => &c.style, Card(c) => &c.style, Container(c) => &c.style, + AudioSpectrum(c) => &c.style, + Waveform(c) => &c.style, } } @@ -893,6 +966,8 @@ pub fn component_kind(c: &Component) -> &'static str { Grid(_) => "grid", Card(_) => "card", Container(_) => "container", + AudioSpectrum(_) => "audio_spectrum", + Waveform(_) => "waveform", } } diff --git a/crates/rustmotion-components/src/lib.rs b/crates/rustmotion-components/src/lib.rs index 4c5ba18..6e26112 100644 --- a/crates/rustmotion-components/src/lib.rs +++ b/crates/rustmotion-components/src/lib.rs @@ -3,6 +3,7 @@ pub mod intrinsic; pub mod legacy_dispatch; pub mod arrow; +pub mod audio_spectrum; pub mod avatar; pub mod avatar_group; pub mod badge; @@ -57,6 +58,7 @@ pub mod timeline; pub mod tooltip; pub mod treemap; pub mod video; +pub mod waveform; pub mod world_bitmap; use schemars::JsonSchema; @@ -65,6 +67,7 @@ use serde::{Deserialize, Serialize}; use rustmotion_core::traits::{Animatable, Painter, Styled, Timed}; pub use arrow::Arrow; +pub use audio_spectrum::AudioSpectrum; pub use avatar::Avatar; pub use avatar_group::AvatarGroup; pub use badge::Badge; @@ -119,6 +122,7 @@ pub use timeline::Timeline; pub use tooltip::Tooltip; pub use treemap::Treemap; pub use video::Video; +pub use waveform::Waveform; // --- Position mode --- @@ -176,6 +180,7 @@ impl ChildComponent { #[derive(Debug, Serialize, Deserialize, JsonSchema)] #[serde(tag = "type", rename_all = "snake_case")] pub enum Component { + AudioSpectrum(AudioSpectrum), Text(Text), Shape(Shape), Image(Image), @@ -233,6 +238,7 @@ pub enum Component { Card(Card), #[serde(rename = "div", alias = "container")] Container(ContainerComponent), + Waveform(Waveform), } // --- Dispatch helpers --- @@ -240,6 +246,8 @@ pub enum Component { impl Component { pub fn as_animatable(&self) -> Option<&dyn Animatable> { match self { + Component::AudioSpectrum(c) => Some(c), + Component::Waveform(c) => Some(c), Component::Text(c) => Some(c), Component::Shape(c) => Some(c), Component::Image(c) => Some(c), @@ -300,6 +308,8 @@ impl Component { pub fn as_timed(&self) -> Option<&dyn Timed> { match self { + Component::AudioSpectrum(c) => Some(c), + Component::Waveform(c) => Some(c), Component::Text(c) => Some(c), Component::Shape(c) => Some(c), Component::Image(c) => Some(c), @@ -360,6 +370,8 @@ impl Component { pub fn as_styled(&self) -> &dyn Styled { match self { + Component::AudioSpectrum(c) => c, + Component::Waveform(c) => c, Component::Text(c) => c, Component::Shape(c) => c, Component::Image(c) => c, @@ -422,6 +434,8 @@ impl Component { /// pipeline; the dispatcher always uses Painter::paint_content. pub fn as_painter(&self) -> Option<&dyn Painter> { match self { + Component::AudioSpectrum(c) => Some(c), + Component::Waveform(c) => Some(c), Component::Card(c) => Some(c), Component::Container(c) => Some(c), Component::Flex(c) => Some(c), diff --git a/crates/rustmotion-components/src/waveform.rs b/crates/rustmotion-components/src/waveform.rs new file mode 100644 index 0000000..e216b72 --- /dev/null +++ b/crates/rustmotion-components/src/waveform.rs @@ -0,0 +1,159 @@ +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use skia_safe::{Canvas, Color, Paint, PaintStyle, Path}; + +use rustmotion_core::css::CssStyle; +use rustmotion_core::engine::animator::AnimatedProperties; +use rustmotion_core::engine::layout_pass::BoxLayout; +use rustmotion_core::engine::renderer::audio_analysis::audio_analysis_cache; +use rustmotion_core::engine::renderer::parse_hex_color; +use rustmotion_core::schema::TimelineStep; +use rustmotion_core::traits::{PaintCtx, Painter, TimingConfig}; + +fn default_color() -> String { + "#38bdf8".to_string() +} +fn default_window() -> f32 { + 2.0 +} + +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum DrawStyle { + #[default] + Line, + Filled, +} + +#[derive(Debug, Serialize, Deserialize, JsonSchema)] +pub struct Waveform { + /// Source audio track (src path). If None, uses the first track in the cache. + #[serde(default)] + pub track: Option, + /// Waveform color as hex string. + #[serde(default = "default_color")] + pub color: String, + /// Visual style: line or filled. + #[serde(default)] + pub draw_style: DrawStyle, + /// Time window in seconds, centered on ctx.time. + #[serde(default = "default_window")] + pub window: f32, + #[serde(flatten)] + pub timing: TimingConfig, + #[serde(default)] + pub style: CssStyle, + #[serde(default)] + pub timeline: Vec, + #[serde(default)] + pub stagger: Option, +} + +rustmotion_core::impl_traits!(Waveform { + Animatable => animation, + Timed => timing, + Styled => style, +}); + +impl Painter for Waveform { + fn paint_content( + &self, + canvas: &Canvas, + layout: &BoxLayout, + _props: &AnimatedProperties, + ctx: &PaintCtx, + ) { + let w = layout.width; + let h = layout.height; + let (r, g, b, a) = parse_hex_color(&self.color); + let color = Color::from_argb(a, r, g, b); + + let cache = audio_analysis_cache(); + let analysis = if let Some(ref src) = self.track { + cache.get(src).map(|r| r.clone()) + } else { + cache.iter().next().map(|r| r.value().clone()) + }; + + let Some(analysis) = analysis else { + // Graceful degradation: flat line at mid-height + let mut paint = Paint::default(); + paint.set_color(color); + paint.set_style(PaintStyle::Stroke); + paint.set_stroke_width(1.0); + paint.set_anti_alias(true); + canvas.draw_line( + skia_safe::Point::new(0.0, h / 2.0), + skia_safe::Point::new(w, h / 2.0), + &paint, + ); + return; + }; + + let half_window = self.window as f64 / 2.0; + let t_start = (ctx.time - half_window).max(0.0); + let t_end = ctx.time + half_window; + + // Sample N points along the window + let n_points = w as usize; + let mut points: Vec<(f32, f32)> = Vec::with_capacity(n_points); + for i in 0..n_points { + let t = t_start + (i as f64 / n_points.max(1) as f64) * (t_end - t_start); + let amp = analysis.amplitude_at(t); + let x = i as f32; + let y = h / 2.0 - amp * (h / 2.0 - 2.0); + points.push((x, y)); + } + + let mut paint = Paint::default(); + paint.set_color(color); + paint.set_anti_alias(true); + + match self.draw_style { + DrawStyle::Line => { + paint.set_style(PaintStyle::Stroke); + paint.set_stroke_width(1.5); + if points.len() < 2 { + return; + } + let mut path = Path::new(); + path.move_to((points[0].0, points[0].1)); + for &(x, y) in &points[1..] { + path.line_to((x, y)); + } + canvas.draw_path(&path, &paint); + } + DrawStyle::Filled => { + // Filled area + let (r2, g2, b2, _) = parse_hex_color(&self.color); + let fill_color = Color::from_argb(100, r2, g2, b2); + paint.set_style(PaintStyle::Fill); + paint.set_color(fill_color); + if points.is_empty() { + return; + } + let mut fill_path = Path::new(); + fill_path.move_to((0.0, h / 2.0)); + for &(x, y) in &points { + fill_path.line_to((x, y)); + } + fill_path.line_to((w, h / 2.0)); + fill_path.close(); + canvas.draw_path(&fill_path, &paint); + + // Outline + paint.set_style(PaintStyle::Stroke); + paint.set_stroke_width(1.5); + paint.set_color(color); + if points.len() >= 2 { + let mut outline = Path::new(); + outline.move_to((points[0].0, points[0].1)); + for &(x, y) in &points[1..] { + outline.line_to((x, y)); + } + canvas.draw_path(&outline, &paint); + } + } + } + } +} diff --git a/crates/rustmotion-core/src/css/style.rs b/crates/rustmotion-core/src/css/style.rs index 5c59e38..fd27621 100644 --- a/crates/rustmotion-core/src/css/style.rs +++ b/crates/rustmotion-core/src/css/style.rs @@ -105,6 +105,10 @@ pub struct CssStyle { /// `opacity`, and `color` on text/counter; everything else snaps at the /// step's `at`. pub transition: Option, + + // ---- Audio reactive binding ---- + #[serde(default)] + pub audio_reactive: Option, } /// `transition` config: bare number = duration in seconds with the default @@ -124,6 +128,55 @@ fn default_transition_easing() -> crate::schema::EasingType { crate::schema::EasingType::EaseInOut } +// ---- Audio reactive binding ---- + +/// Bind a CSS property to audio analysis data. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct AudioReactive { + /// Audio track src key. If None, uses the first entry in the cache. + #[serde(default)] + pub track: Option, + /// Which audio data source to use. + pub source: AudioSource, + /// Which CSS property to modulate. + pub property: AudioReactiveProperty, + /// Value when audio is at 0. + pub min: f64, + /// Value when audio is at 1. + pub max: f64, + /// Number of previous frames to average (0 = no smoothing). + #[serde(default)] + pub smoothing_frames: u32, +} + +/// The audio data source for an AudioReactive binding. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(untagged)] +pub enum AudioSource { + /// Overall amplitude (RMS). JSON: `"amplitude"`. + Amplitude(AudioSourceTag), + /// A specific frequency band (0..15). JSON: `{"band": 3}`. + Band { band: u8 }, +} + +/// Tag-only variant for the amplitude source. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum AudioSourceTag { + Amplitude, +} + +/// Which CSS property to modulate with audio. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum AudioReactiveProperty { + Opacity, + Scale, + TranslateY, + Rotation, +} + impl StyleTransition { pub fn duration(&self) -> f64 { match self { diff --git a/crates/rustmotion-core/src/engine/renderer/audio_analysis.rs b/crates/rustmotion-core/src/engine/renderer/audio_analysis.rs new file mode 100644 index 0000000..5d402c4 --- /dev/null +++ b/crates/rustmotion-core/src/engine/renderer/audio_analysis.rs @@ -0,0 +1,73 @@ +use dashmap::DashMap; +use std::sync::{Arc, OnceLock}; + +/// Per-track audio analysis: amplitude (RMS) and frequency bands per video frame. +#[derive(Debug, Clone)] +pub struct AudioAnalysis { + /// The video frame rate this analysis was computed at. + pub frame_rate: u32, + /// RMS amplitude per frame, normalized 0..1 over the whole track. + pub amplitude: Vec, + /// 16 log-spaced frequency bands (20 Hz–16 kHz) per frame, normalized 0..1. + pub bands: Vec<[f32; 16]>, +} + +impl AudioAnalysis { + /// Get the amplitude at a given time (seconds), clamped to valid range. + pub fn amplitude_at(&self, time: f64) -> f32 { + let idx = (time * self.frame_rate as f64) as usize; + self.amplitude + .get(idx.min(self.amplitude.len().saturating_sub(1))) + .copied() + .unwrap_or(0.0) + } + + /// Get a specific band [0..16) value at a given time. + pub fn band_at(&self, time: f64, band: u8) -> f32 { + let idx = (time * self.frame_rate as f64) as usize; + let idx = idx.min(self.bands.len().saturating_sub(1)); + self.bands + .get(idx) + .map(|b| b[band.min(15) as usize]) + .unwrap_or(0.0) + } + + /// Smoothed amplitude at time: average over the past `smoothing_frames` frames (inclusive of current). + pub fn amplitude_smoothed(&self, time: f64, smoothing_frames: u32) -> f32 { + if smoothing_frames == 0 || self.amplitude.is_empty() { + return self.amplitude_at(time); + } + let end_idx = (time * self.frame_rate as f64) as usize; + let end_idx = end_idx.min(self.amplitude.len().saturating_sub(1)); + let start_idx = end_idx.saturating_sub(smoothing_frames as usize); + let window = &self.amplitude[start_idx..=end_idx]; + if window.is_empty() { + return 0.0; + } + window.iter().sum::() / window.len() as f32 + } + + /// Smoothed band value at time over `smoothing_frames` frames. + pub fn band_smoothed(&self, time: f64, band: u8, smoothing_frames: u32) -> f32 { + if smoothing_frames == 0 || self.bands.is_empty() { + return self.band_at(time, band); + } + let end_idx = (time * self.frame_rate as f64) as usize; + let end_idx = end_idx.min(self.bands.len().saturating_sub(1)); + let start_idx = end_idx.saturating_sub(smoothing_frames as usize); + let window = &self.bands[start_idx..=end_idx]; + if window.is_empty() { + return 0.0; + } + window.iter().map(|b| b[band.min(15) as usize]).sum::() / window.len() as f32 + } +} + +type AudioCacheMap = Arc>>; + +static AUDIO_ANALYSIS_CACHE: OnceLock = OnceLock::new(); + +/// Global audio analysis cache: keyed by the audio track's `src` path. +pub fn audio_analysis_cache() -> &'static AudioCacheMap { + AUDIO_ANALYSIS_CACHE.get_or_init(|| Arc::new(DashMap::new())) +} diff --git a/crates/rustmotion-core/src/engine/renderer/mod.rs b/crates/rustmotion-core/src/engine/renderer/mod.rs index 5684413..5a8e20b 100644 --- a/crates/rustmotion-core/src/engine/renderer/mod.rs +++ b/crates/rustmotion-core/src/engine/renderer/mod.rs @@ -1,4 +1,5 @@ mod assets; +pub mod audio_analysis; mod colors; mod fonts; pub mod google_fonts; @@ -7,6 +8,7 @@ mod text; mod yuv; pub use assets::*; +pub use audio_analysis::*; pub use colors::*; pub use fonts::*; pub use shapes::*; diff --git a/crates/rustmotion-studio/src/app/mod.rs b/crates/rustmotion-studio/src/app/mod.rs index 43c42e3..93a355b 100644 --- a/crates/rustmotion-studio/src/app/mod.rs +++ b/crates/rustmotion-studio/src/app/mod.rs @@ -69,6 +69,7 @@ pub fn run_preview_root( engine::prefetch_icons(&view.scenes); engine::preextract_video_frames(&view.scenes, scenario.video.fps); } + rustmotion::encode::audio_analysis::analyze_scenario_audio(&scenario); if !scenario.fonts.is_empty() { engine::renderer::load_custom_fonts(&scenario.fonts); } diff --git a/crates/rustmotion/Cargo.toml b/crates/rustmotion/Cargo.toml index ce36e78..76b0bb4 100644 --- a/crates/rustmotion/Cargo.toml +++ b/crates/rustmotion/Cargo.toml @@ -30,6 +30,7 @@ resvg = "0.44" usvg = "0.44" tiny-skia = "0.11" ureq = "3" +rustfft = "6" [features] # Opt-in: integration tests that shell out to a real ffmpeg binary. diff --git a/crates/rustmotion/src/encode/audio.rs b/crates/rustmotion/src/encode/audio.rs index dceaa5a..28fde51 100644 --- a/crates/rustmotion/src/encode/audio.rs +++ b/crates/rustmotion/src/encode/audio.rs @@ -14,7 +14,7 @@ const TARGET_SAMPLE_RATE: u32 = 48000; const TARGET_CHANNELS: u32 = 2; /// Decode an audio file into PCM i16 samples (stereo, 44100Hz, interleaved) -fn decode_audio_file(path: &str) -> Result<(Vec, u32, u32)> { +pub(crate) fn decode_audio_file(path: &str) -> Result<(Vec, u32, u32)> { let file = File::open(path).map_err(|e| RustmotionError::AudioOpen { path: path.to_string(), reason: e.to_string(), diff --git a/crates/rustmotion/src/encode/audio_analysis.rs b/crates/rustmotion/src/encode/audio_analysis.rs new file mode 100644 index 0000000..bf0a54e --- /dev/null +++ b/crates/rustmotion/src/encode/audio_analysis.rs @@ -0,0 +1,149 @@ +use std::sync::Arc; + +use rustfft::{num_complex::Complex, FftPlanner}; +use rustmotion_core::engine::renderer::audio_analysis::{audio_analysis_cache, AudioAnalysis}; +use rustmotion_core::schema::ResolvedScenario; + +const FFT_SIZE: usize = 2048; +const NUM_BANDS: usize = 16; + +/// Build the 16 log-spaced band frequency boundaries (Hz) from 20..16000. +fn band_boundaries() -> [(f32, f32); NUM_BANDS] { + let mut bounds = [(0.0f32, 0.0f32); NUM_BANDS]; + let low = 20.0f32.log2(); + let high = 16000.0f32.log2(); + for (i, bound) in bounds.iter_mut().enumerate() { + let lo = 2.0f32.powf(low + (high - low) * i as f32 / NUM_BANDS as f32); + let hi = 2.0f32.powf(low + (high - low) * (i + 1) as f32 / NUM_BANDS as f32); + *bound = (lo, hi); + } + bounds +} + +/// Compute a Hann window of length `n`. +fn hann_window(n: usize) -> Vec { + (0..n) + .map(|i| 0.5 * (1.0 - (2.0 * std::f32::consts::PI * i as f32 / (n - 1) as f32).cos())) + .collect() +} + +/// Analyze all audio tracks in the scenario and populate the global cache. +/// Already-cached tracks are skipped (idempotent). +pub fn analyze_scenario_audio(scenario: &ResolvedScenario) { + let tracks = &scenario.audio; + if tracks.is_empty() { + return; + } + let fps = scenario.video.fps; + let cache = audio_analysis_cache(); + let band_bounds = band_boundaries(); + let hann = hann_window(FFT_SIZE); + let mut planner = FftPlanner::::new(); + let fft = planner.plan_fft_forward(FFT_SIZE); + + for track in tracks { + let src = &track.src; + if cache.contains_key(src) { + continue; + } + + // Decode to PCM f32 + let (samples, sample_rate, channels) = match crate::encode::audio::decode_audio_file(src) { + Ok(v) => v, + Err(_) => continue, // graceful degradation: skip undecodable track + }; + + // Downmix to mono + let mono: Vec = match channels { + 1 => samples.clone(), + _ => samples + .chunks(channels as usize) + .map(|c| c.iter().sum::() / channels as f32) + .collect(), + }; + + let samples_per_frame = (sample_rate as f64 / fps as f64).ceil() as usize; + let num_frames = (mono.len() as f64 / samples_per_frame as f64).ceil() as usize; + + let mut amplitude = Vec::with_capacity(num_frames); + let mut bands_all: Vec<[f32; NUM_BANDS]> = Vec::with_capacity(num_frames); + + for frame_idx in 0..num_frames { + let start = frame_idx * samples_per_frame; + let end = (start + samples_per_frame).min(mono.len()); + let frame_samples = &mono[start..end]; + + // RMS amplitude + let rms = if frame_samples.is_empty() { + 0.0f32 + } else { + (frame_samples.iter().map(|s| s * s).sum::() / frame_samples.len() as f32) + .sqrt() + }; + amplitude.push(rms); + + // FFT: take FFT_SIZE samples from this frame (with zero-padding) + let mut buf: Vec> = (0..FFT_SIZE) + .map(|i| { + let s = if i < frame_samples.len() { + frame_samples[i] + } else { + 0.0 + }; + Complex { + re: s * hann[i], + im: 0.0, + } + }) + .collect(); + fft.process(&mut buf); + + // Map FFT bins to bands + let bin_hz = sample_rate as f32 / FFT_SIZE as f32; + let half = FFT_SIZE / 2; + let mut frame_bands = [0.0f32; NUM_BANDS]; + for (b, &(lo, hi)) in band_bounds.iter().enumerate() { + let lo_bin = (lo / bin_hz) as usize; + let hi_bin = ((hi / bin_hz) as usize + 1).min(half); + let lo_bin = lo_bin.min(half); + if lo_bin >= hi_bin { + frame_bands[b] = 0.0; + continue; + } + let energy: f32 = buf[lo_bin..hi_bin].iter().map(|c| c.norm()).sum::() + / (hi_bin - lo_bin) as f32; + frame_bands[b] = energy; + } + bands_all.push(frame_bands); + } + + // Normalize amplitude to 0..1 + let amp_max = amplitude.iter().cloned().fold(0.0f32, f32::max); + if amp_max > 1e-8 { + for a in &mut amplitude { + *a /= amp_max; + } + } + + // Normalize bands with a single global max so cross-band energy + // ratios stay meaningful (a quiet band stays quiet on screen). + let bands_max = bands_all + .iter() + .flat_map(|fr| fr.iter().copied()) + .fold(0.0f32, f32::max); + if bands_max > 1e-8 { + for fr in &mut bands_all { + for v in fr.iter_mut() { + *v /= bands_max; + } + } + } + + let analysis = Arc::new(AudioAnalysis { + frame_rate: fps, + amplitude, + bands: bands_all, + }); + cache.insert(src.clone(), analysis); + } +} diff --git a/crates/rustmotion/src/encode/mod.rs b/crates/rustmotion/src/encode/mod.rs index c245d05..e1079df 100644 --- a/crates/rustmotion/src/encode/mod.rs +++ b/crates/rustmotion/src/encode/mod.rs @@ -1,4 +1,5 @@ pub mod audio; +pub mod audio_analysis; pub mod video; pub mod video_audio; diff --git a/crates/rustmotion/src/encode/video/ffmpeg.rs b/crates/rustmotion/src/encode/video/ffmpeg.rs index 3d6cda4..af6fa2e 100644 --- a/crates/rustmotion/src/encode/video/ffmpeg.rs +++ b/crates/rustmotion/src/encode/video/ffmpeg.rs @@ -2,6 +2,7 @@ use rayon::prelude::*; use std::io::Write; use std::sync::atomic::{AtomicU32, Ordering}; +use crate::encode::audio_analysis::analyze_scenario_audio; use crate::engine::prefetch_icons; use crate::error::{Result, RustmotionError}; use crate::schema::ResolvedScenario as Scenario; @@ -27,6 +28,7 @@ pub fn encode_with_ffmpeg( for view in &scenario.views { prefetch_icons(&view.scenes); } + analyze_scenario_audio(scenario); let tasks = build_frame_tasks(scenario); let total_frames = tasks.len() as u32; diff --git a/crates/rustmotion/src/encode/video/h264.rs b/crates/rustmotion/src/encode/video/h264.rs index 832a8aa..ec0abc7 100644 --- a/crates/rustmotion/src/encode/video/h264.rs +++ b/crates/rustmotion/src/encode/video/h264.rs @@ -4,6 +4,7 @@ use openh264::OpenH264API; use rayon::prelude::*; use std::sync::atomic::{AtomicU32, Ordering}; +use crate::encode::audio_analysis::analyze_scenario_audio; use crate::engine::{preextract_video_frames, prefetch_icons, rgba_to_yuv420}; use crate::error::{Result, RustmotionError}; use crate::schema::ResolvedScenario as Scenario; @@ -38,6 +39,7 @@ pub fn encode_video( preextract_video_frames(&view.scenes, fps); prefetch_icons(&view.scenes); } + analyze_scenario_audio(scenario); let tasks = build_frame_tasks(scenario); let total_frames = tasks.len() as u32; @@ -128,6 +130,7 @@ pub fn encode_video_incremental( preextract_video_frames(&view.scenes, fps); prefetch_icons(&view.scenes); } + analyze_scenario_audio(scenario); let num_slots = slots.len(); let scene_hashes: Vec = slots diff --git a/crates/rustmotion/src/tests.rs b/crates/rustmotion/src/tests.rs index 92d290e..2c18366 100644 --- a/crates/rustmotion/src/tests.rs +++ b/crates/rustmotion/src/tests.rs @@ -129,6 +129,9 @@ mod component_smoke { ("notification", r#"{"type":"notification","title":"Hi"}"#), ("pill_nav", r#"{"type":"pill_nav","items":["A","B"]}"#), ("tooltip", r#"{"type":"tooltip","text":"hi"}"#), + // Audio reactive components + ("audio_spectrum", r#"{"type":"audio_spectrum"}"#), + ("waveform", r#"{"type":"waveform"}"#), ]; #[test] @@ -1115,3 +1118,363 @@ mod component_smoke { ); } } + +#[cfg(test)] +mod audio_tests { + use std::sync::Arc; + + use rustmotion_core::engine::renderer::audio_analysis::{audio_analysis_cache, AudioAnalysis}; + + /// Build a minimal PCM WAV file in memory: mono i16, 44100 Hz. + /// The first `sine_samples` samples are a 440 Hz sine wave; + /// the rest are silence up to `total_samples`. + fn make_sine_wav( + total_samples: u32, + sine_samples: u32, + freq: f32, + sample_rate: u32, + ) -> Vec { + let data_size = total_samples * 2; // i16 mono + let mut wav = Vec::::new(); + wav.extend_from_slice(b"RIFF"); + wav.extend_from_slice(&(36 + data_size).to_le_bytes()); + wav.extend_from_slice(b"WAVE"); + wav.extend_from_slice(b"fmt "); + wav.extend_from_slice(&16u32.to_le_bytes()); + wav.extend_from_slice(&1u16.to_le_bytes()); // PCM + wav.extend_from_slice(&1u16.to_le_bytes()); // mono + wav.extend_from_slice(&sample_rate.to_le_bytes()); + wav.extend_from_slice(&(sample_rate * 2).to_le_bytes()); // byte rate + wav.extend_from_slice(&2u16.to_le_bytes()); // block align + wav.extend_from_slice(&16u16.to_le_bytes()); // bits per sample + wav.extend_from_slice(b"data"); + wav.extend_from_slice(&data_size.to_le_bytes()); + for i in 0..total_samples { + let s = if i < sine_samples { + let t = i as f32 / sample_rate as f32; + (2.0 * std::f32::consts::PI * freq * t).sin() + } else { + 0.0 + }; + let pcm = (s * 32767.0) as i16; + wav.extend_from_slice(&pcm.to_le_bytes()); + } + wav + } + fn nanos() -> u128 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + } + + /// Wrap a component JSON into a `ChildComponent` absolutely positioned at + /// the canvas origin. + fn child_at_origin(json: serde_json::Value) -> crate::components::ChildComponent { + let component: crate::components::Component = + serde_json::from_value(json).expect("component json"); + crate::components::ChildComponent { + component, + position: Some(crate::components::PositionMode::Absolute { x: 0.0, y: 0.0 }), + x: None, + y: None, + z_index: None, + } + } + + /// Paint a single child through the full box_tree → layout → paint + /// pipeline at `time` and return the RGBA8888 pixels. + fn paint_scene( + child: crate::components::ChildComponent, + w: i32, + h: i32, + time: f64, + fps: u32, + ) -> Vec { + use rustmotion_components::box_builder::{build_scene_with_anim, BuildAnimationCtx}; + use rustmotion_components::legacy_dispatch::LegacyPaintDispatcher; + use rustmotion_core::css::taffy_bridge::ConversionContext; + use rustmotion_core::engine::layout_pass::run_layout; + use rustmotion_core::engine::paint_pass::{paint_tree, PaintFrame}; + + let mut surface = skia_safe::surfaces::raster_n32_premul((w, h)).expect("raster surface"); + let canvas = surface.canvas(); + canvas.clear(skia_safe::Color4f::new(0.0, 0.0, 0.0, 0.0)); + let scene = vec![child]; + let built = build_scene_with_anim( + &scene, + (w as f32, h as f32), + BuildAnimationCtx { + time, + scene_duration: 10.0, + }, + ); + let layout = run_layout( + &built.root, + (w as f32, h as f32), + &ConversionContext::default(), + ); + let dispatcher = LegacyPaintDispatcher::for_scene(&built); + paint_tree( + canvas, + &built.root, + &layout, + &PaintFrame { + time, + frame_index: (time * fps as f64) as u32, + fps, + video_width: w as u32, + video_height: h as u32, + scene_duration: 10.0, + }, + &dispatcher, + ); + let row_bytes = (w * 4) as usize; + let mut pixels = vec![0u8; row_bytes * h as usize]; + let info = skia_safe::ImageInfo::new( + (w, h), + skia_safe::ColorType::RGBA8888, + skia_safe::AlphaType::Premul, + None, + ); + surface.read_pixels(&info, &mut pixels, row_bytes, (0, 0)); + pixels + } + + /// Count non-transparent pixels in the column range `[x0, x1)`. + fn lit_in_columns(pixels: &[u8], width: usize, x0: usize, x1: usize) -> usize { + let height = pixels.len() / (width * 4); + let mut count = 0; + for y in 0..height { + for x in x0..x1 { + if pixels[(y * width + x) * 4 + 3] > 0 { + count += 1; + } + } + } + count + } + + #[test] + fn analyze_scenario_audio_computes_amplitude_and_440hz_band() { + let sample_rate = 44100u32; + // 1.0 s total: 0.5 s of 440 Hz sine, then 0.5 s of silence. + let total_samples = sample_rate; + let sine_samples = sample_rate / 2; + let wav = make_sine_wav(total_samples, sine_samples, 440.0, sample_rate); + + let wav_path = + std::env::temp_dir().join(format!("rustmotion_test_analysis_{}.wav", nanos())); + std::fs::write(&wav_path, &wav).expect("write wav fixture"); + let wav_str = wav_path.to_str().unwrap().to_string(); + + let json = serde_json::json!({ + "video": {"width": 32, "height": 32, "fps": 30}, + "audio": [{"src": wav_str}], + "scenes": [{"duration": 1.0, "children": []}] + }) + .to_string(); + let scenario = + crate::loader::load_scenario_from_source(None, Some(&json)).expect("load scenario"); + crate::encode::audio_analysis::analyze_scenario_audio(&scenario); + + let analysis = audio_analysis_cache() + .get(&wav_str) + .expect("analysis must be cached under the track src") + .clone(); + std::fs::remove_file(&wav_path).ok(); + + assert_eq!(analysis.frame_rate, 30); + assert!( + analysis.amplitude.len() >= 29, + "1 s at 30 fps should give ~30 frames, got {}", + analysis.amplitude.len() + ); + + // Amplitude: ~1.0 during the sine (frames 0..14), ~0 during silence. + let sine_max = analysis.amplitude[..14] + .iter() + .cloned() + .fold(0.0f32, f32::max); + let silence_max = analysis.amplitude[16..] + .iter() + .cloned() + .fold(0.0f32, f32::max); + assert!( + sine_max > 0.9, + "normalized amplitude during the sine should be ~1.0, got {sine_max}" + ); + assert!( + silence_max < 0.05, + "amplitude during silence should be ~0, got {silence_max}" + ); + + // Band energy concentrated in the log band containing 440 Hz. + let lo = 20.0f32.log2(); + let hi = 16000.0f32.log2(); + let expected_band = (((440.0f32.log2() - lo) / ((hi - lo) / 16.0)) as usize).min(15); + let frame = &analysis.bands[5]; // mid-sine frame + let (argmax, max_v) = + frame.iter().enumerate().fold( + (0usize, 0.0f32), + |acc, (i, &v)| if v > acc.1 { (i, v) } else { acc }, + ); + assert_eq!( + argmax, expected_band, + "band {expected_band} should carry the 440 Hz energy (argmax was {argmax}: {frame:?})" + ); + assert!(max_v > 0.5, "440 Hz band should be hot, got {max_v}"); + let second = frame + .iter() + .enumerate() + .filter(|(i, _)| *i != expected_band) + .map(|(_, &v)| v) + .fold(0.0f32, f32::max); + assert!( + frame[expected_band] > second * 3.0, + "440 Hz band ({}) should dominate the runner-up ({second})", + frame[expected_band] + ); + } + + #[test] + fn audio_spectrum_hot_band_renders_taller_bar() { + let key = format!("test-spectrum-hot-{}", nanos()); + let mut bands = vec![[0.0f32; 16]; 30]; + for fr in &mut bands { + fr[15] = 1.0; // hottest band = highest frequencies → right-most bar + } + audio_analysis_cache().insert( + key.clone(), + Arc::new(AudioAnalysis { + frame_rate: 30, + amplitude: vec![1.0; 30], + bands, + }), + ); + + let child = child_at_origin(serde_json::json!({ + "type": "audio_spectrum", + "track": key, + "bars": 16 + })); + // Default intrinsic size 400x120; bar_w = (400 - 15*2)/16 ≈ 23.1, + // bar 15 spans x ≈ 377..400. + let pixels = paint_scene(child, 400, 200, 0.5, 30); + let hot_bar = lit_in_columns(&pixels, 400, 378, 400); + let cold_bar = lit_in_columns(&pixels, 400, 0, 23); + assert!( + hot_bar > cold_bar * 10, + "hot band bar ({hot_bar} lit px) should tower over a cold bar ({cold_bar} lit px)" + ); + + // Missing track → graceful degradation: min_height bars only, glued + // to the bottom edge of the 120px box. + let missing = child_at_origin(serde_json::json!({ + "type": "audio_spectrum", + "track": format!("test-spectrum-missing-{}", nanos()), + "bars": 16 + })); + let pixels = paint_scene(missing, 400, 200, 0.5, 30); + let mut lit_total = 0usize; + let mut lit_above_baseline = 0usize; + for y in 0..200usize { + for x in 0..400usize { + if pixels[(y * 400 + x) * 4 + 3] > 0 { + lit_total += 1; + if y < 115 { + lit_above_baseline += 1; + } + } + } + } + assert!(lit_total > 0, "min_height bars should still render"); + assert_eq!( + lit_above_baseline, 0, + "empty cache must render nothing above the min-height baseline" + ); + } + + #[test] + fn waveform_ramp_renders_increasing_pixels_along_x() { + let key = format!("test-waveform-ramp-{}", nanos()); + let n = 60usize; // 2 s at 30 fps + let amplitude: Vec = (0..n).map(|i| i as f32 / (n - 1) as f32).collect(); + audio_analysis_cache().insert( + key.clone(), + Arc::new(AudioAnalysis { + frame_rate: 30, + amplitude, + bands: vec![[0.0f32; 16]; 60], + }), + ); + + let child = child_at_origin(serde_json::json!({ + "type": "waveform", + "track": key, + "draw_style": "filled", + "window": 2.0 + })); + // At t=1.0 the 2 s window covers the whole ramp: amplitude (and the + // filled area under the curve) grows from left to right. + let pixels = paint_scene(child, 400, 200, 1.0, 30); + let left = lit_in_columns(&pixels, 400, 0, 133); + let right = lit_in_columns(&pixels, 400, 267, 400); + assert!( + left > 0, + "left third should have some lit pixels (outline at minimum)" + ); + assert!( + right > left * 2, + "ramping amplitude: right third ({right} lit px) should clearly exceed left third ({left} lit px)" + ); + } + + #[test] + fn audio_reactive_opacity_binding_differs_between_loud_and_quiet() { + let key = format!("test-ar-binding-{}", nanos()); + // Loud at t=0 (frame 0), silent afterwards. + let mut amplitude = vec![0.0f32; 90]; + amplitude[0] = 1.0; + audio_analysis_cache().insert( + key.clone(), + Arc::new(AudioAnalysis { + frame_rate: 30, + amplitude, + bands: vec![[0.0f32; 16]; 90], + }), + ); + + let make_child = || { + child_at_origin(serde_json::json!({ + "type": "shape", + "shape": "rect", + "fill": "#ff0000", + "style": { + "width": "100px", + "height": "100px", + "audio-reactive": { + "track": key, + "source": "amplitude", + "property": "opacity", + "min": 0.0, + "max": 1.0 + } + } + })) + }; + let red_sum = |pixels: &[u8]| pixels.chunks_exact(4).map(|p| p[0] as u64).sum::(); + + let loud = paint_scene(make_child(), 200, 200, 0.0, 30); + let quiet = paint_scene(make_child(), 200, 200, 0.5, 30); + let (loud_red, quiet_red) = (red_sum(&loud), red_sum(&quiet)); + assert!( + loud_red > 100_000, + "loud frame should render the red rect (red_sum={loud_red})" + ); + assert!( + loud_red > quiet_red.saturating_mul(10).max(1), + "red_sum must differ sharply between loud ({loud_red}) and quiet ({quiet_red}) frames" + ); + } +}