From 28c833f724f35211fc7141db1bcdddb2bbabcddb Mon Sep 17 00:00:00 2001 From: Ilya <47112191+ItsEeleeya@users.noreply.github.com> Date: Fri, 7 Nov 2025 17:24:08 +0330 Subject: [PATCH 01/46] project name formatter, new util moment_format_to_chrono with test --- Cargo.lock | 27 ++- Cargo.toml | 8 +- apps/desktop/src-tauri/Cargo.toml | 29 +-- .../desktop/src-tauri/src/general_settings.rs | 3 + apps/desktop/src-tauri/src/recording.rs | 176 ++++++++++++-- .../src/sources/screen_capture/mod.rs | 8 + crates/utils/Cargo.toml | 15 +- crates/utils/src/lib.rs | 216 +++++++++++++++++- 8 files changed, 437 insertions(+), 45 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1141acf3cd..ab6a3c1df7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -177,9 +177,9 @@ checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" [[package]] name = "aho-corasick" -version = "1.1.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" dependencies = [ "memchr", ] @@ -1174,6 +1174,7 @@ dependencies = [ name = "cap-desktop" version = "0.3.82" dependencies = [ + "aho-corasick", "anyhow", "async-stream", "axum", @@ -1221,9 +1222,11 @@ dependencies = [ "png 0.17.16", "posthog-rs", "rand 0.8.5", + "regex", "relative-path", "reqwest 0.12.24", "rodio", + "sanitize-filename", "scap-direct3d", "scap-screencapturekit", "scap-targets", @@ -1598,6 +1601,7 @@ dependencies = [ name = "cap-utils" version = "0.1.0" dependencies = [ + "aho-corasick", "directories 5.0.1", "flume", "futures", @@ -4122,7 +4126,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.5.10", + "socket2 0.6.0", "system-configuration 0.6.1", "tokio", "tower-service", @@ -4809,7 +4813,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07033963ba89ebaf1584d767badaa2e8fcec21aedea6b8c0346d487d49c28667" dependencies = [ "cfg-if", - "windows-targets 0.48.5", + "windows-targets 0.53.3", ] [[package]] @@ -6826,7 +6830,7 @@ dependencies = [ "quinn-udp", "rustc-hash 2.1.1", "rustls 0.23.31", - "socket2 0.5.10", + "socket2 0.6.0", "thiserror 2.0.16", "tokio", "tracing", @@ -6863,7 +6867,7 @@ dependencies = [ "cfg_aliases 0.2.1", "libc", "once_cell", - "socket2 0.5.10", + "socket2 0.6.0", "tracing", "windows-sys 0.60.2", ] @@ -7638,6 +7642,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "sanitize-filename" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc984f4f9ceb736a7bb755c3e3bd17dc56370af2600c9780dcc48c66453da34d" +dependencies = [ + "regex", +] + [[package]] name = "scap-cpal" version = "0.1.0" @@ -11088,7 +11101,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 0b35f66d45..268db6c433 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,11 @@ [workspace] resolver = "2" -members = ["apps/cli", "apps/desktop/src-tauri", "crates/*", "crates/workspace-hack"] +members = [ + "apps/cli", + "apps/desktop/src-tauri", + "crates/*", + "crates/workspace-hack", +] [workspace.dependencies] anyhow = { version = "1.0.86" } @@ -40,6 +45,7 @@ sentry = { version = "0.42.0", features = [ ] } tracing = "0.1.41" futures = "0.3.31" +aho-corasick = "1.1.4" cidre = { git = "https://github.com/CapSoftware/cidre", rev = "bf84b67079a8", features = [ "macos_12_7", diff --git a/apps/desktop/src-tauri/Cargo.toml b/apps/desktop/src-tauri/Cargo.toml index 02b6b27389..c58d51c6ad 100644 --- a/apps/desktop/src-tauri/Cargo.toml +++ b/apps/desktop/src-tauri/Cargo.toml @@ -20,11 +20,11 @@ swift-rs = { version = "1.0.6", features = ["build"] } [dependencies] tauri = { workspace = true, features = [ - "macos-private-api", - "protocol-asset", - "tray-icon", - "image-png", - "devtools", + "macos-private-api", + "protocol-asset", + "tray-icon", + "image-png", + "devtools", ] } tauri-specta = { version = "=2.0.0-rc.20", features = ["derive", "typescript"] } tauri-plugin-dialog = "2.2.0" @@ -60,6 +60,7 @@ tracing.workspace = true tempfile = "3.9.0" ffmpeg.workspace = true chrono = { version = "0.4.31", features = ["serde"] } +regex = "1.10.4" rodio = "0.19.0" png = "0.17.13" device_query = "4.0.1" @@ -106,22 +107,24 @@ tauri-plugin-sentry = "0.5.0" thiserror.workspace = true bytes = "1.10.1" async-stream = "0.3.6" +sanitize-filename = "0.6.0" tracing-futures = { version = "0.2.5", features = ["futures-03"] } tracing-opentelemetry = "0.32.0" opentelemetry = "0.31.0" -opentelemetry-otlp = "0.31.0" #{ version = , features = ["http-proto", "reqwest-client"] } +opentelemetry-otlp = "0.31.0" #{ version = , features = ["http-proto", "reqwest-client"] } opentelemetry_sdk = { version = "0.31.0", features = ["rt-tokio", "trace"] } posthog-rs = "0.3.7" workspace-hack = { version = "0.1", path = "../../../crates/workspace-hack" } +aho-corasick.workspace = true [target.'cfg(target_os = "macos")'.dependencies] core-graphics = "0.24.0" core-foundation = "0.10.0" objc2-app-kit = { version = "0.3.0", features = [ - "NSWindow", - "NSResponder", - "NSHapticFeedback", + "NSWindow", + "NSResponder", + "NSHapticFeedback", ] } cocoa = "0.26.0" objc = "0.2.7" @@ -131,10 +134,10 @@ cidre = { workspace = true } [target.'cfg(target_os= "windows")'.dependencies] windows = { workspace = true, features = [ - "Win32_Foundation", - "Win32_System", - "Win32_UI_WindowsAndMessaging", - "Win32_Graphics_Gdi", + "Win32_Foundation", + "Win32_System", + "Win32_UI_WindowsAndMessaging", + "Win32_Graphics_Gdi", ] } windows-sys = { workspace = true } diff --git a/apps/desktop/src-tauri/src/general_settings.rs b/apps/desktop/src-tauri/src/general_settings.rs index fbd6189853..96a81e1cfe 100644 --- a/apps/desktop/src-tauri/src/general_settings.rs +++ b/apps/desktop/src-tauri/src/general_settings.rs @@ -120,6 +120,8 @@ pub struct GeneralSettingsStore { pub delete_instant_recordings_after_upload: bool, #[serde(default = "default_instant_mode_max_resolution")] pub instant_mode_max_resolution: u32, + #[serde(default)] + pub default_project_name_template: Option, } fn default_enable_native_camera_preview() -> bool { @@ -184,6 +186,7 @@ impl Default for GeneralSettingsStore { excluded_windows: default_excluded_windows(), delete_instant_recordings_after_upload: false, instant_mode_max_resolution: 1920, + default_project_name_template: None, } } } diff --git a/apps/desktop/src-tauri/src/recording.rs b/apps/desktop/src-tauri/src/recording.rs index 9529e3210e..302924089a 100644 --- a/apps/desktop/src-tauri/src/recording.rs +++ b/apps/desktop/src-tauri/src/recording.rs @@ -7,6 +7,7 @@ use cap_project::{ TimelineConfiguration, TimelineSegment, UploadMeta, ZoomMode, ZoomSegment, cursor::CursorEvents, }; +use cap_recording::RecordingOptionCaptureTarget; use cap_recording::feeds::camera::CameraFeedLock; use cap_recording::{ RecordingMode, @@ -19,10 +20,14 @@ use cap_recording::{ studio_recording, }; use cap_rendering::ProjectRecordingsMeta; -use cap_utils::{ensure_dir, spawn_actor}; +use cap_utils::{ensure_dir, moment_format_to_chrono, spawn_actor}; use futures::stream; +use lazy_static::lazy_static; +use regex::Regex; use serde::{Deserialize, Serialize}; use specta::Type; +use std::borrow::Cow; +use std::sync::OnceLock; use std::{ collections::{HashMap, VecDeque}, path::PathBuf, @@ -257,6 +262,131 @@ pub enum RecordingAction { UpgradeRequired, } +lazy_static! { + static ref DATE_REGEX: Regex = Regex::new(r"\{date(?::([^}]+))?\}").unwrap(); + static ref TIME_REGEX: Regex = Regex::new(r"\{time(?::([^}]+))?\}").unwrap(); + static ref DATETIME_REGEX: Regex = Regex::new(r"\{datetime(?::([^}]+))?\}").unwrap(); + static ref TIMESTAMP_REGEX: Regex = Regex::new(r"\{timestamp(?::([^}]+))?\}").unwrap(); +} + +pub const DEFAULT_FILENAME_TEMPLATE: &str = "{target} {datetime}"; + +/// Formats the project name using a template string. +/// +/// # Template Variables +/// +/// The template supports the following variables that will be replaced with actual values: +/// +/// ## Recording Mode Variables +/// - `{recording_mode}` - The recording mode: "Studio" or "Instant" +/// - `{mode}` - Short form of recording mode: "studio" or "instant" +/// +/// ## Target Variables +/// - `{target_kind}` - The type of capture target: "Display", "Window", or "Area" +/// - `{target_name}` - The specific name of the target (e.g., "Built-in Retina Display", "Chrome", etc.) +/// - `{target}` - Combined target information (e.g., "Display (Built-in Retina Display)") +/// +/// ## Date/Time Variables +/// - `{date}` - Current date in YYYY-MM-DD format (e.g., "2025-09-11") +/// - `{time}` - Current time in HH:MM AM/PM format (e.g., "3:23 PM") +/// - `{datetime}` - Combined date and time (e.g., "2025-09-11 3:23 PM") +/// - `{timestamp}` - Unix timestamp (e.g., "1705326783") +/// +/// ## Customizable Date/Time Formats +/// You can customize date and time formats by adding moment format specifiers: +/// - `{date:YYYY-MM-DD}` - Custom date format +/// - `{time:HH:mm}` - 24-hour time format +/// - `{time:hh:mm A}` - 12-hour time with AM/PM +/// - `{datetime:YYYY-MM-DD HH:mm}` - Combined custom format +/// +/// ## Examples +/// +/// `{recording_mode} Recording {target_kind} ({target_name}) {date} {time}` +/// -> "Instant Recording Display (Built-in Retina Display) 2025-11-12 3:23 PM" +/// +/// `{recording_mode} Recording {target_kind} ({target_name}) {date} {time}` +/// -> "instant_display_20250115_1523" +/// +/// `Cap {target} - {datetime:YYYY-MM-DD HH:mm}` +/// -> "Cap Display (Built-in Retina Display) - 2025-11-12 15:23" +/// +/// +/// # Arguments +/// +/// * `template` - The template string with variables to replace +/// * `inputs` - The recording inputs containing target and mode information +/// +/// # Returns +/// +/// Returns `String` with the formatted project name +pub fn format_project_name<'a>( + template: Option<&str>, + target_name: &'a str, + target_kind: &'a str, + recording_mode: RecordingMode, + datetime: chrono::DateTime, +) -> String { + static AC: OnceLock = OnceLock::new(); + let template = template.unwrap_or(DEFAULT_FILENAME_TEMPLATE); + + // Get recording mode information + let (recording_mode, mode) = match recording_mode { + RecordingMode::Studio => ("Studio", "studio"), + RecordingMode::Instant => ("Instant", "instant"), + }; + + let ac = AC.get_or_init(|| { + aho_corasick::AhoCorasick::new(&[ + "{recording_mode}", + "{mode}", + "{target_kind}", + "{target_name}", + "{target}", + ]) + .expect("Failed to build AhoCorasick automaton") + }); + + let target_combined = format!("{target_kind} ({target_name})"); + let result = ac + .try_replace_all( + &template, + &[ + recording_mode, + mode, + target_kind, + target_name, + &target_combined, + ], + ) + .expect("AhoCorasick replace should never fail with default configuration"); + + let result = DATE_REGEX.replace_all(&result, |caps: ®ex::Captures| { + let format = caps.get(1).map(|m| m.as_str()).unwrap_or("%Y-%m-%d"); + let chrono_format = moment_format_to_chrono(format); + datetime.format(&chrono_format).to_string() + }); + + let result = TIME_REGEX.replace_all(&result, |caps: ®ex::Captures| { + let format = caps.get(1).map(|m| m.as_str()).unwrap_or("%l:%M %p"); + let chrono_format = moment_format_to_chrono(format); + datetime.format(&chrono_format).to_string() + }); + + let result = DATETIME_REGEX.replace_all(&result, |caps: ®ex::Captures| { + let format = caps.get(1).map(|m| m.as_str()).unwrap_or("%Y-%m-%d %H:%M"); + let chrono_format = moment_format_to_chrono(format); + datetime.format(&chrono_format).to_string() + }); + + let result = TIMESTAMP_REGEX.replace_all(&result, |caps: ®ex::Captures| { + caps.get(1) + .map(|m| datetime.format(m.as_str()).to_string()) + .unwrap_or_else(|| datetime.timestamp().to_string()) + }); + + result.into_owned() +} + #[tauri::command] #[specta::specta] #[tracing::instrument(name = "recording", skip_all)] @@ -269,16 +399,37 @@ pub async fn start_recording( return Err("Recording already in progress".to_string()); } - let id = uuid::Uuid::new_v4().to_string(); let general_settings = GeneralSettingsStore::get(&app).ok().flatten(); let general_settings = general_settings.as_ref(); - let recording_dir = app - .path() - .app_data_dir() - .unwrap() - .join("recordings") - .join(format!("{id}.cap")); + let project_name = format_project_name( + general_settings + .and_then(|s| s.default_project_name_template.clone()) + .as_deref(), + inputs + .capture_target + .title() + .as_deref() + .unwrap_or("Unknown"), + inputs.capture_target.kind_str(), + inputs.mode, + chrono::Local::now(), + ); + + let filename = sanitize_filename::sanitize_with_options( + &project_name, + sanitize_filename::Options { + replacement: "-", + ..Default::default() + }, + ); + + let recordings_base_dir = app.path().app_data_dir().unwrap().join("recordings"); + + let recording_dir = recordings_base_dir.join(&cap_utils::ensure_unique_filename( + &filename, + &recordings_base_dir, + )?); ensure_dir(&recording_dir).map_err(|e| format!("Failed to create recording directory: {e}"))?; state_mtx @@ -351,17 +502,10 @@ pub async fn start_recording( RecordingMode::Studio => None, }; - let date_time = if cfg!(windows) { - // Windows doesn't support colon in file paths - chrono::Local::now().format("%Y-%m-%d %H.%M.%S") - } else { - chrono::Local::now().format("%Y-%m-%d %H:%M:%S") - }; - let meta = RecordingMeta { platform: Some(Platform::default()), project_path: recording_dir.clone(), - pretty_name: format!("{target_name} {date_time}"), + pretty_name: project_name, inner: match inputs.mode { RecordingMode::Studio => { RecordingMetaInner::Studio(StudioRecordingMeta::MultipleSegments { diff --git a/crates/recording/src/sources/screen_capture/mod.rs b/crates/recording/src/sources/screen_capture/mod.rs index 4120b047b0..0353d5bb57 100644 --- a/crates/recording/src/sources/screen_capture/mod.rs +++ b/crates/recording/src/sources/screen_capture/mod.rs @@ -193,6 +193,14 @@ impl ScreenCaptureTarget { Self::Area { screen, .. } => Display::from_id(screen).and_then(|d| d.name()), } } + + pub fn kind_str(&self) -> &str { + match self { + ScreenCaptureTarget::Display { .. } => "Display", + ScreenCaptureTarget::Window { .. } => "Window", + ScreenCaptureTarget::Area { .. } => "Area", + } + } } pub struct ScreenCaptureConfig { diff --git a/crates/utils/Cargo.toml b/crates/utils/Cargo.toml index c1b9db8e14..1c01fbd36d 100644 --- a/crates/utils/Cargo.toml +++ b/crates/utils/Cargo.toml @@ -8,13 +8,13 @@ nix = { version = "0.29.0", features = ["fs"] } [target.'cfg(windows)'.dependencies] windows = { version = "0.58.0", features = [ - "Win32_Foundation", - "Win32_System", - "Win32_System_WindowsProgramming", - "Win32_Security", - "Win32_Storage_FileSystem", - "Win32_System_Pipes", - "Win32_System_Diagnostics_Debug", + "Win32_Foundation", + "Win32_System", + "Win32_System_WindowsProgramming", + "Win32_Security", + "Win32_Storage_FileSystem", + "Win32_System_Pipes", + "Win32_System_Diagnostics_Debug", ] } windows-sys = "0.52.0" @@ -27,6 +27,7 @@ serde_json = "1.0" flume = "0.11.0" tracing.workspace = true directories = "5.0" +aho-corasick.workspace = true workspace-hack = { version = "0.1", path = "../workspace-hack" } [lints] diff --git a/crates/utils/src/lib.rs b/crates/utils/src/lib.rs index 73ef3f41db..d5f9482c8c 100644 --- a/crates/utils/src/lib.rs +++ b/crates/utils/src/lib.rs @@ -1,5 +1,6 @@ -use std::{future::Future, path::PathBuf}; +use std::{borrow::Cow, future::Future, path::PathBuf, sync::OnceLock}; +use aho_corasick::{AhoCorasickBuilder, MatchKind}; use tracing::Instrument; /// Wrapper around tokio::spawn that inherits the current tracing subscriber and span. @@ -16,3 +17,216 @@ pub fn ensure_dir(path: &PathBuf) -> Result { std::fs::create_dir_all(path)?; Ok(path.clone()) } + +/// Generates a unique filename by appending incremental numbers if conflicts exist. +/// +/// This function takes a base filename and ensures it's unique by appending `(1)`, `(2)`, etc. +/// if a file with the same name already exists. It works with any file extension. +/// +/// # Arguments +/// +/// * `base_filename` - The desired filename (with extension) +/// * `parent_dir` - The directory where the file should be created +/// +/// # Returns +/// +/// Returns the unique filename that doesn't conflict with existing files. +/// +/// # Example +/// +/// ```rust +/// let unique_name = ensure_unique_filename("My Recording.cap", &recordings_dir); +/// // If "My Recording.cap" exists, returns "My Recording (1).cap" +/// // If that exists too, returns "My Recording (2).cap", etc. +/// +/// let unique_name = ensure_unique_filename("document.pdf", &documents_dir); +/// // If "document.pdf" exists, returns "document (1).pdf" +/// ``` +pub fn ensure_unique_filename( + base_filename: &str, + parent_dir: &std::path::Path, +) -> Result { + let initial_path = parent_dir.join(base_filename); + + if !initial_path.exists() { + println!("Ensure unique filename: is free!"); + return Ok(base_filename.to_string()); + } + + let path = std::path::Path::new(base_filename); + let (name_without_ext, extension) = if let Some(ext) = path.extension() { + let name_without_ext = path + .file_stem() + .and_then(|s| s.to_str()) + .unwrap_or(base_filename); + let extension = format!(".{}", ext.to_string_lossy()); + (name_without_ext, extension) + } else { + (base_filename, String::new()) + }; + + let mut counter = 1; + + loop { + let numbered_filename = if extension.is_empty() { + format!("{} ({})", name_without_ext, counter) + } else { + format!("{} ({}){}", name_without_ext, counter, &extension) + }; + + let test_path = parent_dir.join(&numbered_filename); + + println!("Ensure unique filename: test path count \"{counter}\""); + + if !test_path.exists() { + println!( + "Ensure unique filename: Found free! \"{}\"", + &test_path.display() + ); + return Ok(numbered_filename); + } + + counter += 1; + + // prevent infinite loop + if counter > 1000 { + return Err( + "Too many filename conflicts, unable to create unique filename".to_string(), + ); + } + } +} + +/// Converts user-friendly template format strings to chrono format strings. +/// +/// This function translates common template format patterns to chrono format specifiers, +/// allowing users to write intuitive date/time formats that get converted to chrono's format. +/// +/// # Supported Format Patterns +/// +/// ## Year +/// - `YYYY` → `%Y` - Year with century (e.g., 2025) +/// - `YY` → `%y` - Year without century (e.g., 25) +/// +/// ## Month +/// - `MMMM` → `%B` - Full month name (e.g., January) +/// - `MMM` → `%b` - Abbreviated month name (e.g., Jan) +/// - `MM` → `%m` - Month as zero-padded number (01-12) +/// - `M` → `%-m` - Month as number (1-12, no padding) +/// +/// ## Day +/// - `DDDD` → `%A` - Full weekday name (e.g., Monday) +/// - `DDD` → `%a` - Abbreviated weekday name (e.g., Mon) +/// - `DD` → `%d` - Day of month as zero-padded number (01-31) +/// - `D` → `%-d` - Day of month as number (1-31, no padding) +/// +/// ## Hour +/// - `HH` → `%H` - Hour (24-hour) as zero-padded number (00-23) +/// - `H` → `%-H` - Hour (24-hour) as number (0-23, no padding) +/// - `hh` → `%I` - Hour (12-hour) as zero-padded number (01-12) +/// - `h` → `%-I` - Hour (12-hour) as number (1-12, no padding) +/// +/// ## Minute +/// - `mm` → `%M` - Minute as zero-padded number (00-59) +/// - `m` → `%-M` - Minute as number (0-59, no padding) +/// +/// ## Second +/// - `ss` → `%S` - Second as zero-padded number (00-59) +/// - `s` → `%-S` - Second as number (0-59, no padding) +/// +/// ## AM/PM +/// - `A` → `%p` - AM/PM (uppercase) +/// - `a` → `%P` - am/pm (lowercase) +/// +/// ## Examples +/// +/// ``` +/// // Basic formats +/// YYYY-MM-DD HH:mm → %Y-%m-%d %H:%M +/// // Output: "2025-01-15 14:30" +/// +/// // Full month and day names +/// MMMM DD, YYYY → %B %d, %Y +/// // Output: "January 15, 2025" +/// +/// // Abbreviated names +/// DDD, MMM D, YYYY → %a, %b %-d, %Y +/// // Output: "Mon, Jan 15, 2025" +/// +/// // Compact format +/// YYYYMMDD_HHmmss → %Y%m%d_%H%M%S +/// // Output: "20250115_143045" +/// +/// // 12-hour format with full names +/// DDDD, MMMM DD at h:mm A → %A, %B %d at %-I:%M %p +/// // Output: "Monday, January 15 at 2:30 PM" +/// +/// // ISO week date +/// YYYY-Www-D → %G-W%V-%u +/// // Output: "2025-W03-1" +/// ``` +/// +/// # Note +/// +/// Pattern matching is case-sensitive and processes longer patterns first to avoid +/// conflicts (e.g., `MMMM` is matched before `MM`). +pub fn moment_format_to_chrono(template_format: &str) -> Cow<'_, str> { + static AC: OnceLock = OnceLock::new(); + + let ac = AC.get_or_init(|| { + // Order still matters (longer first helps readability), but using LeftmostLongest + // ensures overlapping shorter patterns won't also match. + AhoCorasickBuilder::new() + .match_kind(MatchKind::LeftmostLongest) + .build(&[ + "MMMM", "MMM", "MM", "M", "DDDD", "DDD", "DD", "D", "YYYY", "YY", "HH", "H", "hh", + "h", "mm", "m", "ss", "s", "A", "a", + ]) + .expect("Failed to build AhoCorasick automaton") + }); + + if !ac.is_match(template_format) { + return Cow::Borrowed(template_format); + } + + let replacements = [ + "%B", "%b", "%m", "%-m", // Month + "%A", "%a", "%d", "%-d", // Day + "%Y", "%y", // Year + "%H", "%-H", // Hour (24) + "%I", "%-I", // Hour (12) + "%M", "%-M", // Minute + "%S", "%-S", // Second + "%p", "%P", // AM/PM + ]; + + let replaced = ac + .try_replace_all(template_format, &replacements) + .expect("AhoCorasick replace should never fail with default configuration"); + + Cow::Owned(replaced) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn moment_format_to_chrono_converts_and_preserves_borrowed_when_unchanged() { + let input = "YYYY-MM-DD HH:mm:ss A a DDDD - DD - MMMM"; + let out = moment_format_to_chrono(input); + let expected = "%Y-%m-%d %H:%M:%S %p %P %A - %d - %B"; + assert_eq!( + out, expected, + "Converted format must match expected chrono format" + ); + + // Identity / borrowed case: no tokens -> should return Cow::Borrowed + let unchanged = "--"; + let out2 = moment_format_to_chrono(unchanged); + match out2 { + Cow::Borrowed(s) => assert_eq!(s, unchanged), + Cow::Owned(_) => panic!("Expected Cow::Borrowed for unchanged input"), + } + } +} From 19ee0965ee34c65481b39713e0e496992e6ee682 Mon Sep 17 00:00:00 2001 From: Ilya <47112191+ItsEeleeya@users.noreply.github.com> Date: Sat, 8 Nov 2025 06:25:34 +0330 Subject: [PATCH 02/46] use LazyLock and lazy_static instead --- apps/desktop/src-tauri/src/recording.rs | 63 +++++++------------------ crates/utils/src/lib.rs | 17 +++---- 2 files changed, 25 insertions(+), 55 deletions(-) diff --git a/apps/desktop/src-tauri/src/recording.rs b/apps/desktop/src-tauri/src/recording.rs index 302924089a..31e1b5ae45 100644 --- a/apps/desktop/src-tauri/src/recording.rs +++ b/apps/desktop/src-tauri/src/recording.rs @@ -7,7 +7,6 @@ use cap_project::{ TimelineConfiguration, TimelineSegment, UploadMeta, ZoomMode, ZoomSegment, cursor::CursorEvents, }; -use cap_recording::RecordingOptionCaptureTarget; use cap_recording::feeds::camera::CameraFeedLock; use cap_recording::{ RecordingMode, @@ -26,8 +25,6 @@ use lazy_static::lazy_static; use regex::Regex; use serde::{Deserialize, Serialize}; use specta::Type; -use std::borrow::Cow; -use std::sync::OnceLock; use std::{ collections::{HashMap, VecDeque}, path::PathBuf, @@ -284,7 +281,6 @@ pub const DEFAULT_FILENAME_TEMPLATE: &str = "{target} {datetime}"; /// ## Target Variables /// - `{target_kind}` - The type of capture target: "Display", "Window", or "Area" /// - `{target_name}` - The specific name of the target (e.g., "Built-in Retina Display", "Chrome", etc.) -/// - `{target}` - Combined target information (e.g., "Display (Built-in Retina Display)") /// /// ## Date/Time Variables /// - `{date}` - Current date in YYYY-MM-DD format (e.g., "2025-09-11") @@ -326,7 +322,17 @@ pub fn format_project_name<'a>( recording_mode: RecordingMode, datetime: chrono::DateTime, ) -> String { - static AC: OnceLock = OnceLock::new(); + lazy_static! { + static ref AC: aho_corasick::AhoCorasick = { + aho_corasick::AhoCorasick::new(&[ + "{recording_mode}", + "{mode}", + "{target_kind}", + "{target_name}", + ]) + .expect("Failed to build AhoCorasick automaton") + }; + } let template = template.unwrap_or(DEFAULT_FILENAME_TEMPLATE); // Get recording mode information @@ -335,29 +341,8 @@ pub fn format_project_name<'a>( RecordingMode::Instant => ("Instant", "instant"), }; - let ac = AC.get_or_init(|| { - aho_corasick::AhoCorasick::new(&[ - "{recording_mode}", - "{mode}", - "{target_kind}", - "{target_name}", - "{target}", - ]) - .expect("Failed to build AhoCorasick automaton") - }); - - let target_combined = format!("{target_kind} ({target_name})"); - let result = ac - .try_replace_all( - &template, - &[ - recording_mode, - mode, - target_kind, - target_name, - &target_combined, - ], - ) + let result = AC + .try_replace_all(&template, &[recording_mode, mode, target_kind, target_name]) .expect("AhoCorasick replace should never fail with default configuration"); let result = DATE_REGEX.replace_all(&result, |caps: ®ex::Captures| { @@ -416,8 +401,9 @@ pub async fn start_recording( chrono::Local::now(), ); + let filename = project_name.replace(":", "."); let filename = sanitize_filename::sanitize_with_options( - &project_name, + &filename, sanitize_filename::Options { replacement: "-", ..Default::default() @@ -438,16 +424,6 @@ pub async fn start_recording( .add_recording_logging_handle(&recording_dir.join("recording-logs.log")) .await?; - let target_name = { - let title = inputs.capture_target.title(); - - match inputs.capture_target.clone() { - ScreenCaptureTarget::Area { .. } => title.unwrap_or_else(|| "Area".to_string()), - ScreenCaptureTarget::Window { .. } => title.unwrap_or_else(|| "Window".to_string()), - ScreenCaptureTarget::Display { .. } => title.unwrap_or_else(|| "Screen".to_string()), - } - }; - if let Some(window) = CapWindowId::Camera.get(&app) { let _ = window.set_content_protected(matches!(inputs.mode, RecordingMode::Studio)); } @@ -461,10 +437,7 @@ pub async fn start_recording( &app, false, None, - Some(format!( - "{target_name} {}", - chrono::Local::now().format("%Y-%m-%d %H:%M:%S") - )), + Some(project_name.clone()), None, inputs.organization_id.clone(), ) @@ -505,7 +478,7 @@ pub async fn start_recording( let meta = RecordingMeta { platform: Some(Platform::default()), project_path: recording_dir.clone(), - pretty_name: project_name, + pretty_name: project_name.clone(), inner: match inputs.mode { RecordingMode::Studio => { RecordingMetaInner::Studio(StudioRecordingMeta::MultipleSegments { @@ -619,7 +592,7 @@ pub async fn start_recording( .ok_or_else(|| "GetShareableContent/NotAvailable".to_string())?; let common = InProgressRecordingCommon { - target_name, + target_name: project_name.clone(), inputs: inputs.clone(), recording_dir: recording_dir.clone(), }; diff --git a/crates/utils/src/lib.rs b/crates/utils/src/lib.rs index d5f9482c8c..65a81f4ee2 100644 --- a/crates/utils/src/lib.rs +++ b/crates/utils/src/lib.rs @@ -1,4 +1,4 @@ -use std::{borrow::Cow, future::Future, path::PathBuf, sync::OnceLock}; +use std::{borrow::Cow, future::Future, path::PathBuf, sync::LazyLock}; use aho_corasick::{AhoCorasickBuilder, MatchKind}; use tracing::Instrument; @@ -97,7 +97,7 @@ pub fn ensure_unique_filename( } } -/// Converts user-friendly template format strings to chrono format strings. +/// Converts user-friendly moment template format strings to chrono format strings. /// /// This function translates common template format patterns to chrono format specifiers, /// allowing users to write intuitive date/time formats that get converted to chrono's format. @@ -171,21 +171,18 @@ pub fn ensure_unique_filename( /// Pattern matching is case-sensitive and processes longer patterns first to avoid /// conflicts (e.g., `MMMM` is matched before `MM`). pub fn moment_format_to_chrono(template_format: &str) -> Cow<'_, str> { - static AC: OnceLock = OnceLock::new(); - - let ac = AC.get_or_init(|| { - // Order still matters (longer first helps readability), but using LeftmostLongest - // ensures overlapping shorter patterns won't also match. + static AC: LazyLock = LazyLock::new(|| { AhoCorasickBuilder::new() + // Use LeftmostLongest patterns to ensure overlapping shorter patterns won't also match. .match_kind(MatchKind::LeftmostLongest) - .build(&[ + .build([ "MMMM", "MMM", "MM", "M", "DDDD", "DDD", "DD", "D", "YYYY", "YY", "HH", "H", "hh", "h", "mm", "m", "ss", "s", "A", "a", ]) .expect("Failed to build AhoCorasick automaton") }); - if !ac.is_match(template_format) { + if !AC.is_match(template_format) { return Cow::Borrowed(template_format); } @@ -200,7 +197,7 @@ pub fn moment_format_to_chrono(template_format: &str) -> Cow<'_, str> { "%p", "%P", // AM/PM ]; - let replaced = ac + let replaced = AC .try_replace_all(template_format, &replacements) .expect("AhoCorasick replace should never fail with default configuration"); From 7127d9fc43dd0ff0b20d67eeb9de8141a592aefc Mon Sep 17 00:00:00 2001 From: Ilya <47112191+ItsEeleeya@users.noreply.github.com> Date: Sat, 8 Nov 2025 06:36:47 +0330 Subject: [PATCH 03/46] Add command `format_project_name` --- Cargo.lock | 1 + Cargo.toml | 1 + apps/desktop/src-tauri/src/lib.rs | 21 ++++++++++++++++++++- 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index ab6a3c1df7..16ba0d9d8f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8509,6 +8509,7 @@ version = "2.0.0-rc.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ccbb212565d2dc177bc15ecb7b039d66c4490da892436a4eee5b394d620c9bc" dependencies = [ + "chrono", "paste", "serde_json", "specta-macros", diff --git a/Cargo.toml b/Cargo.toml index 268db6c433..34b18cf08f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,6 +27,7 @@ specta = { version = "=2.0.0-rc.20", features = [ "derive", "serde_json", "uuid", + "chrono" ] } serde = { version = "1", features = ["derive"] } diff --git a/apps/desktop/src-tauri/src/lib.rs b/apps/desktop/src-tauri/src/lib.rs index d5ccc3c6f7..6088842a16 100644 --- a/apps/desktop/src-tauri/src/lib.rs +++ b/apps/desktop/src-tauri/src/lib.rs @@ -1900,7 +1900,8 @@ pub async fn run(recording_logging_handle: LoggingHandle, logs_dir: PathBuf) { target_select_overlay::display_information, target_select_overlay::get_window_icon, target_select_overlay::focus_window, - editor_delete_project + editor_delete_project, + format_project_name, ]) .events(tauri_specta::collect_events![ RecordingOptionsChanged, @@ -2640,6 +2641,24 @@ async fn write_clipboard_string( .map_err(|e| format!("Failed to write text to clipboard: {e}")) } +#[tauri::command] +#[specta::specta] +async fn format_project_name( + template: Option, + target_name: String, + target_kind: String, + recording_mode: RecordingMode, + datetime: chrono::DateTime, +) -> String { + recording::format_project_name( + template.as_deref(), + target_name.as_str(), + target_kind.as_str(), + recording_mode, + datetime, + ) +} + trait EventExt: tauri_specta::Event { fn listen_any_spawn( app: &AppHandle, From b1e4a603c5038ce8872d670a9cbaa1e7016eefc6 Mon Sep 17 00:00:00 2001 From: Ilya <47112191+ItsEeleeya@users.noreply.github.com> Date: Mon, 10 Nov 2025 15:02:08 +0330 Subject: [PATCH 04/46] Add UI + update collapsible animations with a little blur --- apps/desktop/src-tauri/src/lib.rs | 15 +- apps/desktop/src-tauri/src/recording.rs | 83 +++---- .../(window-chrome)/settings/general.tsx | 217 ++++++++++++++++++ apps/desktop/src/utils/tauri.ts | 8 +- packages/ui-solid/src/auto-imports.d.ts | 2 +- packages/ui-solid/tailwind.config.js | 8 +- 6 files changed, 287 insertions(+), 46 deletions(-) diff --git a/apps/desktop/src-tauri/src/lib.rs b/apps/desktop/src-tauri/src/lib.rs index 6088842a16..9ffc8f03d3 100644 --- a/apps/desktop/src-tauri/src/lib.rs +++ b/apps/desktop/src-tauri/src/lib.rs @@ -1902,6 +1902,7 @@ pub async fn run(recording_logging_handle: LoggingHandle, logs_dir: PathBuf) { target_select_overlay::focus_window, editor_delete_project, format_project_name, + sanitize_filename, ]) .events(tauri_specta::collect_events![ RecordingOptionsChanged, @@ -2648,7 +2649,7 @@ async fn format_project_name( target_name: String, target_kind: String, recording_mode: RecordingMode, - datetime: chrono::DateTime, + datetime: Option>, ) -> String { recording::format_project_name( template.as_deref(), @@ -2659,6 +2660,18 @@ async fn format_project_name( ) } +#[tauri::command] +#[specta::specta] +async fn sanitize_filename(filename: String) -> String { + sanitize_filename::sanitize_with_options( + filename, + sanitize_filename::Options { + replacement: "-", + ..Default::default() + }, + ) +} + trait EventExt: tauri_specta::Event { fn listen_any_spawn( app: &AppHandle, diff --git a/apps/desktop/src-tauri/src/recording.rs b/apps/desktop/src-tauri/src/recording.rs index 31e1b5ae45..bc285a22d9 100644 --- a/apps/desktop/src-tauri/src/recording.rs +++ b/apps/desktop/src-tauri/src/recording.rs @@ -25,6 +25,7 @@ use lazy_static::lazy_static; use regex::Regex; use serde::{Deserialize, Serialize}; use specta::Type; +use std::borrow::Cow; use std::{ collections::{HashMap, VecDeque}, path::PathBuf, @@ -259,15 +260,6 @@ pub enum RecordingAction { UpgradeRequired, } -lazy_static! { - static ref DATE_REGEX: Regex = Regex::new(r"\{date(?::([^}]+))?\}").unwrap(); - static ref TIME_REGEX: Regex = Regex::new(r"\{time(?::([^}]+))?\}").unwrap(); - static ref DATETIME_REGEX: Regex = Regex::new(r"\{datetime(?::([^}]+))?\}").unwrap(); - static ref TIMESTAMP_REGEX: Regex = Regex::new(r"\{timestamp(?::([^}]+))?\}").unwrap(); -} - -pub const DEFAULT_FILENAME_TEMPLATE: &str = "{target} {datetime}"; - /// Formats the project name using a template string. /// /// # Template Variables @@ -285,25 +277,20 @@ pub const DEFAULT_FILENAME_TEMPLATE: &str = "{target} {datetime}"; /// ## Date/Time Variables /// - `{date}` - Current date in YYYY-MM-DD format (e.g., "2025-09-11") /// - `{time}` - Current time in HH:MM AM/PM format (e.g., "3:23 PM") -/// - `{datetime}` - Combined date and time (e.g., "2025-09-11 3:23 PM") -/// - `{timestamp}` - Unix timestamp (e.g., "1705326783") /// /// ## Customizable Date/Time Formats /// You can customize date and time formats by adding moment format specifiers: -/// - `{date:YYYY-MM-DD}` - Custom date format -/// - `{time:HH:mm}` - 24-hour time format -/// - `{time:hh:mm A}` - 12-hour time with AM/PM -/// - `{datetime:YYYY-MM-DD HH:mm}` - Combined custom format +/// - `{moment:YYYY-MM-DD}` - Custom date format +/// - `{moment:HH:mm}` - 24-hour time format +/// - `{moment:hh:mm A}` - 12-hour time with AM/PM +/// - `{moment:YYYY-MM-DD HH:mm}` - Combined custom format /// /// ## Examples /// /// `{recording_mode} Recording {target_kind} ({target_name}) {date} {time}` /// -> "Instant Recording Display (Built-in Retina Display) 2025-11-12 3:23 PM" /// -/// `{recording_mode} Recording {target_kind} ({target_name}) {date} {time}` -/// -> "instant_display_20250115_1523" -/// -/// `Cap {target} - {datetime:YYYY-MM-DD HH:mm}` +/// `Cap {target} - {date} {time}` /// -> "Cap Display (Built-in Retina Display) - 2025-11-12 15:23" /// /// @@ -320,9 +307,15 @@ pub fn format_project_name<'a>( target_name: &'a str, target_kind: &'a str, recording_mode: RecordingMode, - datetime: chrono::DateTime, + datetime: Option>, ) -> String { + const DEFAULT_FILENAME_TEMPLATE: &str = "{target_name} ({target_kind}) {date} {time}"; + let datetime = datetime.unwrap_or(chrono::Local::now()); + lazy_static! { + static ref DATE_REGEX: Regex = Regex::new(r"\{date(?::([^}]+))?\}").unwrap(); + static ref TIME_REGEX: Regex = Regex::new(r"\{time(?::([^}]+))?\}").unwrap(); + static ref MOMENT_REGEX: Regex = Regex::new(r"\{moment(?::([^}]+))?\}").unwrap(); static ref AC: aho_corasick::AhoCorasick = { aho_corasick::AhoCorasick::new(&[ "{recording_mode}", @@ -333,7 +326,7 @@ pub fn format_project_name<'a>( .expect("Failed to build AhoCorasick automaton") }; } - let template = template.unwrap_or(DEFAULT_FILENAME_TEMPLATE); + let haystack = template.unwrap_or(DEFAULT_FILENAME_TEMPLATE); // Get recording mode information let (recording_mode, mode) = match recording_mode { @@ -342,31 +335,43 @@ pub fn format_project_name<'a>( }; let result = AC - .try_replace_all(&template, &[recording_mode, mode, target_kind, target_name]) + .try_replace_all(&haystack, &[recording_mode, mode, target_kind, target_name]) .expect("AhoCorasick replace should never fail with default configuration"); let result = DATE_REGEX.replace_all(&result, |caps: ®ex::Captures| { - let format = caps.get(1).map(|m| m.as_str()).unwrap_or("%Y-%m-%d"); - let chrono_format = moment_format_to_chrono(format); - datetime.format(&chrono_format).to_string() + datetime + .format( + &caps + .get(1) + .map(|m| m.as_str()) + .map(moment_format_to_chrono) + .unwrap_or(Cow::Borrowed("%Y-%m-%d")), + ) + .to_string() }); let result = TIME_REGEX.replace_all(&result, |caps: ®ex::Captures| { - let format = caps.get(1).map(|m| m.as_str()).unwrap_or("%l:%M %p"); - let chrono_format = moment_format_to_chrono(format); - datetime.format(&chrono_format).to_string() - }); - - let result = DATETIME_REGEX.replace_all(&result, |caps: ®ex::Captures| { - let format = caps.get(1).map(|m| m.as_str()).unwrap_or("%Y-%m-%d %H:%M"); - let chrono_format = moment_format_to_chrono(format); - datetime.format(&chrono_format).to_string() + datetime + .format( + &caps + .get(1) + .map(|m| m.as_str()) + .map(moment_format_to_chrono) + .unwrap_or(Cow::Borrowed("%I:%M %p")), + ) + .to_string() }); - let result = TIMESTAMP_REGEX.replace_all(&result, |caps: ®ex::Captures| { - caps.get(1) - .map(|m| datetime.format(m.as_str()).to_string()) - .unwrap_or_else(|| datetime.timestamp().to_string()) + let result = MOMENT_REGEX.replace_all(&result, |caps: ®ex::Captures| { + datetime + .format( + &caps + .get(1) + .map(|m| m.as_str()) + .map(moment_format_to_chrono) + .unwrap_or(Cow::Borrowed("%Y-%m-%d %H:%M")), + ) + .to_string() }); result.into_owned() @@ -398,7 +403,7 @@ pub async fn start_recording( .unwrap_or("Unknown"), inputs.capture_target.kind_str(), inputs.mode, - chrono::Local::now(), + None, ); let filename = project_name.replace(":", "."); diff --git a/apps/desktop/src/routes/(window-chrome)/settings/general.tsx b/apps/desktop/src/routes/(window-chrome)/settings/general.tsx index 52cd6384bd..7495791f0f 100644 --- a/apps/desktop/src/routes/(window-chrome)/settings/general.tsx +++ b/apps/desktop/src/routes/(window-chrome)/settings/general.tsx @@ -6,6 +6,7 @@ import { } from "@tauri-apps/plugin-notification"; import { type OsType, type } from "@tauri-apps/plugin-os"; import "@total-typescript/ts-reset/filter-boolean"; +import { Collapsible } from "@kobalte/core/collapsible"; import { CheckMenuItem, Menu, MenuItem } from "@tauri-apps/api/menu"; import { confirm } from "@tauri-apps/plugin-dialog"; import { cx } from "cva"; @@ -13,7 +14,9 @@ import { createEffect, createMemo, createResource, + createSignal, For, + onMount, type ParentProps, Show, } from "solid-js"; @@ -98,6 +101,9 @@ const INSTANT_MODE_RESOLUTION_OPTIONS = [ label: string; }[]; +const DEFAULT_PROJECT_NAME_TEMPLATE = + "{target_name} ({target_kind}) {date} {time}"; + export default function GeneralSettings() { const [store] = createResource(() => generalSettingsStore.get()); @@ -498,6 +504,13 @@ function Inner(props: { initialStore: GeneralSettingsStore | null }) { /> + + handleChange("defaultProjectNameTemplate", value) + } + value={settings.defaultProjectNameTemplate ?? null} + /> + Promise; +}) { + const MOMENT_EXAMPLE_TEMPLATE = "{moment:DDDD, MMMM D, YYYY h:mm A}"; + const macos = type() === "macos"; + const today = new Date(); + const datetime = new Date( + today.getFullYear(), + today.getMonth(), + today.getDate(), + macos ? 9 : 12, + macos ? 41 : 0, + 0, + 0, + ).toISOString(); + + let inputRef: HTMLInputElement | undefined; + + const dateString = today.toISOString().split("T")[0]; + const initialTemplate = () => props.value ?? DEFAULT_PROJECT_NAME_TEMPLATE; + + const [inputValue, setInputValue] = createSignal(initialTemplate()); + const [preview, setPreview] = createSignal(null); + const [momentExample, setMomentExample] = createSignal(""); + + async function updatePreview(val = inputValue()) { + const formatted = await commands.formatProjectName( + val, + macos ? "Safari" : "Chrome", + "Window", + "instant", + datetime, + ); + setPreview(formatted); + } + + onMount(() => { + commands + .formatProjectName( + MOMENT_EXAMPLE_TEMPLATE, + macos ? "Safari" : "Chrome", + "Window", + "instant", + datetime, + ) + .then(setMomentExample); + + const seed = initialTemplate(); + setInputValue(seed); + if (inputRef) inputRef.value = seed; + updatePreview(seed); + }); + + const isSaveDisabled = () => { + const input = inputValue(); + return ( + !input || + input === (props.value ?? DEFAULT_PROJECT_NAME_TEMPLATE) || + input.length <= 3 + ); + }; + + function CodeView(props: { children: string }) { + return ( + + ); + } + + return ( +
+
+
+

Default Project Name

+

+ Choose the template to use as the default project name. +
+ This will also be used as the default file name for new projects. +

+ + { + setInputValue(e.currentTarget.value); + updatePreview(e.currentTarget.value); + }} + /> + +
+ +

{preview()}

+
+ + + + +

How to customize?

+ + + +

+ Use placeholders in your template that will be automatically + filled in. +

+ +
+

Recording Mode

+

+ {"{recording_mode}"} → "Studio" or + "Instant" +

+

+ {"{mode}"} → "studio" or "instant" +

+
+ +
+

Target

+

+ {"{target_kind}"} → "Display", "Window", + or "Area" +

+

+ {"{target_name}"} → The name of the + monitor or the title of the app depending on the recording + mode. +

+
+ +
+

Date & Time

+

+ {"{date}"} → {dateString} +

+

+ {"{time}"} →{" "} + {macos ? "09:41 AM" : "12:00 PM"} +

+
+ +
+

Custom Formats

+

+ You can also use a custom format for time. The placeholders + are case-sensitive. For 24-hour time, use{" "} + {"{moment:HH:mm}"} or use lower cased{" "} + hh for 12-hour format. +

+

+ {MOMENT_EXAMPLE_TEMPLATE} →{" "} + {momentExample()} +

+
+ + + +
+ + + +
+
+
+
+ ); +} + function ExcludedWindowsCard(props: { excludedWindows: WindowExclusion[]; availableWindows: CaptureWindow[]; diff --git a/apps/desktop/src/utils/tauri.ts b/apps/desktop/src/utils/tauri.ts index 0a01999121..6dd1c2dce3 100644 --- a/apps/desktop/src/utils/tauri.ts +++ b/apps/desktop/src/utils/tauri.ts @@ -288,6 +288,12 @@ async focusWindow(windowId: WindowId) : Promise { }, async editorDeleteProject() : Promise { return await TAURI_INVOKE("editor_delete_project"); +}, +async formatProjectName(template: string | null, targetName: string, targetKind: string, recordingMode: RecordingMode, datetime: string | null) : Promise { + return await TAURI_INVOKE("format_project_name", { template, targetName, targetKind, recordingMode, datetime }); +}, +async sanitizeFilename(filename: string) : Promise { + return await TAURI_INVOKE("sanitize_filename", { filename }); } } @@ -401,7 +407,7 @@ export type ExportSettings = ({ format: "Mp4" } & Mp4ExportSettings) | ({ format export type FileType = "recording" | "screenshot" export type Flags = { captions: boolean } export type FramesRendered = { renderedCount: number; totalFrames: number; type: "FramesRendered" } -export type GeneralSettingsStore = { instanceId?: string; uploadIndividualFiles?: boolean; hideDockIcon?: boolean; autoCreateShareableLink?: boolean; enableNotifications?: boolean; disableAutoOpenLinks?: boolean; hasCompletedStartup?: boolean; theme?: AppTheme; commercialLicense?: CommercialLicense | null; lastVersion?: string | null; windowTransparency?: boolean; postStudioRecordingBehaviour?: PostStudioRecordingBehaviour; mainWindowRecordingStartBehaviour?: MainWindowRecordingStartBehaviour; custom_cursor_capture2?: boolean; serverUrl?: string; recordingCountdown?: number | null; enableNativeCameraPreview: boolean; autoZoomOnClicks?: boolean; enableNewRecordingFlow: boolean; postDeletionBehaviour?: PostDeletionBehaviour; excludedWindows?: WindowExclusion[]; deleteInstantRecordingsAfterUpload?: boolean; instantModeMaxResolution?: number } +export type GeneralSettingsStore = { instanceId?: string; uploadIndividualFiles?: boolean; hideDockIcon?: boolean; autoCreateShareableLink?: boolean; enableNotifications?: boolean; disableAutoOpenLinks?: boolean; hasCompletedStartup?: boolean; theme?: AppTheme; commercialLicense?: CommercialLicense | null; lastVersion?: string | null; windowTransparency?: boolean; postStudioRecordingBehaviour?: PostStudioRecordingBehaviour; mainWindowRecordingStartBehaviour?: MainWindowRecordingStartBehaviour; custom_cursor_capture2?: boolean; serverUrl?: string; recordingCountdown?: number | null; enableNativeCameraPreview: boolean; autoZoomOnClicks?: boolean; enableNewRecordingFlow: boolean; postDeletionBehaviour?: PostDeletionBehaviour; excludedWindows?: WindowExclusion[]; deleteInstantRecordingsAfterUpload?: boolean; instantModeMaxResolution?: number; defaultProjectNameTemplate?: string | null } export type GifExportSettings = { fps: number; resolution_base: XY; quality: GifQuality | null } export type GifQuality = { /** diff --git a/packages/ui-solid/src/auto-imports.d.ts b/packages/ui-solid/src/auto-imports.d.ts index 57ce5370b4..7d8000b7fc 100644 --- a/packages/ui-solid/src/auto-imports.d.ts +++ b/packages/ui-solid/src/auto-imports.d.ts @@ -8,7 +8,7 @@ export {} declare global { const IconCapArrows: typeof import('~icons/cap/arrows.jsx')['default'] const IconCapAudioOn: typeof import('~icons/cap/audio-on.jsx')['default'] - const IconCapAuto: typeof import("~icons/cap/auto.jsx")["default"] + const IconCapAuto: typeof import('~icons/cap/auto.jsx')['default'] const IconCapBgBlur: typeof import('~icons/cap/bg-blur.jsx')['default'] const IconCapBlur: typeof import("~icons/cap/blur.jsx")["default"] const IconCapCamera: typeof import('~icons/cap/camera.jsx')['default'] diff --git a/packages/ui-solid/tailwind.config.js b/packages/ui-solid/tailwind.config.js index 77adf26586..0092467055 100644 --- a/packages/ui-solid/tailwind.config.js +++ b/packages/ui-solid/tailwind.config.js @@ -89,12 +89,12 @@ module.exports = { }, keyframes: { "collapsible-down": { - from: { height: 0 }, - to: { height: "var(--kb-collapsible-content-height)" }, + from: { height: 0, filter: "blur(4px)" }, + to: { height: "var(--kb-collapsible-content-height)", filter: "blur(0px)" }, }, "collapsible-up": { - from: { height: "var(--kb-collapsible-content-height)" }, - to: { height: 0 }, + from: { height: "var(--kb-collapsible-content-height)", filter: "blur(0px)" }, + to: { height: 0, filter: "blur(4px)" }, }, }, animation: { From c5a2878ffa5964f2c919ed9637a540839a32bc06 Mon Sep 17 00:00:00 2001 From: Ilya <47112191+ItsEeleeya@users.noreply.github.com> Date: Mon, 10 Nov 2025 15:14:39 +0330 Subject: [PATCH 05/46] Make the new DefaultProjectNameCard more consistent with ExcludedWindowsCard + Fix button text spilling --- .../(window-chrome)/settings/general.tsx | 198 ++++++++++-------- 1 file changed, 114 insertions(+), 84 deletions(-) diff --git a/apps/desktop/src/routes/(window-chrome)/settings/general.tsx b/apps/desktop/src/routes/(window-chrome)/settings/general.tsx index 7495791f0f..8e44276580 100644 --- a/apps/desktop/src/routes/(window-chrome)/settings/general.tsx +++ b/apps/desktop/src/routes/(window-chrome)/settings/general.tsx @@ -674,96 +674,127 @@ function DefaultProjectNameCard(props: { return (
-
-
+
+

Default Project Name

-

- Choose the template to use as the default project name. -
- This will also be used as the default file name for new projects. +

+ Choose the template to use as the default project and file name.

+
+
+ - { - setInputValue(e.currentTarget.value); - updatePreview(e.currentTarget.value); + +
+
-
- -

{preview()}

-
+
+ { + setInputValue(e.currentTarget.value); + updatePreview(e.currentTarget.value); + }} + /> + +
+ +

{preview()}

+
- - - -

How to customize?

- + + + +

How to customize?

+ - -

- Use placeholders in your template that will be automatically - filled in. + +

+ Use placeholders in your template that will be automatically + filled in. +

+ +
+

Recording Mode

+

+ {"{recording_mode}"} → "Studio" or + "Instant" +

+

+ {"{mode}"} → "studio" or "instant"

+
+ +
+

Target

+

+ {"{target_kind}"} → "Display", "Window", or + "Area" +

+

+ {"{target_name}"} → The name of the monitor + or the title of the app depending on the recording mode. +

+
-
-

Recording Mode

-

- {"{recording_mode}"} → "Studio" or - "Instant" -

-

- {"{mode}"} → "studio" or "instant" -

-
- -
-

Target

-

- {"{target_kind}"} → "Display", "Window", - or "Area" -

-

- {"{target_name}"} → The name of the - monitor or the title of the app depending on the recording - mode. -

-
- -
-

Date & Time

-

- {"{date}"} → {dateString} -

-

- {"{time}"} →{" "} - {macos ? "09:41 AM" : "12:00 PM"} -

-
- -
-

Custom Formats

-

- You can also use a custom format for time. The placeholders - are case-sensitive. For 24-hour time, use{" "} - {"{moment:HH:mm}"} or use lower cased{" "} - hh for 12-hour format. -

-

- {MOMENT_EXAMPLE_TEMPLATE} →{" "} - {momentExample()} -

-
- - - -
+
+

Date & Time

+

+ {"{date}"} → {dateString} +

+

+ {"{time}"} →{" "} + {macos ? "09:41 AM" : "12:00 PM"} +

+
+ +
+

Custom Formats

+

+ You can also use a custom format for time. The placeholders are + case-sensitive. For 24-hour time, use{" "} + {"{moment:HH:mm}"} or use lower cased{" "} + hh for 12-hour format. +

+

+ {MOMENT_EXAMPLE_TEMPLATE} →{" "} + {momentExample()} +

+
+ + + + {/*
-
-
+
*/}
); @@ -882,7 +912,7 @@ function ExcludedWindowsCard(props: {

-
+
- - {/*
- - - -
*/}
); diff --git a/crates/utils/src/lib.rs b/crates/utils/src/lib.rs index 65a81f4ee2..9c7d828f1e 100644 --- a/crates/utils/src/lib.rs +++ b/crates/utils/src/lib.rs @@ -49,7 +49,6 @@ pub fn ensure_unique_filename( let initial_path = parent_dir.join(base_filename); if !initial_path.exists() { - println!("Ensure unique filename: is free!"); return Ok(base_filename.to_string()); } @@ -76,13 +75,7 @@ pub fn ensure_unique_filename( let test_path = parent_dir.join(&numbered_filename); - println!("Ensure unique filename: test path count \"{counter}\""); - if !test_path.exists() { - println!( - "Ensure unique filename: Found free! \"{}\"", - &test_path.display() - ); return Ok(numbered_filename); } From f72e79ac42b727861d10a5c392fa1da786736607 Mon Sep 17 00:00:00 2001 From: Ilya <47112191+ItsEeleeya@users.noreply.github.com> Date: Mon, 10 Nov 2025 19:08:28 +0330 Subject: [PATCH 08/46] Fix rustdoc for format_project_name --- apps/desktop/src-tauri/src/recording.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/apps/desktop/src-tauri/src/recording.rs b/apps/desktop/src-tauri/src/recording.rs index 195f74538b..6ee3894ebe 100644 --- a/apps/desktop/src-tauri/src/recording.rs +++ b/apps/desktop/src-tauri/src/recording.rs @@ -285,15 +285,11 @@ pub enum RecordingAction { /// - `{moment:hh:mm A}` - 12-hour time with AM/PM /// - `{moment:YYYY-MM-DD HH:mm}` - Combined custom format /// -/// ## Examples +/// ## Example /// /// `{recording_mode} Recording {target_kind} ({target_name}) {date} {time}` /// -> "Instant Recording Display (Built-in Retina Display) 2025-11-12 3:23 PM" /// -/// `Cap {target} - {date} {time}` -/// -> "Cap Display (Built-in Retina Display) - 2025-11-12 15:23" -/// -/// /// # Arguments /// /// * `template` - The template string with variables to replace From 56996e22670947a848d4af6b9fa1b7847e0d6e8f Mon Sep 17 00:00:00 2001 From: Ilya <47112191+ItsEeleeya@users.noreply.github.com> Date: Mon, 10 Nov 2025 19:13:26 +0330 Subject: [PATCH 09/46] Update rustdoc for format_project_name (again) Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- apps/desktop/src-tauri/src/recording.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/apps/desktop/src-tauri/src/recording.rs b/apps/desktop/src-tauri/src/recording.rs index 6ee3894ebe..cb12c4e823 100644 --- a/apps/desktop/src-tauri/src/recording.rs +++ b/apps/desktop/src-tauri/src/recording.rs @@ -293,7 +293,10 @@ pub enum RecordingAction { /// # Arguments /// /// * `template` - The template string with variables to replace -/// * `inputs` - The recording inputs containing target and mode information +/// * `target_name` - The specific name of the capture target +/// * `target_kind` - The type of capture target (Display, Window, or Area) +/// * `recording_mode` - The recording mode (Studio or Instant) +/// * `datetime` - Optional datetime to use for formatting; defaults to current time /// /// # Returns /// From 925927de2761d64092c57ad4820de875bc7a5978 Mon Sep 17 00:00:00 2001 From: Ilya <47112191+ItsEeleeya@users.noreply.github.com> Date: Wed, 19 Nov 2025 14:20:52 +0330 Subject: [PATCH 10/46] Update tauri.ts --- apps/desktop/src/utils/tauri.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/desktop/src/utils/tauri.ts b/apps/desktop/src/utils/tauri.ts index 4dd6618c1e..6bec54e407 100644 --- a/apps/desktop/src/utils/tauri.ts +++ b/apps/desktop/src/utils/tauri.ts @@ -407,7 +407,7 @@ export type ExportSettings = ({ format: "Mp4" } & Mp4ExportSettings) | ({ format export type FileType = "recording" | "screenshot" export type Flags = { captions: boolean } export type FramesRendered = { renderedCount: number; totalFrames: number; type: "FramesRendered" } -export type GeneralSettingsStore = { instanceId?: string; uploadIndividualFiles?: boolean; hideDockIcon?: boolean; autoCreateShareableLink?: boolean; enableNotifications?: boolean; disableAutoOpenLinks?: boolean; hasCompletedStartup?: boolean; theme?: AppTheme; commercialLicense?: CommercialLicense | null; lastVersion?: string | null; windowTransparency?: boolean; postStudioRecordingBehaviour?: PostStudioRecordingBehaviour; mainWindowRecordingStartBehaviour?: MainWindowRecordingStartBehaviour; custom_cursor_capture2?: boolean; serverUrl?: string; recordingCountdown?: number | null; enableNativeCameraPreview: boolean; autoZoomOnClicks?: boolean; enableNewRecordingFlow: boolean; recordingPickerPreferenceSet?: boolean; postDeletionBehaviour?: PostDeletionBehaviour; excludedWindows?: WindowExclusion[]; deleteInstantRecordingsAfterUpload?: boolean; instantModeMaxResolution?: number } +export type GeneralSettingsStore = { instanceId?: string; uploadIndividualFiles?: boolean; hideDockIcon?: boolean; autoCreateShareableLink?: boolean; enableNotifications?: boolean; disableAutoOpenLinks?: boolean; hasCompletedStartup?: boolean; theme?: AppTheme; commercialLicense?: CommercialLicense | null; lastVersion?: string | null; windowTransparency?: boolean; postStudioRecordingBehaviour?: PostStudioRecordingBehaviour; mainWindowRecordingStartBehaviour?: MainWindowRecordingStartBehaviour; custom_cursor_capture2?: boolean; serverUrl?: string; recordingCountdown?: number | null; enableNativeCameraPreview: boolean; autoZoomOnClicks?: boolean; enableNewRecordingFlow: boolean; recordingPickerPreferenceSet?: boolean; postDeletionBehaviour?: PostDeletionBehaviour; excludedWindows?: WindowExclusion[]; deleteInstantRecordingsAfterUpload?: boolean; instantModeMaxResolution?: number; defaultProjectNameTemplate?: string | null } export type GifExportSettings = { fps: number; resolution_base: XY; quality: GifQuality | null } export type GifQuality = { /** From eb12d2f5b4716554c03e7d1d5ecc7b9e9ac95559 Mon Sep 17 00:00:00 2001 From: Ilya <47112191+ItsEeleeya@users.noreply.github.com> Date: Wed, 19 Nov 2025 14:22:44 +0330 Subject: [PATCH 11/46] Remove needless borrowing --- apps/desktop/src-tauri/src/recording.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/desktop/src-tauri/src/recording.rs b/apps/desktop/src-tauri/src/recording.rs index 0ee15ee3e1..8deb4f55d6 100644 --- a/apps/desktop/src-tauri/src/recording.rs +++ b/apps/desktop/src-tauri/src/recording.rs @@ -408,7 +408,7 @@ pub fn format_project_name<'a>( static ref TIME_REGEX: Regex = Regex::new(r"\{time(?::([^}]+))?\}").unwrap(); static ref MOMENT_REGEX: Regex = Regex::new(r"\{moment(?::([^}]+))?\}").unwrap(); static ref AC: aho_corasick::AhoCorasick = { - aho_corasick::AhoCorasick::new(&[ + aho_corasick::AhoCorasick::new([ "{recording_mode}", "{mode}", "{target_kind}", @@ -426,7 +426,7 @@ pub fn format_project_name<'a>( }; let result = AC - .try_replace_all(&haystack, &[recording_mode, mode, target_kind, target_name]) + .try_replace_all(haystack, &[recording_mode, mode, target_kind, target_name]) .expect("AhoCorasick replace should never fail with default configuration"); let result = DATE_REGEX.replace_all(&result, |caps: ®ex::Captures| { From ffbf6b8b0f07f4647d0daaf9256e0520162d9ae3 Mon Sep 17 00:00:00 2001 From: Ilya <47112191+ItsEeleeya@users.noreply.github.com> Date: Wed, 19 Nov 2025 14:54:27 +0330 Subject: [PATCH 12/46] Measure project name migration times --- .../src-tauri/src/update_project_names.rs | 60 ++++++++++++++++--- 1 file changed, 51 insertions(+), 9 deletions(-) diff --git a/apps/desktop/src-tauri/src/update_project_names.rs b/apps/desktop/src-tauri/src/update_project_names.rs index ed49653240..88cd0994f8 100644 --- a/apps/desktop/src-tauri/src/update_project_names.rs +++ b/apps/desktop/src-tauri/src/update_project_names.rs @@ -36,6 +36,8 @@ pub fn migrate_if_needed(app: &AppHandle) -> Result<(), String> { Ok(()) } +use std::time::Instant; + /// Performs a one-time migration of all UUID-named projects to pretty name-based naming. pub async fn migrate(app: &AppHandle) -> Result<(), String> { let recordings_dir = recordings_path(app); @@ -61,38 +63,78 @@ pub async fn migrate(app: &AppHandle) -> Result<(), String> { let concurrency_limit = std::thread::available_parallelism() .map(|n| n.get()) .unwrap_or(4) - .max(2) - .min(16) + .clamp(2, 16) .min(total_found); tracing::debug!("Using concurrency limit of {}", concurrency_limit); + let wall_start = Instant::now(); + + // (project_name, result, duration) let migration_results = futures::stream::iter(uuid_projects) - .map(migrate_single_project) + .map(|project_path| async move { + let project_name = project_path + .file_name() + .map(|s| s.to_string_lossy().into_owned()) + .unwrap_or_else(|| project_path.display().to_string()); + + let start = Instant::now(); + let res = migrate_single_project(project_path).await; + let dur = start.elapsed(); + + (project_name, res, dur) + }) .buffer_unordered(concurrency_limit) .collect::>() .await; - // Aggregate results - let mut migrated = 0; - let mut skipped = 0; - let mut failed = 0; + let wall_elapsed = wall_start.elapsed(); + + let mut migrated = 0usize; + let mut skipped = 0usize; + let mut failed = 0usize; - for result in migration_results { + let mut total_ms: u128 = 0; + let mut per_project: Vec<(String, std::time::Duration)> = + Vec::with_capacity(migration_results.len()); + + for (name, result, dur) in migration_results.into_iter() { match result { Ok(ProjectMigrationResult::Migrated) => migrated += 1, Ok(ProjectMigrationResult::Skipped) => skipped += 1, Err(_) => failed += 1, } + total_ms += dur.as_millis(); + per_project.push((name, dur)); } + let avg_ms = if total_found > 0 { + (total_ms as f64) / (total_found as f64) + } else { + 0.0 + }; + + // Sort by duration descending to pick slowest + per_project.sort_by(|a, b| b.1.cmp(&a.1)); + tracing::info!( total_found = total_found, migrated = migrated, skipped = skipped, failed = failed, + wall_ms = wall_elapsed.as_millis(), + avg_per_project_ms = ?avg_ms, "Migration complete" ); + // Log top slowest N (choose 5 or less) + let top_n = 5.min(per_project.len()); + if top_n > 0 { + tracing::info!("Top {} slowest project migrations:", top_n); + for (name, dur) in per_project.into_iter().take(top_n) { + tracing::info!(project = %name, ms = dur.as_millis()); + } + } + Ok(()) } @@ -166,7 +208,7 @@ async fn migrate_project_filename_async( project_path: &Path, meta: &RecordingMeta, ) -> Result { - let sanitized = sanitize_filename::sanitize(&meta.pretty_name.replace(":", ".")); + let sanitized = sanitize_filename::sanitize(meta.pretty_name.replace(":", ".")); let filename = if sanitized.ends_with(".cap") { sanitized From 81d395c64f4bb49bd93e30cad1a81b7b60e3fac2 Mon Sep 17 00:00:00 2001 From: Ilya <47112191+ItsEeleeya@users.noreply.github.com> Date: Wed, 19 Nov 2025 14:59:16 +0330 Subject: [PATCH 13/46] format --- packages/ui-solid/tailwind.config.js | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/ui-solid/tailwind.config.js b/packages/ui-solid/tailwind.config.js index 0092467055..094a6bad66 100644 --- a/packages/ui-solid/tailwind.config.js +++ b/packages/ui-solid/tailwind.config.js @@ -90,10 +90,16 @@ module.exports = { keyframes: { "collapsible-down": { from: { height: 0, filter: "blur(4px)" }, - to: { height: "var(--kb-collapsible-content-height)", filter: "blur(0px)" }, + to: { + height: "var(--kb-collapsible-content-height)", + filter: "blur(0px)", + }, }, "collapsible-up": { - from: { height: "var(--kb-collapsible-content-height)", filter: "blur(0px)" }, + from: { + height: "var(--kb-collapsible-content-height)", + filter: "blur(0px)", + }, to: { height: 0, filter: "blur(4px)" }, }, }, From 2f5a267fa4516429f6f3a09bf02e7bb5838d2a8e Mon Sep 17 00:00:00 2001 From: Ilya <47112191+ItsEeleeya@users.noreply.github.com> Date: Wed, 19 Nov 2025 15:27:33 +0330 Subject: [PATCH 14/46] Avoid race condition (TOCTOU) is multiple files have the same pretty name --- .../src-tauri/src/update_project_names.rs | 45 ++++++++++++++++++- crates/utils/src/lib.rs | 13 +++++- 2 files changed, 54 insertions(+), 4 deletions(-) diff --git a/apps/desktop/src-tauri/src/update_project_names.rs b/apps/desktop/src-tauri/src/update_project_names.rs index 88cd0994f8..46a3c35520 100644 --- a/apps/desktop/src-tauri/src/update_project_names.rs +++ b/apps/desktop/src-tauri/src/update_project_names.rs @@ -1,7 +1,11 @@ -use std::path::{Path, PathBuf}; +use std::{ + collections::HashSet, + path::{Path, PathBuf}, +}; use cap_project::RecordingMeta; use futures::StreamExt; +use lazy_static::lazy_static; use tauri::AppHandle; use tokio::fs; @@ -9,6 +13,11 @@ use crate::recordings_path; const STORE_KEY: &str = "uuid_projects_migrated"; +lazy_static! { + static ref IN_FLIGHT_BASES: tokio::sync::Mutex> = + tokio::sync::Mutex::new(HashSet::new()); +} + pub fn migrate_if_needed(app: &AppHandle) -> Result<(), String> { use tauri_plugin_store::StoreExt; @@ -186,7 +195,39 @@ async fn migrate_single_project(path: PathBuf) -> Result 0 { + tracing::debug!( + "Project {} acquired lock for \"{}\" after {} waits", + filename, + base_name, + wait_count + ); + } + } + + let result = migrate_project_filename_async(&path, &meta).await; + + IN_FLIGHT_BASES.lock().await.remove(&base_name); + + match result { Ok(new_path) => { if new_path != path { let new_name = new_path.file_name().unwrap().to_string_lossy(); diff --git a/crates/utils/src/lib.rs b/crates/utils/src/lib.rs index 9c7d828f1e..f65765cf5e 100644 --- a/crates/utils/src/lib.rs +++ b/crates/utils/src/lib.rs @@ -35,16 +35,25 @@ pub fn ensure_dir(path: &PathBuf) -> Result { /// # Example /// /// ```rust -/// let unique_name = ensure_unique_filename("My Recording.cap", &recordings_dir); +/// let unique_name = ensure_unique_filename("My Recording.cap", &recordings_dir,); /// // If "My Recording.cap" exists, returns "My Recording (1).cap" /// // If that exists too, returns "My Recording (2).cap", etc. /// /// let unique_name = ensure_unique_filename("document.pdf", &documents_dir); /// // If "document.pdf" exists, returns "document (1).pdf" /// ``` +#[inline] pub fn ensure_unique_filename( base_filename: &str, parent_dir: &std::path::Path, +) -> Result { + ensure_unique_filename_with_attempts(base_filename, parent_dir, 20) +} + +pub fn ensure_unique_filename_with_attempts( + base_filename: &str, + parent_dir: &std::path::Path, + attempts: i32, ) -> Result { let initial_path = parent_dir.join(base_filename); @@ -82,7 +91,7 @@ pub fn ensure_unique_filename( counter += 1; // prevent infinite loop - if counter > 1000 { + if counter > attempts { return Err( "Too many filename conflicts, unable to create unique filename".to_string(), ); From c147e8b1f7c9c832115e80ef80770294d07a8e5a Mon Sep 17 00:00:00 2001 From: Ilya <47112191+ItsEeleeya@users.noreply.github.com> Date: Wed, 19 Nov 2025 15:31:08 +0330 Subject: [PATCH 15/46] cargo fmt --- apps/desktop/src-tauri/src/update_project_names.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/desktop/src-tauri/src/update_project_names.rs b/apps/desktop/src-tauri/src/update_project_names.rs index 46a3c35520..d5f7f908dd 100644 --- a/apps/desktop/src-tauri/src/update_project_names.rs +++ b/apps/desktop/src-tauri/src/update_project_names.rs @@ -224,9 +224,9 @@ async fn migrate_single_project(path: PathBuf) -> Result { if new_path != path { From 03d737b759bc9d0426695fed9466774ccfa645c3 Mon Sep 17 00:00:00 2001 From: Ilya <47112191+ItsEeleeya@users.noreply.github.com> Date: Wed, 19 Nov 2025 15:51:29 +0330 Subject: [PATCH 16/46] Use NonZero --- .../(window-chrome)/settings/general.tsx | 6 ++--- crates/utils/src/lib.rs | 24 ++++++++++++------- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/apps/desktop/src/routes/(window-chrome)/settings/general.tsx b/apps/desktop/src/routes/(window-chrome)/settings/general.tsx index fcda2cd2c6..c4b21dd445 100644 --- a/apps/desktop/src/routes/(window-chrome)/settings/general.tsx +++ b/apps/desktop/src/routes/(window-chrome)/settings/general.tsx @@ -743,9 +743,9 @@ function DefaultProjectNameCard(props: { size="sm" variant="dark" disabled={isSaveDisabled()} - onClick={() => { - props.onChange(inputValue() ?? null); - updatePreview(); + onClick={async () => { + await props.onChange(inputValue() ?? null); + await updatePreview(); }} > Save diff --git a/crates/utils/src/lib.rs b/crates/utils/src/lib.rs index f65765cf5e..4928f787f9 100644 --- a/crates/utils/src/lib.rs +++ b/crates/utils/src/lib.rs @@ -1,4 +1,10 @@ -use std::{borrow::Cow, future::Future, path::PathBuf, sync::LazyLock}; +use std::{ + borrow::Cow, + future::Future, + num::{NonZero, NonZeroI16, NonZeroI32}, + path::PathBuf, + sync::LazyLock, +}; use aho_corasick::{AhoCorasickBuilder, MatchKind}; use tracing::Instrument; @@ -47,13 +53,18 @@ pub fn ensure_unique_filename( base_filename: &str, parent_dir: &std::path::Path, ) -> Result { - ensure_unique_filename_with_attempts(base_filename, parent_dir, 20) + ensure_unique_filename_with_attempts( + base_filename, + parent_dir, + // SAFETY: 20 is non zero + unsafe { NonZero::new_unchecked(50) }, + ) } pub fn ensure_unique_filename_with_attempts( base_filename: &str, parent_dir: &std::path::Path, - attempts: i32, + attempts: NonZeroI32, ) -> Result { let initial_path = parent_dir.join(base_filename); @@ -73,6 +84,7 @@ pub fn ensure_unique_filename_with_attempts( (base_filename, String::new()) }; + let max_attemps = attempts.get(); let mut counter = 1; loop { @@ -91,7 +103,7 @@ pub fn ensure_unique_filename_with_attempts( counter += 1; // prevent infinite loop - if counter > attempts { + if counter > max_attemps { return Err( "Too many filename conflicts, unable to create unique filename".to_string(), ); @@ -162,10 +174,6 @@ pub fn ensure_unique_filename_with_attempts( /// // 12-hour format with full names /// DDDD, MMMM DD at h:mm A → %A, %B %d at %-I:%M %p /// // Output: "Monday, January 15 at 2:30 PM" -/// -/// // ISO week date -/// YYYY-Www-D → %G-W%V-%u -/// // Output: "2025-W03-1" /// ``` /// /// # Note From 09025351f951334cfc39e1fc85016e65d54d4c41 Mon Sep 17 00:00:00 2001 From: Ilya <47112191+ItsEeleeya@users.noreply.github.com> Date: Wed, 19 Nov 2025 17:31:53 +0330 Subject: [PATCH 17/46] don't use static ref for in_flight_bases + cleanup and update comments --- .../src-tauri/src/update_project_names.rs | 46 ++++++++++--------- crates/utils/src/lib.rs | 17 ++++--- 2 files changed, 34 insertions(+), 29 deletions(-) diff --git a/apps/desktop/src-tauri/src/update_project_names.rs b/apps/desktop/src-tauri/src/update_project_names.rs index d5f7f908dd..f52737ab76 100644 --- a/apps/desktop/src-tauri/src/update_project_names.rs +++ b/apps/desktop/src-tauri/src/update_project_names.rs @@ -1,23 +1,18 @@ use std::{ collections::HashSet, path::{Path, PathBuf}, + sync::Arc, }; use cap_project::RecordingMeta; use futures::StreamExt; -use lazy_static::lazy_static; use tauri::AppHandle; -use tokio::fs; +use tokio::{fs, sync::Mutex}; use crate::recordings_path; const STORE_KEY: &str = "uuid_projects_migrated"; -lazy_static! { - static ref IN_FLIGHT_BASES: tokio::sync::Mutex> = - tokio::sync::Mutex::new(HashSet::new()); -} - pub fn migrate_if_needed(app: &AppHandle) -> Result<(), String> { use tauri_plugin_store::StoreExt; @@ -77,20 +72,24 @@ pub async fn migrate(app: &AppHandle) -> Result<(), String> { tracing::debug!("Using concurrency limit of {}", concurrency_limit); let wall_start = Instant::now(); + let in_flight_bases = Arc::new(Mutex::new(HashSet::new())); // (project_name, result, duration) let migration_results = futures::stream::iter(uuid_projects) - .map(|project_path| async move { - let project_name = project_path - .file_name() - .map(|s| s.to_string_lossy().into_owned()) - .unwrap_or_else(|| project_path.display().to_string()); - - let start = Instant::now(); - let res = migrate_single_project(project_path).await; - let dur = start.elapsed(); - - (project_name, res, dur) + .map(|project_path| { + let in_flight = in_flight_bases.clone(); + async move { + let project_name = project_path + .file_name() + .map(|s| s.to_string_lossy().into_owned()) + .unwrap_or_else(|| project_path.display().to_string()); + + let start = Instant::now(); + let res = migrate_single_project(project_path, in_flight).await; + let dur = start.elapsed(); + + (project_name, res, dur) + } }) .buffer_unordered(concurrency_limit) .collect::>() @@ -181,7 +180,10 @@ enum ProjectMigrationResult { Skipped, } -async fn migrate_single_project(path: PathBuf) -> Result { +async fn migrate_single_project( + path: PathBuf, + in_flight_basis: Arc>>, +) -> Result { let filename = path .file_name() .and_then(|s| s.to_str()) @@ -198,7 +200,7 @@ async fn migrate_single_project(path: PathBuf) -> Result Result 0 { tracing::debug!( @@ -225,7 +227,7 @@ async fn migrate_single_project(path: PathBuf) -> Result { diff --git a/crates/utils/src/lib.rs b/crates/utils/src/lib.rs index 4928f787f9..e7d38edf2d 100644 --- a/crates/utils/src/lib.rs +++ b/crates/utils/src/lib.rs @@ -1,7 +1,7 @@ use std::{ borrow::Cow, future::Future, - num::{NonZero, NonZeroI16, NonZeroI32}, + num::{NonZero, NonZeroI32}, path::PathBuf, sync::LazyLock, }; @@ -56,7 +56,7 @@ pub fn ensure_unique_filename( ensure_unique_filename_with_attempts( base_filename, parent_dir, - // SAFETY: 20 is non zero + // SAFETY: 50 is non zero unsafe { NonZero::new_unchecked(50) }, ) } @@ -84,7 +84,7 @@ pub fn ensure_unique_filename_with_attempts( (base_filename, String::new()) }; - let max_attemps = attempts.get(); + let max_attempts = attempts.get(); let mut counter = 1; loop { @@ -103,7 +103,7 @@ pub fn ensure_unique_filename_with_attempts( counter += 1; // prevent infinite loop - if counter > max_attemps { + if counter > max_attempts { return Err( "Too many filename conflicts, unable to create unique filename".to_string(), ); @@ -111,10 +111,13 @@ pub fn ensure_unique_filename_with_attempts( } } -/// Converts user-friendly moment template format strings to chrono format strings. +/// Converts moment-style template format strings to chrono format strings. /// -/// This function translates common template format patterns to chrono format specifiers, -/// allowing users to write intuitive date/time formats that get converted to chrono's format. +/// This function translates a custom subset of date/time patterns to chrono format specifiers. +/// +/// **Note**: This is NOT fully compatible with moment.js. Notably, `DDD`/`DDDD` map to +/// weekday names here, whereas in moment.js they represent day-of-year. Day-of-year and +/// ISO week tokens are not supported. /// /// # Supported Format Patterns /// From 9331f84d3ae9cd136f2dbef69430ed471e6ff170 Mon Sep 17 00:00:00 2001 From: Ilya <47112191+ItsEeleeya@users.noreply.github.com> Date: Wed, 19 Nov 2025 17:59:11 +0330 Subject: [PATCH 18/46] More tests for cap-utils --- Cargo.lock | 1 + crates/utils/Cargo.toml | 3 + crates/utils/src/lib.rs | 199 +++++++++++++++++++++++++++++++++++++--- 3 files changed, 188 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0e6f7cc759..1465878c16 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1608,6 +1608,7 @@ dependencies = [ "nix 0.29.0", "serde", "serde_json", + "tempfile", "tokio", "tracing", "uuid", diff --git a/crates/utils/Cargo.toml b/crates/utils/Cargo.toml index 1c01fbd36d..66c180a553 100644 --- a/crates/utils/Cargo.toml +++ b/crates/utils/Cargo.toml @@ -30,5 +30,8 @@ directories = "5.0" aho-corasick.workspace = true workspace-hack = { version = "0.1", path = "../workspace-hack" } +[dev-dependencies] +tempfile = "3" + [lints] workspace = true diff --git a/crates/utils/src/lib.rs b/crates/utils/src/lib.rs index e7d38edf2d..7b0ed42f28 100644 --- a/crates/utils/src/lib.rs +++ b/crates/utils/src/lib.rs @@ -53,12 +53,8 @@ pub fn ensure_unique_filename( base_filename: &str, parent_dir: &std::path::Path, ) -> Result { - ensure_unique_filename_with_attempts( - base_filename, - parent_dir, - // SAFETY: 50 is non zero - unsafe { NonZero::new_unchecked(50) }, - ) + const DEFAULT_MAX_ATTEMPTS: NonZero = NonZero::new(50).unwrap(); + ensure_unique_filename_with_attempts(base_filename, parent_dir, DEFAULT_MAX_ATTEMPTS) } pub fn ensure_unique_filename_with_attempts( @@ -66,6 +62,10 @@ pub fn ensure_unique_filename_with_attempts( parent_dir: &std::path::Path, attempts: NonZeroI32, ) -> Result { + if base_filename.contains('/') || base_filename.contains('\\') { + return Err("Filename cannot contain path separators".to_string()); + } + let initial_path = parent_dir.join(base_filename); if !initial_path.exists() { @@ -220,23 +220,192 @@ pub fn moment_format_to_chrono(template_format: &str) -> Cow<'_, str> { #[cfg(test)] mod tests { use super::*; + use std::fs; + + // moment_format_to_chrono tests #[test] - fn moment_format_to_chrono_converts_and_preserves_borrowed_when_unchanged() { + fn moment_format_converts_all_patterns() { let input = "YYYY-MM-DD HH:mm:ss A a DDDD - DD - MMMM"; let out = moment_format_to_chrono(input); let expected = "%Y-%m-%d %H:%M:%S %p %P %A - %d - %B"; + assert_eq!(out, expected); + } + + #[test] + fn moment_format_handles_overlapping_patterns() { + // MMMM should be matched before MMM, MM, M + assert_eq!(moment_format_to_chrono("MMMM"), "%B"); + assert_eq!(moment_format_to_chrono("MMM"), "%b"); + assert_eq!(moment_format_to_chrono("MM"), "%m"); + assert_eq!(moment_format_to_chrono("M"), "%-m"); + + // DDDD should be matched before DDD, DD, D + assert_eq!(moment_format_to_chrono("DDDD"), "%A"); + assert_eq!(moment_format_to_chrono("DDD"), "%a"); + assert_eq!(moment_format_to_chrono("DD"), "%d"); + assert_eq!(moment_format_to_chrono("D"), "%-d"); + } + + #[test] + fn moment_format_handles_adjacent_tokens() { + // No separator between tokens + assert_eq!(moment_format_to_chrono("YYYYMMDD"), "%Y%m%d"); + assert_eq!(moment_format_to_chrono("HHmmss"), "%H%M%S"); assert_eq!( - out, expected, - "Converted format must match expected chrono format" + moment_format_to_chrono("DDDDMMMMYYYYHHmmss"), + "%A%B%Y%H%M%S" ); + } + + #[test] + fn moment_format_handles_12_and_24_hour() { + assert_eq!(moment_format_to_chrono("HH:mm"), "%H:%M"); // 24-hour + assert_eq!(moment_format_to_chrono("hh:mm A"), "%I:%M %p"); // 12-hour + assert_eq!(moment_format_to_chrono("H"), "%-H"); // No padding + assert_eq!(moment_format_to_chrono("h"), "%-I"); // No padding + } + + #[test] + fn moment_format_handles_padding_variants() { + // Padded versions + assert_eq!(moment_format_to_chrono("DD"), "%d"); + assert_eq!(moment_format_to_chrono("MM"), "%m"); + assert_eq!(moment_format_to_chrono("HH"), "%H"); + + // Unpadded versions + assert_eq!(moment_format_to_chrono("D"), "%-d"); + assert_eq!(moment_format_to_chrono("M"), "%-m"); + assert_eq!(moment_format_to_chrono("H"), "%-H"); + } - // Identity / borrowed case: no tokens -> should return Cow::Borrowed - let unchanged = "--"; - let out2 = moment_format_to_chrono(unchanged); - match out2 { - Cow::Borrowed(s) => assert_eq!(s, unchanged), - Cow::Owned(_) => panic!("Expected Cow::Borrowed for unchanged input"), + #[test] + fn moment_format_empty_string() { + let out = moment_format_to_chrono(""); + match out { + Cow::Borrowed(s) => assert_eq!(s, ""), + Cow::Owned(_) => panic!("Expected Cow::Borrowed for empty string"), } } + + // ensure_unique_filename tests + + #[test] + fn unique_filename_when_no_conflict() { + let temp_dir = tempfile::tempdir().unwrap(); + let result = ensure_unique_filename("test.cap", temp_dir.path()).unwrap(); + assert_eq!(result, "test.cap"); + } + + #[test] + fn unique_filename_appends_counter_on_conflict() { + let temp_dir = tempfile::tempdir().unwrap(); + + // Create existing file + fs::write(temp_dir.path().join("test.cap"), "").unwrap(); + + let result = ensure_unique_filename("test.cap", temp_dir.path()).unwrap(); + assert_eq!(result, "test (1).cap"); + } + + #[test] + fn unique_filename_increments_counter() { + let temp_dir = tempfile::tempdir().unwrap(); + + // Create existing files + fs::write(temp_dir.path().join("test.cap"), "").unwrap(); + fs::write(temp_dir.path().join("test (1).cap"), "").unwrap(); + fs::write(temp_dir.path().join("test (2).cap"), "").unwrap(); + + let result = ensure_unique_filename("test.cap", temp_dir.path()).unwrap(); + assert_eq!(result, "test (3).cap"); + } + + #[test] + fn unique_filename_handles_no_extension() { + let temp_dir = tempfile::tempdir().unwrap(); + + fs::write(temp_dir.path().join("README"), "").unwrap(); + + let result = ensure_unique_filename("README", temp_dir.path()).unwrap(); + assert_eq!(result, "README (1)"); + } + + #[test] + fn unique_filename_handles_multiple_dots() { + let temp_dir = tempfile::tempdir().unwrap(); + + fs::write(temp_dir.path().join("archive.tar.gz"), "").unwrap(); + + let result = ensure_unique_filename("archive.tar.gz", temp_dir.path()).unwrap(); + // Only the last extension is considered + assert_eq!(result, "archive.tar (1).gz"); + } + + #[test] + fn unique_filename_respects_max_attempts() { + let temp_dir = tempfile::tempdir().unwrap(); + + // Create base file + fs::write(temp_dir.path().join("test.cap"), "").unwrap(); + + // Try with only 3 attempts + let attempts = NonZero::new(3).unwrap(); + + // Create conflicts for attempts 1, 2, 3 + fs::write(temp_dir.path().join("test (1).cap"), "").unwrap(); + fs::write(temp_dir.path().join("test (2).cap"), "").unwrap(); + fs::write(temp_dir.path().join("test (3).cap"), "").unwrap(); + + let result = ensure_unique_filename_with_attempts("test.cap", temp_dir.path(), attempts); + + assert!(result.is_err()); + assert!(result.unwrap_err().contains("Too many filename conflicts")); + } + + #[test] + fn unique_filename_handles_directories_as_conflicts() { + let temp_dir = tempfile::tempdir().unwrap(); + + // Create a directory with the target name + fs::create_dir(temp_dir.path().join("test.cap")).unwrap(); + + let result = ensure_unique_filename("test.cap", temp_dir.path()).unwrap(); + assert_eq!(result, "test (1).cap"); + } + + #[test] + fn unique_filename_handles_special_characters() { + let temp_dir = tempfile::tempdir().unwrap(); + + fs::write(temp_dir.path().join("My Recording (2024).cap"), "").unwrap(); + + let result = ensure_unique_filename("My Recording (2024).cap", temp_dir.path()).unwrap(); + assert_eq!(result, "My Recording (2024) (1).cap"); + } + + #[test] + fn unique_filename_handles_spaces() { + let temp_dir = tempfile::tempdir().unwrap(); + + fs::write(temp_dir.path().join("My Project.cap"), "").unwrap(); + + let result = ensure_unique_filename("My Project.cap", temp_dir.path()).unwrap(); + assert_eq!(result, "My Project (1).cap"); + } + + #[test] + fn unique_filename_finds_gap_in_sequence() { + let temp_dir = tempfile::tempdir().unwrap(); + + // Create files with a gap in numbering + fs::write(temp_dir.path().join("test.cap"), "").unwrap(); + fs::write(temp_dir.path().join("test (1).cap"), "").unwrap(); + // Gap: test (2).cap doesn't exist + fs::write(temp_dir.path().join("test (3).cap"), "").unwrap(); + + let result = ensure_unique_filename("test.cap", temp_dir.path()).unwrap(); + // Should find the gap at (2) + assert_eq!(result, "test (2).cap"); + } } From bd32090084b4c6285cd095dcbaa657c4c9c9be09 Mon Sep 17 00:00:00 2001 From: Ilya <47112191+ItsEeleeya@users.noreply.github.com> Date: Wed, 19 Nov 2025 18:07:27 +0330 Subject: [PATCH 19/46] Don't propagate migration error --- apps/desktop/src-tauri/src/lib.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/apps/desktop/src-tauri/src/lib.rs b/apps/desktop/src-tauri/src/lib.rs index 58ba9d391e..81fa4b9062 100644 --- a/apps/desktop/src-tauri/src/lib.rs +++ b/apps/desktop/src-tauri/src/lib.rs @@ -2444,7 +2444,11 @@ pub async fn run(recording_logging_handle: LoggingHandle, logs_dir: PathBuf) { .invoke_handler(specta_builder.invoke_handler()) .setup(move |app| { let app = app.handle().clone(); - update_project_names::migrate_if_needed(&app)?; + + if let Err(err) = update_project_names::migrate_if_needed(&app) { + tracing::error!("Failed to migrate project file names: {}", err); + } + specta_builder.mount_events(&app); hotkeys::init(&app); general_settings::init(&app); From 897f777d2d512fd0ae1f429bc56f9c0a78f126e6 Mon Sep 17 00:00:00 2001 From: Ilya <47112191+ItsEeleeya@users.noreply.github.com> Date: Wed, 19 Nov 2025 18:14:21 +0330 Subject: [PATCH 20/46] use tauri::command(async) on cap_desktop::format_project_name --- apps/desktop/src-tauri/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/desktop/src-tauri/src/lib.rs b/apps/desktop/src-tauri/src/lib.rs index 81fa4b9062..b268fb7b4f 100644 --- a/apps/desktop/src-tauri/src/lib.rs +++ b/apps/desktop/src-tauri/src/lib.rs @@ -3059,9 +3059,9 @@ async fn write_clipboard_string( .map_err(|e| format!("Failed to write text to clipboard: {e}")) } -#[tauri::command] +#[tauri::command(async)] #[specta::specta] -async fn format_project_name( +fn format_project_name( template: Option, target_name: String, target_kind: String, From e3f777ac311af82a677a7b01d60abeb0b35f2b86 Mon Sep 17 00:00:00 2001 From: Ilya <47112191+ItsEeleeya@users.noreply.github.com> Date: Sat, 29 Nov 2025 12:17:55 +0330 Subject: [PATCH 21/46] Add match arm for screenshot mode --- apps/desktop/src-tauri/src/recording.rs | 43 +------------------------ 1 file changed, 1 insertion(+), 42 deletions(-) diff --git a/apps/desktop/src-tauri/src/recording.rs b/apps/desktop/src-tauri/src/recording.rs index ba041f078a..0fa07d2526 100644 --- a/apps/desktop/src-tauri/src/recording.rs +++ b/apps/desktop/src-tauri/src/recording.rs @@ -352,47 +352,6 @@ pub enum RecordingAction { UpgradeRequired, } -/// Formats the project name using a template string. -/// -/// # Template Variables -/// -/// The template supports the following variables that will be replaced with actual values: -/// -/// ## Recording Mode Variables -/// - `{recording_mode}` - The recording mode: "Studio" or "Instant" -/// - `{mode}` - Short form of recording mode: "studio" or "instant" -/// -/// ## Target Variables -/// - `{target_kind}` - The type of capture target: "Display", "Window", or "Area" -/// - `{target_name}` - The specific name of the target (e.g., "Built-in Retina Display", "Chrome", etc.) -/// -/// ## Date/Time Variables -/// - `{date}` - Current date in YYYY-MM-DD format (e.g., "2025-09-11") -/// - `{time}` - Current time in HH:MM AM/PM format (e.g., "3:23 PM") -/// -/// ## Customizable Date/Time Formats -/// You can customize date and time formats by adding moment format specifiers: -/// - `{moment:YYYY-MM-DD}` - Custom date format -/// - `{moment:HH:mm}` - 24-hour time format -/// - `{moment:hh:mm A}` - 12-hour time with AM/PM -/// - `{moment:YYYY-MM-DD HH:mm}` - Combined custom format -/// -/// ## Example -/// -/// `{recording_mode} Recording {target_kind} ({target_name}) {date} {time}` -/// -> "Instant Recording Display (Built-in Retina Display) 2025-11-12 3:23 PM" -/// -/// # Arguments -/// -/// * `template` - The template string with variables to replace -/// * `target_name` - The specific name of the capture target -/// * `target_kind` - The type of capture target (Display, Window, or Area) -/// * `recording_mode` - The recording mode (Studio or Instant) -/// * `datetime` - Optional datetime to use for formatting; defaults to current time -/// -/// # Returns -/// -/// Returns `String` with the formatted project name pub fn format_project_name<'a>( template: Option<&str>, target_name: &'a str, @@ -423,6 +382,7 @@ pub fn format_project_name<'a>( let (recording_mode, mode) = match recording_mode { RecordingMode::Studio => ("Studio", "studio"), RecordingMode::Instant => ("Instant", "instant"), + RecordingMode::Screenshot => ("Screenshot", "screenshot"), }; let result = AC @@ -660,7 +620,6 @@ pub async fn start_recording( let state_mtx = Arc::clone(&state_mtx); let general_settings = general_settings.cloned(); let recording_dir = recording_dir.clone(); - // let project_name = project_name.clone(); let inputs = inputs.clone(); async move { fail!("recording::spawn_actor"); From 9776ecd8cd2422c3fd9f945efb28f654f48447c0 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 29 Nov 2025 21:25:20 +0800 Subject: [PATCH 22/46] Add logical size to display info and improve screenshot debug --- .../src-tauri/src/target_select_overlay.rs | 4 +- .../src/routes/target-select-overlay.tsx | 30 +++++++++++++-- crates/recording/src/screenshot.rs | 38 ++++++++++++++++--- 3 files changed, 62 insertions(+), 10 deletions(-) diff --git a/apps/desktop/src-tauri/src/target_select_overlay.rs b/apps/desktop/src-tauri/src/target_select_overlay.rs index d52424235e..1ea7a0642f 100644 --- a/apps/desktop/src-tauri/src/target_select_overlay.rs +++ b/apps/desktop/src-tauri/src/target_select_overlay.rs @@ -15,7 +15,7 @@ use crate::{ }; use scap_targets::{ Display, DisplayId, Window, WindowId, - bounds::{LogicalBounds, PhysicalSize}, + bounds::{LogicalBounds, LogicalSize, PhysicalSize}, }; use serde::Serialize; use specta::Type; @@ -42,6 +42,7 @@ pub struct WindowUnderCursor { pub struct DisplayInformation { name: Option, physical_size: Option, + logical_size: Option, refresh_rate: String, } @@ -217,6 +218,7 @@ pub async fn display_information(display_id: &str) -> Result ({ + queryKey: ["areaDisplayInfo", displayId()], + queryFn: async () => { + return await commands.displayInformation(displayId()); + }, + })); + const [aspect, setAspect] = createSignal(null); const [snapToRatioEnabled, setSnapToRatioEnabled] = createSignal(true); @@ -698,21 +705,36 @@ function Inner() { if (was && !interacting) { if (options.mode === "screenshot" && isValid()) { + const cropBounds = crop(); + const displayInfo = areaDisplayInfo.data; + console.log("[Screenshot Debug] crop bounds:", cropBounds); + console.log("[Screenshot Debug] display info:", displayInfo); + console.log( + "[Screenshot Debug] window.innerWidth/Height:", + window.innerWidth, + window.innerHeight, + ); + const target: ScreenCaptureTarget = { variant: "area", screen: displayId(), bounds: { position: { - x: crop().x, - y: crop().y, + x: cropBounds.x, + y: cropBounds.y, }, size: { - width: crop().width, - height: crop().height, + width: cropBounds.width, + height: cropBounds.height, }, }, }; + console.log( + "[Screenshot Debug] target being sent:", + JSON.stringify(target, null, 2), + ); + try { const path = await invoke("take_screenshot", { target, diff --git a/crates/recording/src/screenshot.rs b/crates/recording/src/screenshot.rs index 56adedf4c6..f3d1dfee05 100644 --- a/crates/recording/src/screenshot.rs +++ b/crates/recording/src/screenshot.rs @@ -149,13 +149,35 @@ fn try_fast_capture(target: &ScreenCaptureTarget) -> Option { let display = scap_targets::Display::from_id(screen)?; let display_id = display.raw_handle().inner().id; let scale = display.raw_handle().scale().unwrap_or(1.0); + let display_bounds = display.raw_handle().logical_bounds(); + let display_physical = display.physical_size(); + + tracing::info!( + "Area screenshot debug: display_id={}, display_logical_bounds={:?}, display_physical={:?}", + display_id, + display_bounds, + display_physical, + ); + tracing::info!( + "Area screenshot: input logical bounds=({}, {}, {}x{}), scale={}", + bounds.position().x(), + bounds.position().y(), + bounds.size().width(), + bounds.size().height(), + scale, + ); let rect = CGRect::new( - &CGPoint::new(bounds.position().x() * scale, bounds.position().y() * scale), - &CGSize::new( - bounds.size().width() * scale, - bounds.size().height() * scale, - ), + &CGPoint::new(bounds.position().x(), bounds.position().y()), + &CGSize::new(bounds.size().width(), bounds.size().height()), + ); + + tracing::info!( + "Area screenshot: CGRect for capture (logical/points) = origin({}, {}), size({}x{})", + rect.origin.x, + rect.origin.y, + rect.size.width, + rect.size.height, ); let image = unsafe { CGDisplayCreateImageForRect(display_id, rect) }; @@ -170,6 +192,12 @@ fn try_fast_capture(target: &ScreenCaptureTarget) -> Option { let height = cg_image.height(); let bytes_per_row = cg_image.bytes_per_row(); + tracing::info!( + "Fast capture result: image dimensions = {}x{}", + width, + height, + ); + use core_foundation::data::CFData; let cf_data: CFData = cg_image.data(); let data = cf_data.bytes(); From 5cc649a25fc7f5b22bbbac946911c9d68edb7335 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 29 Nov 2025 21:25:57 +0800 Subject: [PATCH 23/46] Add TrackManager component and update timeline track types --- .../routes/editor/Timeline/TrackManager.tsx | 77 +++++++++++++++++++ apps/desktop/src/routes/editor/context.ts | 9 ++- apps/desktop/src/utils/tauri.ts | 2 +- packages/ui-solid/src/auto-imports.d.ts | 1 + 4 files changed, 87 insertions(+), 2 deletions(-) create mode 100644 apps/desktop/src/routes/editor/Timeline/TrackManager.tsx diff --git a/apps/desktop/src/routes/editor/Timeline/TrackManager.tsx b/apps/desktop/src/routes/editor/Timeline/TrackManager.tsx new file mode 100644 index 0000000000..7b875de5e1 --- /dev/null +++ b/apps/desktop/src/routes/editor/Timeline/TrackManager.tsx @@ -0,0 +1,77 @@ +import { LogicalPosition } from "@tauri-apps/api/dpi"; +import { CheckMenuItem, Menu } from "@tauri-apps/api/menu"; +import type { JSX } from "solid-js"; + +import type { TimelineTrackType } from "../context"; + +type TrackManagerOption = { + type: TimelineTrackType; + label: string; + icon: JSX.Element; + active: boolean; + available: boolean; +}; + +export function TrackManager(props: { + options: TrackManagerOption[]; + onToggle(type: TimelineTrackType, next: boolean): void; +}) { + let addButton: HTMLButtonElement | undefined; + + const handleOpenMenu = async () => { + try { + const items = await Promise.all( + props.options.map((option) => { + if (option.type === "scene") { + return CheckMenuItem.new({ + text: option.label, + checked: option.active, + enabled: option.available, + action: () => props.onToggle(option.type, !option.active), + }); + } + + return CheckMenuItem.new({ + text: option.label, + checked: true, + enabled: false, + }); + }), + ); + + const menu = await Menu.new({ items }); + const rect = addButton?.getBoundingClientRect(); + if (rect) { + menu.popup(new LogicalPosition(rect.x, rect.y + rect.height + 4)); + } else { + menu.popup(); + } + } catch (error) { + console.error("Failed to open track menu", error); + } + }; + + return ( + + ); +} + +export function TrackIcon(props: { icon: JSX.Element }) { + return ( +
e.stopPropagation()} + > + {props.icon} +
+ ); +} diff --git a/apps/desktop/src/routes/editor/context.ts b/apps/desktop/src/routes/editor/context.ts index efc0d97d91..3aae121e9c 100644 --- a/apps/desktop/src/routes/editor/context.ts +++ b/apps/desktop/src/routes/editor/context.ts @@ -50,6 +50,8 @@ export const OUTPUT_SIZE = { y: 1080, }; +export type TimelineTrackType = "clip" | "zoom" | "scene"; + export const MAX_ZOOM_IN = 3; const PROJECT_SAVE_DEBOUNCE_MS = 250; @@ -477,7 +479,12 @@ export const [EditorContextProvider, useEditorContext] = createContextProvider( ); }, }, - hoveredTrack: null as null | "clip" | "zoom" | "scene", + tracks: { + clip: true, + zoom: true, + scene: true, + }, + hoveredTrack: null as null | TimelineTrackType, }, }); diff --git a/apps/desktop/src/utils/tauri.ts b/apps/desktop/src/utils/tauri.ts index e0e2b94359..d47dea60b1 100644 --- a/apps/desktop/src/utils/tauri.ts +++ b/apps/desktop/src/utils/tauri.ts @@ -409,7 +409,7 @@ export type CursorType = "pointer" | "circle" export type Cursors = { [key in string]: string } | { [key in string]: CursorMeta } export type DeviceOrModelID = { DeviceID: string } | { ModelID: ModelIDType } export type DisplayId = string -export type DisplayInformation = { name: string | null; physical_size: PhysicalSize | null; refresh_rate: string } +export type DisplayInformation = { name: string | null; physical_size: PhysicalSize | null; logical_size: LogicalSize | null; refresh_rate: string } export type DownloadProgress = { progress: number; message: string } export type EditorStateChanged = { playhead_position: number } export type ExportCompression = "Minimal" | "Social" | "Web" | "Potato" diff --git a/packages/ui-solid/src/auto-imports.d.ts b/packages/ui-solid/src/auto-imports.d.ts index 930b3ac259..c10b785a8b 100644 --- a/packages/ui-solid/src/auto-imports.d.ts +++ b/packages/ui-solid/src/auto-imports.d.ts @@ -69,6 +69,7 @@ declare global { const IconLucideBell: typeof import('~icons/lucide/bell.jsx')['default'] const IconLucideBug: typeof import('~icons/lucide/bug.jsx')['default'] const IconLucideCheck: typeof import('~icons/lucide/check.jsx')['default'] + const IconLucideClapperboard: typeof import('~icons/lucide/clapperboard.jsx')['default'] const IconLucideClock: typeof import('~icons/lucide/clock.jsx')['default'] const IconLucideDatabase: typeof import('~icons/lucide/database.jsx')['default'] const IconLucideEdit: typeof import('~icons/lucide/edit.jsx')['default'] From 6faa2f14cf10a18a91816ed3e5a895abe9f7a446 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 29 Nov 2025 21:26:10 +0800 Subject: [PATCH 24/46] Optimize frame rendering --- apps/desktop/src/routes/editor/Editor.tsx | 18 ++- .../src/routes/editor/Timeline/index.tsx | 149 +++++++++++++++--- crates/editor/src/editor.rs | 104 +++++++++--- crates/editor/src/editor_instance.rs | 103 +++++++----- crates/editor/src/playback.rs | 7 +- crates/rendering/src/decoder/avassetreader.rs | 5 +- crates/rendering/src/decoder/ffmpeg.rs | 5 +- 7 files changed, 297 insertions(+), 94 deletions(-) diff --git a/apps/desktop/src/routes/editor/Editor.tsx b/apps/desktop/src/routes/editor/Editor.tsx index 38020839e3..8b84d86db7 100644 --- a/apps/desktop/src/routes/editor/Editor.tsx +++ b/apps/desktop/src/routes/editor/Editor.tsx @@ -1,7 +1,7 @@ import { Button } from "@cap/ui-solid"; import { NumberField } from "@kobalte/core/number-field"; import { trackDeep } from "@solid-primitives/deep"; -import { throttle } from "@solid-primitives/scheduled"; +import { debounce, throttle } from "@solid-primitives/scheduled"; import { makePersisted } from "@solid-primitives/storage"; import { createMutation } from "@tanstack/solid-query"; import { convertFileSrc } from "@tauri-apps/api/core"; @@ -85,11 +85,12 @@ function Inner() { const { project, editorState, setEditorState } = useEditorContext(); createTauriEventListener(events.editorStateChanged, (payload) => { - renderFrame.clear(); + throttledRenderFrame.clear(); + trailingRenderFrame.clear(); setEditorState("playbackTime", payload.playhead_position / FPS); }); - const renderFrame = throttle((time: number) => { + const emitRenderFrame = (time: number) => { if (!editorState.playing) { events.renderFrameEvent.emit({ frame_number: Math.max(Math.floor(time * FPS), 0), @@ -97,7 +98,16 @@ function Inner() { resolution_base: OUTPUT_SIZE, }); } - }, 1000 / FPS); + }; + + const throttledRenderFrame = throttle(emitRenderFrame, 1000 / FPS); + + const trailingRenderFrame = debounce(emitRenderFrame, 1000 / FPS + 16); + + const renderFrame = (time: number) => { + throttledRenderFrame(time); + trailingRenderFrame(time); + }; const frameNumberToRender = createMemo(() => { const preview = editorState.previewTime; diff --git a/apps/desktop/src/routes/editor/Timeline/index.tsx b/apps/desktop/src/routes/editor/Timeline/index.tsx index 4f849a0f83..a0043fd6f3 100644 --- a/apps/desktop/src/routes/editor/Timeline/index.tsx +++ b/apps/desktop/src/routes/editor/Timeline/index.tsx @@ -2,20 +2,72 @@ import { createElementBounds } from "@solid-primitives/bounds"; import { createEventListener } from "@solid-primitives/event-listener"; import { platform } from "@tauri-apps/plugin-os"; import { cx } from "cva"; -import { batch, createRoot, createSignal, For, onMount, Show } from "solid-js"; +import { + batch, + createRoot, + createSignal, + For, + type JSX, + onMount, + Show, +} from "solid-js"; import { produce } from "solid-js/store"; import "./styles.css"; +import Tooltip from "~/components/Tooltip"; import { commands } from "~/utils/tauri"; -import { FPS, OUTPUT_SIZE, useEditorContext } from "../context"; +import { + FPS, + OUTPUT_SIZE, + type TimelineTrackType, + useEditorContext, +} from "../context"; import { formatTime } from "../utils"; import { ClipTrack } from "./ClipTrack"; import { TimelineContextProvider, useTimelineContext } from "./context"; import { type SceneSegmentDragState, SceneTrack } from "./SceneTrack"; +import { TrackIcon, TrackManager } from "./TrackManager"; import { type ZoomSegmentDragState, ZoomTrack } from "./ZoomTrack"; const TIMELINE_PADDING = 16; +const TRACK_GUTTER = 64; +const TIMELINE_HEADER_HEIGHT = 32; +const TRACK_MANAGER_BUTTON_SIZE = 36; + +const trackIcons: Record = { + clip: , + zoom: , + scene: , +}; + +type TrackDefinition = { + type: TimelineTrackType; + label: string; + icon: JSX.Element; + locked: boolean; +}; + +const trackDefinitions: TrackDefinition[] = [ + { + type: "clip", + label: "Clip", + icon: trackIcons.clip, + locked: true, + }, + { + type: "zoom", + label: "Zoom", + icon: trackIcons.zoom, + locked: true, + }, + { + type: "scene", + label: "Scene", + icon: trackIcons.scene, + locked: false, + }, +]; export function Timeline() { const { @@ -38,6 +90,21 @@ export function Timeline() { const secsPerPixel = () => transform().zoom / (timelineBounds.width ?? 1); + const trackState = () => editorState.timeline.tracks; + const sceneAvailable = () => meta().hasCamera && !project.camera.hide; + const trackOptions = () => + trackDefinitions.map((definition) => ({ + ...definition, + active: definition.type === "scene" ? trackState().scene : true, + available: definition.type === "scene" ? sceneAvailable() : true, + })); + const sceneTrackVisible = () => trackState().scene && sceneAvailable(); + + function handleToggleTrack(type: TimelineTrackType, next: boolean) { + if (type !== "scene") return; + setEditorState("timeline", "tracks", "scene", next); + } + onMount(() => { if (!project.timeline) { const resume = projectHistory.pause(); @@ -197,9 +264,10 @@ export function Timeline() { onMouseMove={(e) => { const { left } = timelineBounds; if (editorState.playing) return; + if (left == null) return; setEditorState( "previewTime", - transform().position + secsPerPixel() * (e.clientX - left!), + transform().position + secsPerPixel() * (e.clientX - left), ); }} onMouseEnter={() => setEditorState("timeline", "hoveredTrack", null)} @@ -243,19 +311,35 @@ export function Timeline() { } }} > - +
+
+ +
+
+ + + +
+
{(time) => (
- - { - zoomSegmentDragState = v; - }} - handleUpdatePlayhead={handleUpdatePlayhead} - /> - - + + + + { - sceneSegmentDragState = v; + zoomSegmentDragState = v; }} handleUpdatePlayhead={handleUpdatePlayhead} /> + + + + { + sceneSegmentDragState = v; + }} + handleUpdatePlayhead={handleUpdatePlayhead} + /> +
); } +function TrackRow(props: { icon: JSX.Element; children: JSX.Element }) { + return ( +
+ +
+ {props.children} +
+
+ ); +} + function TimelineMarkings() { const { editorState } = useEditorContext(); const { secsPerPixel, markingResolution } = useTimelineContext(); @@ -321,7 +423,10 @@ function TimelineMarkings() { }; return ( -
+
{(second) => ( 0}> diff --git a/crates/editor/src/editor.rs b/crates/editor/src/editor.rs index e03acb64ab..8d0336dff7 100644 --- a/crates/editor/src/editor.rs +++ b/crates/editor/src/editor.rs @@ -5,10 +5,7 @@ use cap_rendering::{ DecodedSegmentFrames, FrameRenderer, ProjectRecordingsMeta, ProjectUniforms, RenderVideoConstants, RenderedFrame, RendererLayers, }; -use tokio::{ - sync::{mpsc, oneshot}, - task::JoinHandle, -}; +use tokio::sync::{mpsc, oneshot}; #[allow(clippy::large_enum_variant)] pub enum RendererMessage { @@ -17,6 +14,7 @@ pub enum RendererMessage { uniforms: ProjectUniforms, finished: oneshot::Sender<()>, cursor: Arc, + frame_number: u32, }, Stop { finished: oneshot::Sender<()>, @@ -73,48 +71,100 @@ impl Renderer { } async fn run(mut self) { - let mut frame_task: Option> = None; - let mut frame_renderer = FrameRenderer::new(&self.render_constants); let mut layers = RendererLayers::new(&self.render_constants.device, &self.render_constants.queue); + struct PendingFrame { + segment_frames: DecodedSegmentFrames, + uniforms: ProjectUniforms, + finished: oneshot::Sender<()>, + cursor: Arc, + frame_number: u32, + } + + let mut pending_frame: Option = None; + let mut last_rendered_frame: Option = None; + loop { - while let Some(msg) = self.rx.recv().await { + let frame_to_render = if let Some(pending) = pending_frame.take() { + Some(pending) + } else { + match self.rx.recv().await { + Some(RendererMessage::RenderFrame { + segment_frames, + uniforms, + finished, + cursor, + frame_number, + }) => Some(PendingFrame { + segment_frames, + uniforms, + finished, + cursor, + frame_number, + }), + Some(RendererMessage::Stop { finished }) => { + let _ = finished.send(()); + return; + } + None => return, + } + }; + + let Some(mut current) = frame_to_render else { + continue; + }; + + while let Ok(msg) = self.rx.try_recv() { match msg { RendererMessage::RenderFrame { segment_frames, uniforms, finished, cursor, + frame_number, } => { - if let Some(task) = frame_task.as_ref() { - if task.is_finished() { - frame_task = None - } else { - continue; - } - } - - let frame = frame_renderer - .render(segment_frames, uniforms, &cursor, &mut layers) - .await - .unwrap(); - - (self.frame_cb)(frame); - - let _ = finished.send(()); + let _ = current.finished.send(()); + current = PendingFrame { + segment_frames, + uniforms, + finished, + cursor, + frame_number, + }; } RendererMessage::Stop { finished } => { - if let Some(task) = frame_task.take() { - task.abort(); - } + let _ = current.finished.send(()); let _ = finished.send(()); return; } } } + + if let Some(last) = last_rendered_frame { + if current.frame_number == last { + let _ = current.finished.send(()); + continue; + } + } + + let frame = frame_renderer + .render( + current.segment_frames, + current.uniforms, + ¤t.cursor, + &mut layers, + ) + .await + .unwrap(); + + last_rendered_frame = Some(current.frame_number); + + (self.frame_cb)(frame); + + let _ = current.finished.send(()); } } } @@ -129,6 +179,7 @@ impl RendererHandle { segment_frames: DecodedSegmentFrames, uniforms: ProjectUniforms, cursor: Arc, + frame_number: u32, ) { let (finished_tx, finished_rx) = oneshot::channel(); @@ -137,6 +188,7 @@ impl RendererHandle { uniforms, finished: finished_tx, cursor, + frame_number, }) .await; diff --git a/crates/editor/src/editor_instance.rs b/crates/editor/src/editor_instance.rs index e5824dcf71..721bb46e52 100644 --- a/crates/editor/src/editor_instance.rs +++ b/crates/editor/src/editor_instance.rs @@ -7,7 +7,6 @@ use cap_rendering::{ ProjectRecordingsMeta, ProjectUniforms, RecordingSegmentDecoders, RenderVideoConstants, RenderedFrame, SegmentVideoPaths, get_duration, }; -use std::ops::Deref; use std::{path::PathBuf, sync::Arc}; use tokio::sync::{Mutex, watch}; use tracing::{trace, warn}; @@ -198,45 +197,75 @@ impl EditorInstance { mut preview_rx: watch::Receiver)>>, ) -> tokio::task::JoinHandle<()> { tokio::spawn(async move { + let mut last_rendered_frame: Option = None; + loop { preview_rx.changed().await.unwrap(); - let Some((frame_number, fps, resolution_base)) = *preview_rx.borrow().deref() - else { - continue; - }; - - let project = self.project_config.1.borrow().clone(); - - let Some((segment_time, segment)) = - project.get_segment_time(frame_number as f64 / fps as f64) - else { - continue; - }; - - let segment_medias = &self.segment_medias[segment.recording_clip as usize]; - let clip_config = project - .clips - .iter() - .find(|v| v.index == segment.recording_clip); - let clip_offsets = clip_config.map(|v| v.offsets).unwrap_or_default(); - - if let Some(segment_frames) = segment_medias - .decoders - .get_frames(segment_time as f32, !project.camera.hide, clip_offsets) - .await - { - let uniforms = ProjectUniforms::new( - &self.render_constants, - &project, - frame_number, - fps, - resolution_base, - &segment_medias.cursor, - &segment_frames, + + loop { + let Some((frame_number, fps, resolution_base)) = + *preview_rx.borrow_and_update() + else { + break; + }; + + if last_rendered_frame == Some(frame_number) { + break; + } + + let project = self.project_config.1.borrow().clone(); + + let Some((segment_time, segment)) = + project.get_segment_time(frame_number as f64 / fps as f64) + else { + break; + }; + + let segment_medias = &self.segment_medias[segment.recording_clip as usize]; + let clip_config = project + .clips + .iter() + .find(|v| v.index == segment.recording_clip); + let clip_offsets = clip_config.map(|v| v.offsets).unwrap_or_default(); + + let get_frames_future = segment_medias.decoders.get_frames( + segment_time as f32, + !project.camera.hide, + clip_offsets, ); - self.renderer - .render_frame(segment_frames, uniforms, segment_medias.cursor.clone()) - .await; + + tokio::select! { + biased; + + _ = preview_rx.changed() => { + continue; + } + + segment_frames_opt = get_frames_future => { + if preview_rx.has_changed().unwrap_or(false) { + continue; + } + + if let Some(segment_frames) = segment_frames_opt { + let uniforms = ProjectUniforms::new( + &self.render_constants, + &project, + frame_number, + fps, + resolution_base, + &segment_medias.cursor, + &segment_frames, + ); + self.renderer + .render_frame(segment_frames, uniforms, segment_medias.cursor.clone(), frame_number) + .await; + + last_rendered_frame = Some(frame_number); + } + } + } + + break; } } }) diff --git a/crates/editor/src/playback.rs b/crates/editor/src/playback.rs index 272a5312cd..b0ed53883b 100644 --- a/crates/editor/src/playback.rs +++ b/crates/editor/src/playback.rs @@ -146,7 +146,12 @@ impl Playback { ); self.renderer - .render_frame(segment_frames, uniforms, segment_media.cursor.clone()) + .render_frame( + segment_frames, + uniforms, + segment_media.cursor.clone(), + frame_number, + ) .await; } diff --git a/crates/rendering/src/decoder/avassetreader.rs b/crates/rendering/src/decoder/avassetreader.rs index 8bc0dac427..f4ad42eb77 100644 --- a/crates/rendering/src/decoder/avassetreader.rs +++ b/crates/rendering/src/decoder/avassetreader.rs @@ -261,13 +261,14 @@ impl AVAssetReaderDecoder { .as_ref() .map(|last| { requested_frame < last.number - // seek forward for big jumps. this threshold is arbitrary but should be derived from i-frames in future - || requested_frame - last.number > FRAME_CACHE_SIZE as u32 + || requested_frame - last.number > FRAME_CACHE_SIZE as u32 }) .unwrap_or(true) { this.reset(requested_time); frames = this.inner.frames(); + *last_sent_frame.borrow_mut() = None; + cache.clear(); } last_active_frame = Some(requested_frame); diff --git a/crates/rendering/src/decoder/ffmpeg.rs b/crates/rendering/src/decoder/ffmpeg.rs index 2776c7d799..5a25eca6fc 100644 --- a/crates/rendering/src/decoder/ffmpeg.rs +++ b/crates/rendering/src/decoder/ffmpeg.rs @@ -133,8 +133,7 @@ impl FfmpegDecoder { .as_ref() .map(|last| { requested_frame < last.number - // seek forward for big jumps. this threshold is arbitrary but should be derived from i-frames in future - || requested_frame - last.number > FRAME_CACHE_SIZE as u32 + || requested_frame - last.number > FRAME_CACHE_SIZE as u32 }) .unwrap_or(true) { @@ -142,6 +141,8 @@ impl FfmpegDecoder { let _ = this.reset(requested_time); frames = this.frames(); + *last_sent_frame.borrow_mut() = None; + cache.clear(); } last_active_frame = Some(requested_frame); From 64df88b888278aa5db3208b1979e4055be564033 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sun, 30 Nov 2025 20:53:28 +0800 Subject: [PATCH 25/46] Remove wobble animation and adjust timeline UI --- .../src/routes/editor/Timeline/ClipTrack.tsx | 4 +--- .../src/routes/editor/Timeline/SceneTrack.tsx | 4 +--- .../src/routes/editor/Timeline/TrackManager.tsx | 2 +- .../src/routes/editor/Timeline/ZoomTrack.tsx | 4 +--- apps/desktop/src/routes/editor/Timeline/index.tsx | 13 +++++++++---- .../desktop/src/routes/editor/Timeline/styles.css | 15 --------------- 6 files changed, 13 insertions(+), 29 deletions(-) diff --git a/apps/desktop/src/routes/editor/Timeline/ClipTrack.tsx b/apps/desktop/src/routes/editor/Timeline/ClipTrack.tsx index 0d1fcc13fd..258103521f 100644 --- a/apps/desktop/src/routes/editor/Timeline/ClipTrack.tsx +++ b/apps/desktop/src/routes/editor/Timeline/ClipTrack.tsx @@ -389,9 +389,7 @@ export function ClipTrack( class={cx( "border transition-colors duration-200 group hover:border-gray-12", "bg-gradient-to-r from-[#2675DB] via-[#4FA0FF] to-[#2675DB] shadow-[inset_0_5px_10px_5px_rgba(255,255,255,0.2)]", - isSelected() - ? "wobble-wrapper border-gray-12" - : "border-transparent", + isSelected() ? "border-gray-12" : "border-transparent", )} innerClass="ring-blue-9" segment={relativeSegment()} diff --git a/apps/desktop/src/routes/editor/Timeline/SceneTrack.tsx b/apps/desktop/src/routes/editor/Timeline/SceneTrack.tsx index 3d785fb14e..193ce67b35 100644 --- a/apps/desktop/src/routes/editor/Timeline/SceneTrack.tsx +++ b/apps/desktop/src/routes/editor/Timeline/SceneTrack.tsx @@ -357,9 +357,7 @@ export function SceneTrack(props: { class={cx( "border transition-colors duration-200 hover:border-gray-12 group", `bg-gradient-to-r from-[#5C1BC4] via-[#975CFA] to-[#5C1BC4] shadow-[inset_0_8px_12px_3px_rgba(255,255,255,0.2)]`, - isSelected() - ? "wobble-wrapper border-gray-12" - : "border-transparent", + isSelected() ? "border-gray-12" : "border-transparent", )} innerClass="ring-blue-5" segment={segment} diff --git a/apps/desktop/src/routes/editor/Timeline/TrackManager.tsx b/apps/desktop/src/routes/editor/Timeline/TrackManager.tsx index 7b875de5e1..353e071b88 100644 --- a/apps/desktop/src/routes/editor/Timeline/TrackManager.tsx +++ b/apps/desktop/src/routes/editor/Timeline/TrackManager.tsx @@ -56,7 +56,7 @@ export function TrackManager(props: { ref={(el) => { addButton = el; }} - class="flex h-8 w-9 items-center justify-center rounded-lg border border-gray-4/80 bg-gray-2 text-sm font-medium text-gray-12 transition-colors duration-150 hover:bg-gray-3 dark:border-gray-5/60 dark:bg-gray-4/50" + class="flex h-8 w-[3.5rem] items-center justify-center rounded-lg border border-gray-4/80 bg-gray-2 text-sm font-medium text-gray-12 transition-colors duration-150 hover:bg-gray-3 dark:border-gray-5/60 dark:bg-gray-4/50" onClick={handleOpenMenu} onMouseDown={(e) => e.stopPropagation()} > diff --git a/apps/desktop/src/routes/editor/Timeline/ZoomTrack.tsx b/apps/desktop/src/routes/editor/Timeline/ZoomTrack.tsx index 1d476bdf8b..1e34facc98 100644 --- a/apps/desktop/src/routes/editor/Timeline/ZoomTrack.tsx +++ b/apps/desktop/src/routes/editor/Timeline/ZoomTrack.tsx @@ -420,9 +420,7 @@ export function ZoomTrack(props: { class={cx( "border duration-200 hover:border-gray-12 transition-colors group", "bg-gradient-to-r from-[#292929] via-[#434343] to-[#292929] shadow-[inset_0_8px_12px_3px_rgba(255,255,255,0.2)]", - isSelected() - ? "wobble-wrapper border-gray-12" - : "border-transparent", + isSelected() ? "border-gray-12" : "border-transparent", )} innerClass="ring-red-5" segment={segment} diff --git a/apps/desktop/src/routes/editor/Timeline/index.tsx b/apps/desktop/src/routes/editor/Timeline/index.tsx index a0043fd6f3..f1cd68c97d 100644 --- a/apps/desktop/src/routes/editor/Timeline/index.tsx +++ b/apps/desktop/src/routes/editor/Timeline/index.tsx @@ -33,7 +33,7 @@ import { type ZoomSegmentDragState, ZoomTrack } from "./ZoomTrack"; const TIMELINE_PADDING = 16; const TRACK_GUTTER = 64; const TIMELINE_HEADER_HEIGHT = 32; -const TRACK_MANAGER_BUTTON_SIZE = 36; +const TRACK_MANAGER_BUTTON_SIZE = 56; const trackIcons: Record = { clip: , @@ -262,12 +262,17 @@ export function Timeline() { }); }} onMouseMove={(e) => { - const { left } = timelineBounds; + const { left, width } = timelineBounds; if (editorState.playing) return; - if (left == null) return; + if (left == null || !width || width <= 0) return; + const offsetX = e.clientX - left; + if (offsetX < 0 || offsetX > width) { + setEditorState("previewTime", null); + return; + } setEditorState( "previewTime", - transform().position + secsPerPixel() * (e.clientX - left), + transform().position + secsPerPixel() * offsetX, ); }} onMouseEnter={() => setEditorState("timeline", "hoveredTrack", null)} diff --git a/apps/desktop/src/routes/editor/Timeline/styles.css b/apps/desktop/src/routes/editor/Timeline/styles.css index fcf1312df8..12084fda39 100644 --- a/apps/desktop/src/routes/editor/Timeline/styles.css +++ b/apps/desktop/src/routes/editor/Timeline/styles.css @@ -1,18 +1,3 @@ -@keyframes wobble { - 0%, - 100% { - transform: translateY(0) translateX(var(--segment-x)); - } - 50% { - transform: translateY(-3px) translateX(var(--segment-x)); - } -} - -.wobble-wrapper { - animation: wobble 1s ease-in-out infinite; - will-change: transform; -} - .timeline-scissors-cursor { cursor: url("data:image/svg+xml,%3Csvg width='24' height='24' viewBox='0 0 20 20' fill='none' xmlns='http://www.w3.org/2000/svg' shape-rendering='geometricPrecision'%3E%3Cg transform='rotate(90 10 10)'%3E%3Cpath fill-rule='evenodd' clip-rule='evenodd' d='M5 2.5C3.15905 2.5 1.66666 3.99238 1.66666 5.83333C1.66666 7.67428 3.15905 9.16667 5 9.16667C5.85421 9.16667 6.63337 8.84536 7.22322 8.317L9.74771 10L7.22322 11.683C6.63337 11.1546 5.85421 10.8333 5 10.8333C3.15905 10.8333 1.66666 12.3257 1.66666 14.1667C1.66666 16.0076 3.15905 17.5 5 17.5C6.84095 17.5 8.33333 16.0076 8.33333 14.1667C8.33333 13.7822 8.26824 13.4129 8.14846 13.0692L18.6556 6.06446C18.1451 5.29858 17.1103 5.09162 16.3444 5.60221L11.25 8.99846L8.14846 6.93075C8.26824 6.5871 8.33333 6.21782 8.33333 5.83333C8.33333 3.99238 6.84095 2.5 5 2.5ZM3.33333 5.83333C3.33333 4.91286 4.07952 4.16667 5 4.16667C5.92047 4.16667 6.66666 4.91286 6.66666 5.83333C6.66666 6.75381 5.92047 7.5 5 7.5C4.07952 7.5 3.33333 6.75381 3.33333 5.83333ZM3.33333 14.1667C3.33333 13.2462 4.07952 12.5 5 12.5C5.92047 12.5 6.66666 13.2462 6.66666 14.1667C6.66666 15.0871 5.92047 15.8333 5 15.8333C4.07952 15.8333 3.33333 15.0871 3.33333 14.1667Z' fill='%23ffffff' stroke='%23000000' stroke-width='0.6'/%3E%3Cpath d='M16.3444 14.3978L12.0012 11.5023L13.5035 10.5008L18.6556 13.9355C18.1451 14.7014 17.1103 14.9084 16.3444 14.3978Z' fill='%23ffffff' stroke='%23000000' stroke-width='0.6'/%3E%3C/g%3E%3C/svg%3E") From 1c25535120fe9b85024af195daf9d0098a806704 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sun, 30 Nov 2025 20:54:12 +0800 Subject: [PATCH 26/46] Remove redundant frame deduplication logic --- apps/desktop/src-tauri/src/lib.rs | 8 ++++---- crates/editor/src/editor.rs | 10 ---------- crates/editor/src/editor_instance.rs | 8 -------- 3 files changed, 4 insertions(+), 22 deletions(-) diff --git a/apps/desktop/src-tauri/src/lib.rs b/apps/desktop/src-tauri/src/lib.rs index 4e776fe3b1..923db76f23 100644 --- a/apps/desktop/src-tauri/src/lib.rs +++ b/apps/desktop/src-tauri/src/lib.rs @@ -3061,13 +3061,13 @@ async fn create_editor_instance_impl( RenderFrameEvent::listen_any(&app, { let preview_tx = instance.preview_tx.clone(); move |e| { - preview_tx - .send(Some(( + preview_tx.send_modify(|v| { + *v = Some(( e.payload.frame_number, e.payload.fps, e.payload.resolution_base, - ))) - .ok(); + )); + }); } }); diff --git a/crates/editor/src/editor.rs b/crates/editor/src/editor.rs index 8d0336dff7..93c73127be 100644 --- a/crates/editor/src/editor.rs +++ b/crates/editor/src/editor.rs @@ -85,7 +85,6 @@ impl Renderer { } let mut pending_frame: Option = None; - let mut last_rendered_frame: Option = None; loop { let frame_to_render = if let Some(pending) = pending_frame.take() { @@ -143,13 +142,6 @@ impl Renderer { } } - if let Some(last) = last_rendered_frame { - if current.frame_number == last { - let _ = current.finished.send(()); - continue; - } - } - let frame = frame_renderer .render( current.segment_frames, @@ -160,8 +152,6 @@ impl Renderer { .await .unwrap(); - last_rendered_frame = Some(current.frame_number); - (self.frame_cb)(frame); let _ = current.finished.send(()); diff --git a/crates/editor/src/editor_instance.rs b/crates/editor/src/editor_instance.rs index 721bb46e52..f834a75b16 100644 --- a/crates/editor/src/editor_instance.rs +++ b/crates/editor/src/editor_instance.rs @@ -197,8 +197,6 @@ impl EditorInstance { mut preview_rx: watch::Receiver)>>, ) -> tokio::task::JoinHandle<()> { tokio::spawn(async move { - let mut last_rendered_frame: Option = None; - loop { preview_rx.changed().await.unwrap(); @@ -209,10 +207,6 @@ impl EditorInstance { break; }; - if last_rendered_frame == Some(frame_number) { - break; - } - let project = self.project_config.1.borrow().clone(); let Some((segment_time, segment)) = @@ -259,8 +253,6 @@ impl EditorInstance { self.renderer .render_frame(segment_frames, uniforms, segment_medias.cursor.clone(), frame_number) .await; - - last_rendered_frame = Some(frame_number); } } } From 02d2260293e5cf3083b17007a1a352578f62faf2 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sun, 30 Nov 2025 22:49:12 +0800 Subject: [PATCH 27/46] Add preview quality selection to editor player --- .../desktop/src/routes/editor/CaptionsTab.tsx | 15 +- apps/desktop/src/routes/editor/Editor.tsx | 17 ++- apps/desktop/src/routes/editor/Player.tsx | 129 +++++++++++++++--- apps/desktop/src/routes/editor/context.ts | 28 +++- 4 files changed, 159 insertions(+), 30 deletions(-) diff --git a/apps/desktop/src/routes/editor/CaptionsTab.tsx b/apps/desktop/src/routes/editor/CaptionsTab.tsx index ff46d77452..d6677d16b4 100644 --- a/apps/desktop/src/routes/editor/CaptionsTab.tsx +++ b/apps/desktop/src/routes/editor/CaptionsTab.tsx @@ -10,7 +10,7 @@ import toast from "solid-toast"; import { Toggle } from "~/components/Toggle"; import type { CaptionSegment, CaptionSettings } from "~/utils/tauri"; import { commands, events } from "~/utils/tauri"; -import { FPS, OUTPUT_SIZE, useEditorContext } from "./context"; +import { FPS, useEditorContext } from "./context"; import { TextInput } from "./TextInput"; import { Field, @@ -166,8 +166,13 @@ function RgbInput(props: { value: string; onChange: (value: string) => void }) { // Add scroll position preservation for the container export function CaptionsTab() { - const { project, setProject, editorInstance, editorState } = - useEditorContext(); + const { + project, + setProject, + editorInstance, + editorState, + previewResolutionBase, + } = useEditorContext(); // Scroll management let scrollContainerRef: HTMLDivElement | undefined; @@ -215,7 +220,7 @@ export function CaptionsTab() { events.renderFrameEvent.emit({ frame_number: Math.floor(editorState.playbackTime * FPS), fps: FPS, - resolution_base: OUTPUT_SIZE, + resolution_base: previewResolutionBase(), }); }); } @@ -248,7 +253,7 @@ export function CaptionsTab() { events.renderFrameEvent.emit({ frame_number: Math.floor(editorState.playbackTime * FPS), fps: FPS, - resolution_base: OUTPUT_SIZE, + resolution_base: previewResolutionBase(), }); } }; diff --git a/apps/desktop/src/routes/editor/Editor.tsx b/apps/desktop/src/routes/editor/Editor.tsx index 8b84d86db7..87a6a89a7c 100644 --- a/apps/desktop/src/routes/editor/Editor.tsx +++ b/apps/desktop/src/routes/editor/Editor.tsx @@ -35,7 +35,6 @@ import { EditorContextProvider, EditorInstanceContextProvider, FPS, - OUTPUT_SIZE, useEditorContext, useEditorInstanceContext, } from "./context"; @@ -82,7 +81,8 @@ export function Editor() { } function Inner() { - const { project, editorState, setEditorState } = useEditorContext(); + const { project, editorState, setEditorState, previewResolutionBase } = + useEditorContext(); createTauriEventListener(events.editorStateChanged, (payload) => { throttledRenderFrame.clear(); @@ -95,7 +95,7 @@ function Inner() { events.renderFrameEvent.emit({ frame_number: Math.max(Math.floor(time * FPS), 0), fps: FPS, - resolution_base: OUTPUT_SIZE, + resolution_base: previewResolutionBase(), }); } }; @@ -116,10 +116,13 @@ function Inner() { }); createEffect( - on(frameNumberToRender, (number) => { - if (editorState.playing) return; - renderFrame(number); - }), + on( + () => [frameNumberToRender(), previewResolutionBase()], + ([number]) => { + if (editorState.playing) return; + renderFrame(number); + }, + ), ); createEffect( diff --git a/apps/desktop/src/routes/editor/Player.tsx b/apps/desktop/src/routes/editor/Player.tsx index 9b0245d12d..ac38bdc061 100644 --- a/apps/desktop/src/routes/editor/Player.tsx +++ b/apps/desktop/src/routes/editor/Player.tsx @@ -1,3 +1,4 @@ +import { Select as KSelect } from "@kobalte/core/select"; import { ToggleButton as KToggleButton } from "@kobalte/core/toggle-button"; import { createElementBounds } from "@solid-primitives/bounds"; import { cx } from "cva"; @@ -9,11 +10,18 @@ import { commands } from "~/utils/tauri"; import AspectRatioSelect from "./AspectRatioSelect"; import { FPS, - OUTPUT_SIZE, + type PreviewQuality, serializeProjectConfiguration, useEditorContext, } from "./context"; -import { EditorButton, Slider } from "./ui"; +import { + EditorButton, + MenuItem, + MenuItemList, + PopperContent, + Slider, + topLeftAnimateClasses, +} from "./ui"; import { useEditorShortcuts } from "./useEditorShortcuts"; import { formatTime } from "./utils"; @@ -27,8 +35,16 @@ export function Player() { setEditorState, zoomOutLimit, setProject, + previewResolutionBase, + previewQuality, + setPreviewQuality, } = useEditorContext(); + const previewOptions = [ + { label: "Full", value: "full" as PreviewQuality }, + { label: "Half", value: "half" as PreviewQuality }, + ]; + // Load captions on mount onMount(async () => { if (editorInstance && editorInstance.path) { @@ -95,10 +111,6 @@ export function Player() { } }); - const [canvasContainerRef, setCanvasContainerRef] = - createSignal(); - const containerBounds = createElementBounds(canvasContainerRef); - const isAtEnd = () => { const total = totalDuration(); return total > 0 && total - editorState.playbackTime <= 0.1; @@ -123,6 +135,31 @@ export function Player() { setEditorState("playing", false); }; + const handlePreviewQualityChange = async (quality: PreviewQuality) => { + if (quality === previewQuality()) return; + + const wasPlaying = editorState.playing; + const currentFrame = Math.max( + Math.floor(editorState.playbackTime * FPS), + 0, + ); + + setPreviewQuality(quality); + + if (!wasPlaying) return; + + try { + await commands.stopPlayback(); + setEditorState("playing", false); + await commands.seekTo(currentFrame); + await commands.startPlayback(FPS, previewResolutionBase()); + setEditorState("playing", true); + } catch (error) { + console.error("Failed to update preview quality:", error); + setEditorState("playing", false); + } + }; + createEffect(() => { if (isAtEnd() && editorState.playing) { commands.stopPlayback(); @@ -136,7 +173,7 @@ export function Player() { await commands.stopPlayback(); setEditorState("playbackTime", 0); await commands.seekTo(0); - await commands.startPlayback(FPS, OUTPUT_SIZE); + await commands.startPlayback(FPS, previewResolutionBase()); setEditorState("playing", true); } else if (editorState.playing) { await commands.stopPlayback(); @@ -144,7 +181,7 @@ export function Player() { } else { // Ensure we seek to the current playback time before starting playback await commands.seekTo(Math.floor(editorState.playbackTime * FPS)); - await commands.startPlayback(FPS, OUTPUT_SIZE); + await commands.startPlayback(FPS, previewResolutionBase()); setEditorState("playing", true); } if (editorState.playing) setEditorState("previewTime", null); @@ -209,15 +246,73 @@ export function Player() { return (
-
- - } - > - Crop - +
+
+ + } + > + Crop + +
+
+ Preview + + options={previewOptions} + optionValue="value" + optionTextValue="label" + value={previewOptions.find( + (option) => option.value === previewQuality(), + )} + onChange={(next) => { + if (next) handlePreviewQualityChange(next.value); + }} + disallowEmptySelection + itemComponent={(props) => ( + + as={KSelect.Item} + item={props.item} + > + + {props.item.rawValue.label} + + + + + + )} + > + + + class="flex-1 text-left truncate" + placeholder="Select preview quality" + > + {(state) => + state.selectedOption()?.label ?? "Select preview quality" + } + + + + + + + + as={KSelect.Content} + class={cx(topLeftAnimateClasses, "w-44")} + > + + Select preview quality + + + as={KSelect.Listbox} + class="max-h-40" + /> + + + +
diff --git a/apps/desktop/src/routes/editor/context.ts b/apps/desktop/src/routes/editor/context.ts index 3aae121e9c..15c17e0919 100644 --- a/apps/desktop/src/routes/editor/context.ts +++ b/apps/desktop/src/routes/editor/context.ts @@ -50,6 +50,23 @@ export const OUTPUT_SIZE = { y: 1080, }; +export type PreviewQuality = "half" | "full"; + +export const DEFAULT_PREVIEW_QUALITY: PreviewQuality = "full"; + +const previewQualityScale: Record = { + full: 1, + half: 0.5, +}; + +export const getPreviewResolution = (quality: PreviewQuality): XY => { + const scale = previewQualityScale[quality]; + const width = (Math.max(2, Math.round(OUTPUT_SIZE.x * scale)) + 1) & ~1; + const height = (Math.max(2, Math.round(OUTPUT_SIZE.y * scale)) + 1) & ~1; + + return { x: width, y: height }; +}; + export type TimelineTrackType = "clip" | "zoom" | "scene"; export const MAX_ZOOM_IN = 3; @@ -358,6 +375,12 @@ export const [EditorContextProvider, useEditorContext] = createContextProvider( ), ); + const [previewQuality, setPreviewQuality] = createSignal( + DEFAULT_PREVIEW_QUALITY, + ); + + const previewResolutionBase = () => getPreviewResolution(previewQuality()); + const [dialog, setDialog] = createSignal({ open: false, }); @@ -516,6 +539,9 @@ export const [EditorContextProvider, useEditorContext] = createContextProvider( setExportState, micWaveforms, systemAudioWaveforms, + previewQuality, + setPreviewQuality, + previewResolutionBase, }; }, // biome-ignore lint/style/noNonNullAssertion: it's ok @@ -584,7 +610,7 @@ export const [EditorInstanceContextProvider, useEditorInstanceContext] = events.renderFrameEvent.emit({ frame_number: Math.floor(0), fps: FPS, - resolution_base: OUTPUT_SIZE, + resolution_base: getPreviewResolution(DEFAULT_PREVIEW_QUALITY), }); } }); From 8b79cf776e896c0092ea5b855dd96aa4bbb45f19 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sun, 30 Nov 2025 22:49:24 +0800 Subject: [PATCH 28/46] Add dynamic fade mask to timeline scroll area --- .../routes/editor/Timeline/TrackManager.tsx | 2 +- .../src/routes/editor/Timeline/index.tsx | 61 +++++++++++++++---- 2 files changed, 51 insertions(+), 12 deletions(-) diff --git a/apps/desktop/src/routes/editor/Timeline/TrackManager.tsx b/apps/desktop/src/routes/editor/Timeline/TrackManager.tsx index 353e071b88..798f721785 100644 --- a/apps/desktop/src/routes/editor/Timeline/TrackManager.tsx +++ b/apps/desktop/src/routes/editor/Timeline/TrackManager.tsx @@ -56,7 +56,7 @@ export function TrackManager(props: { ref={(el) => { addButton = el; }} - class="flex h-8 w-[3.5rem] items-center justify-center rounded-lg border border-gray-4/80 bg-gray-2 text-sm font-medium text-gray-12 transition-colors duration-150 hover:bg-gray-3 dark:border-gray-5/60 dark:bg-gray-4/50" + class="flex h-[3.25rem] w-[3.5rem] items-center justify-center rounded-xl border border-gray-4/70 bg-gray-2/60 text-sm font-medium text-gray-12 transition-colors duration-150 hover:bg-gray-3 dark:border-gray-4/60 dark:bg-gray-3/40 shadow-[0_4px_16px_-12px_rgba(0,0,0,0.8)]" onClick={handleOpenMenu} onMouseDown={(e) => e.stopPropagation()} > diff --git a/apps/desktop/src/routes/editor/Timeline/index.tsx b/apps/desktop/src/routes/editor/Timeline/index.tsx index f1cd68c97d..b53bc8df55 100644 --- a/apps/desktop/src/routes/editor/Timeline/index.tsx +++ b/apps/desktop/src/routes/editor/Timeline/index.tsx @@ -17,12 +17,7 @@ import "./styles.css"; import Tooltip from "~/components/Tooltip"; import { commands } from "~/utils/tauri"; -import { - FPS, - OUTPUT_SIZE, - type TimelineTrackType, - useEditorContext, -} from "../context"; +import { FPS, type TimelineTrackType, useEditorContext } from "../context"; import { formatTime } from "../utils"; import { ClipTrack } from "./ClipTrack"; import { TimelineContextProvider, useTimelineContext } from "./context"; @@ -80,6 +75,7 @@ export function Timeline() { editorState, projectActions, meta, + previewResolutionBase, } = useEditorContext(); const duration = () => editorInstance.recordingDuration; @@ -187,7 +183,7 @@ export function Timeline() { return; } - await commands.startPlayback(FPS, OUTPUT_SIZE); + await commands.startPlayback(FPS, previewResolutionBase()); setEditorState("playing", true); } catch (err) { console.error("Failed to seek during playback:", err); @@ -236,6 +232,50 @@ export function Timeline() { const split = () => editorState.timeline.interactMode === "split"; + const maskImage = () => { + const pos = transform().position; + const zoom = transform().zoom; + const total = totalDuration(); + const secPerPx = secsPerPixel(); + + const FADE_WIDTH = 64; + const FADE_RAMP_PX = 50; // Pixels of scrolling over which the fade animates in + const LEFT_OFFSET = TIMELINE_PADDING + TRACK_GUTTER; + const RIGHT_PADDING = TIMELINE_PADDING; + + // Calculate alpha for left fade (0 = fully faded, 1 = no fade) + // When pos is 0, we are at start -> no fade needed -> strength 0 + // When pos increases, we want fade to appear -> strength 1 + const scrollLeftPx = pos / secPerPx; + const leftFadeStrength = Math.min(1, scrollLeftPx / FADE_RAMP_PX); + + // Calculate alpha for right fade + // When at end, right scroll is 0 -> no fade -> strength 0 + const scrollRightPx = (total - (pos + zoom)) / secPerPx; + const rightFadeStrength = Math.min(1, scrollRightPx / FADE_RAMP_PX); + + const leftStartColor = `rgba(0, 0, 0, ${1 - leftFadeStrength})`; + const rightEndColor = `rgba(0, 0, 0, ${1 - rightFadeStrength})`; + + // Left stops: + // 0px to LEFT_OFFSET: Always black (icons area) + // LEFT_OFFSET: Starts fading. If strength is 0 (start), it's black. If strength is 1, it's transparent. + // LEFT_OFFSET + FADE_WIDTH: Always black (content fully visible) + const leftStops = `black 0px, black ${LEFT_OFFSET}px, ${leftStartColor} ${LEFT_OFFSET}px, black ${ + LEFT_OFFSET + FADE_WIDTH + }px`; + + // Right stops: + // calc(100% - (RIGHT_PADDING + FADE_WIDTH)): Always black (content fully visible) + // calc(100% - RIGHT_PADDING): Ends fading. If strength is 0 (end), it's black. If strength is 1, it's transparent. + // 100%: Transparent + const rightStops = `black calc(100% - ${ + RIGHT_PADDING + FADE_WIDTH + }px), ${rightEndColor} calc(100% - ${RIGHT_PADDING}px), transparent 100%`; + + return `linear-gradient(to right, ${leftStops}, ${rightStops})`; + }; + return ( { createRoot((dispose) => { @@ -320,10 +362,7 @@ export function Timeline() {
-
+
Date: Mon, 1 Dec 2025 00:44:31 +0800 Subject: [PATCH 29/46] Add mask track with editable mask segments to editor --- apps/desktop/src-tauri/src/recording.rs | 1 + .../src/routes/editor/ConfigSidebar.tsx | 232 +++++++++++ apps/desktop/src/routes/editor/Editor.tsx | 98 ++++- .../desktop/src/routes/editor/MaskOverlay.tsx | 230 +++++++++++ apps/desktop/src/routes/editor/Player.tsx | 26 +- .../src/routes/editor/Timeline/MaskTrack.tsx | 375 ++++++++++++++++++ .../src/routes/editor/Timeline/Track.tsx | 8 +- .../routes/editor/Timeline/TrackManager.tsx | 5 +- .../src/routes/editor/Timeline/index.tsx | 212 +++++++--- apps/desktop/src/routes/editor/context.ts | 91 ++++- apps/desktop/src/routes/editor/masks.ts | 117 ++++++ apps/desktop/src/utils/tauri.ts | 7 +- crates/project/src/configuration.rs | 67 ++++ crates/rendering/src/layers/mask.rs | 235 +++++++++++ crates/rendering/src/layers/mod.rs | 2 + crates/rendering/src/lib.rs | 73 +++- crates/rendering/src/mask.rs | 126 ++++++ crates/rendering/src/shaders/mask.wgsl | 71 ++++ packages/ui-solid/src/auto-imports.d.ts | 2 + 19 files changed, 1892 insertions(+), 86 deletions(-) create mode 100644 apps/desktop/src/routes/editor/MaskOverlay.tsx create mode 100644 apps/desktop/src/routes/editor/Timeline/MaskTrack.tsx create mode 100644 apps/desktop/src/routes/editor/masks.ts create mode 100644 crates/rendering/src/layers/mask.rs create mode 100644 crates/rendering/src/mask.rs create mode 100644 crates/rendering/src/shaders/mask.wgsl diff --git a/apps/desktop/src-tauri/src/recording.rs b/apps/desktop/src-tauri/src/recording.rs index 0fc5b6cb73..af9b13617d 100644 --- a/apps/desktop/src-tauri/src/recording.rs +++ b/apps/desktop/src-tauri/src/recording.rs @@ -1705,6 +1705,7 @@ fn project_config_from_recording( segments: timeline_segments, zoom_segments, scene_segments: Vec::new(), + mask_segments: Vec::new(), }); config diff --git a/apps/desktop/src/routes/editor/ConfigSidebar.tsx b/apps/desktop/src/routes/editor/ConfigSidebar.tsx index be5deaee77..e2b1d74cfe 100644 --- a/apps/desktop/src/routes/editor/ConfigSidebar.tsx +++ b/apps/desktop/src/routes/editor/ConfigSidebar.tsx @@ -52,13 +52,20 @@ import { type TimelineSegment, type ZoomSegment, } from "~/utils/tauri"; +import IconLucideBoxSelect from "~icons/lucide/box-select"; +import IconLucideGauge from "~icons/lucide/gauge"; +import IconLucideGrid from "~icons/lucide/grid"; import IconLucideMonitor from "~icons/lucide/monitor"; +import IconLucideMoon from "~icons/lucide/moon"; +import IconLucideMove from "~icons/lucide/move"; import IconLucideRabbit from "~icons/lucide/rabbit"; +import IconLucideScan from "~icons/lucide/scan"; import IconLucideSparkles from "~icons/lucide/sparkles"; import IconLucideTimer from "~icons/lucide/timer"; import IconLucideWind from "~icons/lucide/wind"; import { CaptionsTab } from "./CaptionsTab"; import { type CornerRoundingType, useEditorContext } from "./context"; +import { evaluateMask, type MaskKind, type MaskSegment } from "./masks"; import { DEFAULT_GRADIENT_FROM, DEFAULT_GRADIENT_TO, @@ -783,6 +790,73 @@ export function ConfigSidebar() { {(selection) => ( + { + const maskSelection = selection(); + if (maskSelection.type !== "mask") return; + + const segments = maskSelection.indices + .map((index) => ({ + index, + segment: project.timeline?.maskSegments?.[index], + })) + .filter( + (item): item is { index: number; segment: MaskSegment } => + item.segment !== undefined, + ); + + if (segments.length === 0) { + setEditorState("timeline", "selection", null); + return; + } + return { selection: maskSelection, segments }; + })()} + > + {(value) => ( +
+
+
+ + setEditorState("timeline", "selection", null) + } + leftIcon={} + > + Done + + + {value().segments.length} mask{" "} + {value().segments.length === 1 + ? "segment" + : "segments"}{" "} + selected + +
+ + projectActions.deleteMaskSegments( + value().segments.map((s) => s.index), + ) + } + leftIcon={} + > + Delete + +
+ + {(item) => ( +
+ +
+ )} + +
+ )} + { const zoomSelection = selection(); @@ -2306,6 +2380,164 @@ function CornerStyleSelect(props: { ); } +function MaskSegmentConfig(props: { + segmentIndex: number; + segment: MaskSegment; +}) { + const { setProject, editorState } = useEditorContext(); + + const updateSegment = (fn: (segment: MaskSegment) => void) => { + setProject( + "timeline", + "maskSegments", + produce((segments) => { + const target = segments?.[props.segmentIndex]; + if (!target) return; + target.keyframes ??= { position: [], size: [], intensity: [] }; + fn(target); + }), + ); + }; + + createEffect(() => { + const keyframes = props.segment.keyframes; + if ( + !keyframes || + (keyframes.position.length === 0 && + keyframes.size.length === 0 && + keyframes.intensity.length === 0) + ) + return; + updateSegment((segment) => { + segment.keyframes = { position: [], size: [], intensity: [] }; + }); + }); + + const currentAbsoluteTime = () => + editorState.previewTime ?? editorState.playbackTime ?? props.segment.start; + const maskState = () => evaluateMask(props.segment, currentAbsoluteTime()); + + const clearKeyframes = (segment: MaskSegment) => { + segment.keyframes.position = []; + segment.keyframes.size = []; + segment.keyframes.intensity = []; + }; + + const setPosition = (value: { x: number; y: number }) => + updateSegment((segment) => { + segment.center = value; + clearKeyframes(segment); + }); + + const setSize = (value: { x: number; y: number }) => + updateSegment((segment) => { + segment.size = value; + clearKeyframes(segment); + }); + + const setIntensity = (value: number) => + updateSegment((segment) => { + segment.opacity = value; + clearKeyframes(segment); + }); + + return ( +
+ } + > +
+ + updateSegment((segment) => { + segment.maskType = value as MaskKind; + if (segment.maskType === "highlight") { + segment.feather = 0; + segment.opacity = 1; + } else { + segment.feather = 0.1; + } + }) + } + > + {[ + { value: "sensitive", label: "Sensitive" }, + { value: "highlight", label: "Highlight" }, + ].map((option) => ( + + + + + {option.label} + + + ))} + +
+ Enabled + + updateSegment((segment) => { + segment.enabled = value; + }) + } + /> +
+
+ + + }> + setIntensity(v)} + minValue={0} + maxValue={1} + step={0.01} + formatTooltip="%" + /> + + + + }> + + updateSegment((segment) => { + segment.pixelation = v; + }) + } + minValue={1} + maxValue={80} + step={1} + /> + + + + }> + + updateSegment((segment) => { + segment.darkness = v; + }) + } + minValue={0} + maxValue={1} + step={0.01} + /> + + +
+ ); +} + function ZoomSegmentPreview(props: { segmentIndex: number; segment: ZoomSegment; diff --git a/apps/desktop/src/routes/editor/Editor.tsx b/apps/desktop/src/routes/editor/Editor.tsx index 87a6a89a7c..b489c33ed4 100644 --- a/apps/desktop/src/routes/editor/Editor.tsx +++ b/apps/desktop/src/routes/editor/Editor.tsx @@ -1,5 +1,6 @@ import { Button } from "@cap/ui-solid"; import { NumberField } from "@kobalte/core/number-field"; +import { createElementBounds } from "@solid-primitives/bounds"; import { trackDeep } from "@solid-primitives/deep"; import { debounce, throttle } from "@solid-primitives/scheduled"; import { makePersisted } from "@solid-primitives/storage"; @@ -44,6 +45,12 @@ import { Player } from "./Player"; import { Timeline } from "./Timeline"; import { Dialog, DialogContent, EditorButton, Input, Subfield } from "./ui"; +const DEFAULT_TIMELINE_HEIGHT = 260; +const MIN_PLAYER_CONTENT_HEIGHT = 320; +const MIN_TIMELINE_HEIGHT = 240; +const RESIZE_HANDLE_HEIGHT = 16; +const MIN_PLAYER_HEIGHT = MIN_PLAYER_CONTENT_HEIGHT + RESIZE_HANDLE_HEIGHT; + export function Editor() { return ( @@ -84,6 +91,58 @@ function Inner() { const { project, editorState, setEditorState, previewResolutionBase } = useEditorContext(); + const [layoutRef, setLayoutRef] = createSignal(); + const layoutBounds = createElementBounds(layoutRef); + const [storedTimelineHeight, setStoredTimelineHeight] = makePersisted( + createSignal(DEFAULT_TIMELINE_HEIGHT), + { name: "editorTimelineHeight" }, + ); + const [isResizingTimeline, setIsResizingTimeline] = createSignal(false); + + const clampTimelineHeight = (value: number) => { + const available = layoutBounds.height ?? 0; + const maxHeight = + available > 0 + ? Math.max(MIN_TIMELINE_HEIGHT, available - MIN_PLAYER_HEIGHT) + : Number.POSITIVE_INFINITY; + const upperBound = Number.isFinite(maxHeight) + ? maxHeight + : Math.max(value, MIN_TIMELINE_HEIGHT); + return Math.min(Math.max(value, MIN_TIMELINE_HEIGHT), upperBound); + }; + + const timelineHeight = createMemo(() => + Math.round(clampTimelineHeight(storedTimelineHeight())), + ); + + const handleTimelineResizeStart = (event: MouseEvent) => { + if (event.button !== 0) return; + event.preventDefault(); + const startY = event.clientY; + const startHeight = timelineHeight(); + setIsResizingTimeline(true); + + const handleMove = (moveEvent: MouseEvent) => { + const delta = moveEvent.clientY - startY; + setStoredTimelineHeight(clampTimelineHeight(startHeight - delta)); + }; + + const handleUp = () => { + setIsResizingTimeline(false); + window.removeEventListener("mousemove", handleMove); + window.removeEventListener("mouseup", handleUp); + }; + + window.addEventListener("mousemove", handleMove); + window.addEventListener("mouseup", handleUp); + }; + + createEffect(() => { + const available = layoutBounds.height; + if (!available) return; + setStoredTimelineHeight((height) => clampTimelineHeight(height)); + }); + createTauriEventListener(events.editorStateChanged, (payload) => { throttledRenderFrame.clear(); trailingRenderFrame.clear(); @@ -139,12 +198,45 @@ function Inner() { class="flex overflow-y-hidden flex-col flex-1 gap-2 pb-4 w-full min-h-0 leading-5 animate-in fade-in" data-tauri-drag-region > -
-
+
+
+ +
+
+ +
-
diff --git a/apps/desktop/src/routes/editor/MaskOverlay.tsx b/apps/desktop/src/routes/editor/MaskOverlay.tsx new file mode 100644 index 0000000000..c203c02289 --- /dev/null +++ b/apps/desktop/src/routes/editor/MaskOverlay.tsx @@ -0,0 +1,230 @@ +import { createEventListenerMap } from "@solid-primitives/event-listener"; +import { cx } from "cva"; +import { createMemo, createRoot, Show } from "solid-js"; +import { produce } from "solid-js/store"; + +import { useEditorContext } from "./context"; +import { evaluateMask, type MaskSegment } from "./masks"; + +type MaskOverlayProps = { + size: { width: number; height: number }; +}; + +export function MaskOverlay(props: MaskOverlayProps) { + const { project, setProject, editorState, projectHistory } = + useEditorContext(); + + const selectedMask = createMemo(() => { + const selection = editorState.timeline.selection; + if (!selection || selection.type !== "mask") return; + const index = selection.indices[0]; + const segment = project.timeline?.maskSegments?.[index]; + if (!segment) return; + return { index, segment }; + }); + + const currentAbsoluteTime = () => + editorState.previewTime ?? editorState.playbackTime ?? 0; + + const maskState = createMemo(() => { + const selected = selectedMask(); + if (!selected) return; + return evaluateMask(selected.segment, currentAbsoluteTime()); + }); + + const updateSegment = (fn: (segment: MaskSegment) => void) => { + const index = selectedMask()?.index; + if (index === undefined) return; + setProject( + "timeline", + "maskSegments", + index, + produce((segment) => { + segment.keyframes ??= { position: [], size: [], intensity: [] }; + segment.keyframes.position = []; + segment.keyframes.size = []; + fn(segment); + }), + ); + }; + + function createMouseDownDrag( + setup: () => T, + update: ( + e: MouseEvent, + value: T, + initialMouse: { x: number; y: number }, + ) => void, + ) { + return (downEvent: MouseEvent) => { + downEvent.preventDefault(); + downEvent.stopPropagation(); + + const initial = setup(); + const initialMouse = { x: downEvent.clientX, y: downEvent.clientY }; + const resumeHistory = projectHistory.pause(); + + function handleUpdate(event: MouseEvent) { + update(event, initial, initialMouse); + } + + function finish() { + resumeHistory(); + dispose(); + } + + const dispose = createRoot((dispose) => { + createEventListenerMap(window, { + mousemove: handleUpdate, + mouseup: () => { + finish(); + }, + }); + return dispose; + }); + }; + } + + return ( + + {() => { + const state = () => maskState()!; + const rect = () => { + const width = state().size.x * props.size.width; + const height = state().size.y * props.size.height; + const left = state().position.x * props.size.width - width / 2; + const top = state().position.y * props.size.height - height / 2; + return { width, height, left, top }; + }; + + const onMove = createMouseDownDrag( + () => ({ + startPos: { ...state().position }, + }), + (e, { startPos }, initialMouse) => { + const dx = (e.clientX - initialMouse.x) / props.size.width; + const dy = (e.clientY - initialMouse.y) / props.size.height; + + updateSegment((s) => { + s.center.x = Math.max(0, Math.min(1, startPos.x + dx)); + s.center.y = Math.max(0, Math.min(1, startPos.y + dy)); + }); + }, + ); + + const createResizeHandler = (dirX: -1 | 0 | 1, dirY: -1 | 0 | 1) => { + return createMouseDownDrag( + () => ({ + startPos: { ...state().position }, + startSize: { ...state().size }, + }), + (e, { startPos, startSize }, initialMouse) => { + const dx = (e.clientX - initialMouse.x) / props.size.width; + const dy = (e.clientY - initialMouse.y) / props.size.height; + + updateSegment((s) => { + if (dirX !== 0) { + const newWidth = Math.max( + 0.01, + startSize.x + dx * dirX, // if dirX is -1 (left), dx needs to be inverted for width? No. If I move mouse left (dx negative), width should increase. So dx * dirX -> (-ve * -1) = +ve. Correct. + ); + // If we clamp width, we need to adjust center calc? + // Simple version: + s.size.x = newWidth; + s.center.x = + startPos.x + (dx * dirX) / 2 + dx * (dirX === -1 ? 0 : 0); + // Wait, my logic before: + // Right (dirX=1): Width = W + dx. Center = C + dx/2. + // Left (dirX=-1): Width = W - dx. Center = C + dx/2. + // So Center is always C + dx/2 regardless of direction? + // Let's re-verify. + // Right Handle: move right (+dx). W grows (+dx). Center moves right (+dx/2). Correct. + // Left Handle: move left (-dx). W grows (-dx i.e. +ve). Center moves left (-dx/2). Correct. + // So yes, Center += dx/2. Width += dx * dirX. + + s.center.x = startPos.x + dx / 2; + } + + if (dirY !== 0) { + const newHeight = Math.max(0.01, startSize.y + dy * dirY); + s.size.y = newHeight; + s.center.y = startPos.y + dy / 2; + } + }); + }, + ); + }; + + return ( +
+
+ {/* Border/Highlight */} +
+ + {/* Handles */} + {/* Corners */} + + + + + + {/* Sides */} + + + + +
+
+ ); + }} + + ); +} + +function ResizeHandle(props: { + class?: string; + onMouseDown: (e: MouseEvent) => void; +}) { + return ( +
+ ); +} diff --git a/apps/desktop/src/routes/editor/Player.tsx b/apps/desktop/src/routes/editor/Player.tsx index ac38bdc061..b8f6d95f32 100644 --- a/apps/desktop/src/routes/editor/Player.tsx +++ b/apps/desktop/src/routes/editor/Player.tsx @@ -14,6 +14,7 @@ import { serializeProjectConfiguration, useEditorContext, } from "./context"; +import { MaskOverlay } from "./MaskOverlay"; import { EditorButton, MenuItem, @@ -518,18 +519,27 @@ function PreviewCanvas() { return (
- + > + + +
); }} diff --git a/apps/desktop/src/routes/editor/Timeline/MaskTrack.tsx b/apps/desktop/src/routes/editor/Timeline/MaskTrack.tsx new file mode 100644 index 0000000000..303b43b30c --- /dev/null +++ b/apps/desktop/src/routes/editor/Timeline/MaskTrack.tsx @@ -0,0 +1,375 @@ +import { createEventListenerMap } from "@solid-primitives/event-listener"; +import { cx } from "cva"; +import { createMemo, createRoot, For } from "solid-js"; +import { produce } from "solid-js/store"; + +import { useEditorContext } from "../context"; +import { defaultMaskSegment } from "../masks"; +import { useTimelineContext } from "./context"; +import { SegmentContent, SegmentHandle, SegmentRoot, TrackRoot } from "./Track"; + +export type MaskSegmentDragState = + | { type: "idle" } + | { type: "movePending" } + | { type: "moving" }; + +const MIN_SEGMENT_SECS = 1; +const MIN_SEGMENT_PIXELS = 80; + +export function MaskTrack(props: { + onDragStateChanged: (v: MaskSegmentDragState) => void; + handleUpdatePlayhead: (e: MouseEvent) => void; +}) { + const { + project, + setProject, + editorState, + setEditorState, + totalDuration, + projectHistory, + projectActions, + } = useEditorContext(); + const { secsPerPixel, timelineBounds } = useTimelineContext(); + + const minDuration = () => + Math.max(MIN_SEGMENT_SECS, secsPerPixel() * MIN_SEGMENT_PIXELS); + + const maskSegments = () => project.timeline?.maskSegments ?? []; + + const neighborBounds = (index: number) => { + const segments = maskSegments(); + return { + prevEnd: segments[index - 1]?.end ?? 0, + nextStart: segments[index + 1]?.start ?? totalDuration(), + }; + }; + + const findPlacement = (time: number, length: number) => { + const gaps: Array<{ start: number; end: number }> = []; + const sorted = maskSegments() + .slice() + .sort((a, b) => a.start - b.start); + + let cursor = 0; + for (const segment of sorted) { + if (segment.start - cursor >= length) { + gaps.push({ start: cursor, end: segment.start }); + } + cursor = Math.max(cursor, segment.end); + } + + if (totalDuration() - cursor >= length) { + gaps.push({ start: cursor, end: totalDuration() }); + } + + if (gaps.length === 0) return null; + + const maxStart = Math.max(totalDuration() - length, 0); + const desiredStart = Math.min(Math.max(time - length / 2, 0), maxStart); + + const containingGap = + gaps.find( + (gap) => desiredStart >= gap.start && desiredStart + length <= gap.end, + ) ?? + gaps.find((gap) => gap.start >= desiredStart) ?? + gaps[gaps.length - 1]; + + const start = Math.min( + Math.max(desiredStart, containingGap.start), + containingGap.end - length, + ); + + return { start, end: start + length }; + }; + + const addSegmentAt = (time: number) => { + const length = Math.min(minDuration(), totalDuration()); + if (length <= 0) return; + + const placement = findPlacement(time, length); + if (!placement) return; + + setProject( + "timeline", + "maskSegments", + produce((segments) => { + segments ??= []; + segments.push(defaultMaskSegment(placement.start, placement.end)); + segments.sort((a, b) => a.start - b.start); + }), + ); + }; + + const handleBackgroundMouseDown = (e: MouseEvent) => { + if (e.button !== 0) return; + if ((e.target as HTMLElement).closest("[data-mask-segment]")) return; + const timelineTime = + editorState.previewTime ?? + editorState.playbackTime ?? + secsPerPixel() * (e.clientX - (timelineBounds.left ?? 0)); + addSegmentAt(timelineTime); + }; + + function createMouseDownDrag( + segmentIndex: () => number, + setup: () => T, + update: (e: MouseEvent, value: T, initialMouseX: number) => void, + ) { + return (downEvent: MouseEvent) => { + if (editorState.timeline.interactMode !== "seek") return; + downEvent.stopPropagation(); + const initial = setup(); + let moved = false; + let initialMouseX: number | null = null; + + const resumeHistory = projectHistory.pause(); + props.onDragStateChanged({ type: "movePending" }); + + function finish(e: MouseEvent) { + resumeHistory(); + if (!moved) { + e.stopPropagation(); + const currentSelection = editorState.timeline.selection; + const index = segmentIndex(); + const isMultiSelect = e.ctrlKey || e.metaKey; + const isRangeSelect = e.shiftKey; + + if (isRangeSelect && currentSelection?.type === "mask") { + const existingIndices = currentSelection.indices; + const lastIndex = existingIndices[existingIndices.length - 1]; + const start = Math.min(lastIndex, index); + const end = Math.max(lastIndex, index); + const rangeIndices: number[] = []; + for (let idx = start; idx <= end; idx++) rangeIndices.push(idx); + setEditorState("timeline", "selection", { + type: "mask", + indices: rangeIndices, + }); + } else if (isMultiSelect) { + if (currentSelection?.type === "mask") { + const base = currentSelection.indices; + const exists = base.includes(index); + const next = exists + ? base.filter((i) => i !== index) + : [...base, index]; + setEditorState( + "timeline", + "selection", + next.length > 0 + ? { + type: "mask", + indices: next, + } + : null, + ); + } else { + setEditorState("timeline", "selection", { + type: "mask", + indices: [index], + }); + } + } else { + setEditorState("timeline", "selection", { + type: "mask", + indices: [index], + }); + } + props.handleUpdatePlayhead(e); + } + props.onDragStateChanged({ type: "idle" }); + } + + function handleUpdate(event: MouseEvent) { + if (Math.abs(event.clientX - downEvent.clientX) > 2) { + if (!moved) { + moved = true; + initialMouseX = event.clientX; + props.onDragStateChanged({ type: "moving" }); + } + } + + if (initialMouseX === null) return; + update(event, initial, initialMouseX); + } + + createRoot((dispose) => { + createEventListenerMap(window, { + mousemove: (e) => handleUpdate(e), + mouseup: (e) => { + handleUpdate(e); + finish(e); + dispose(); + }, + }); + }); + }; + } + + return ( + setEditorState("timeline", "hoveredTrack", "mask")} + onMouseLeave={() => setEditorState("timeline", "hoveredTrack", null)} + onMouseDown={handleBackgroundMouseDown} + > + +
Click to add a mask
+
+ (Combine sensitive blur or highlight masks) +
+
+ } + > + {(segment, i) => { + const isSelected = createMemo(() => { + const selection = editorState.timeline.selection; + if (!selection || selection.type !== "mask") return false; + return selection.indices.includes(i()); + }); + + const contentLabel = () => + segment.maskType === "sensitive" ? "Sensitive" : "Highlight"; + + const segmentWidth = () => segment.end - segment.start; + + return ( + { + e.stopPropagation(); + if (editorState.timeline.interactMode === "split") { + const rect = e.currentTarget.getBoundingClientRect(); + const fraction = (e.clientX - rect.left) / rect.width; + const splitTime = fraction * segmentWidth(); + projectActions.splitMaskSegment(i(), splitTime); + } + }} + > + { + const bounds = neighborBounds(i()); + const start = segment.start; + const minValue = bounds.prevEnd; + const maxValue = Math.max( + minValue, + Math.min( + segment.end - minDuration(), + bounds.nextStart - minDuration(), + ), + ); + return { start, minValue, maxValue }; + }, + (e, value, initialMouseX) => { + const delta = (e.clientX - initialMouseX) * secsPerPixel(); + const next = Math.max( + value.minValue, + Math.min(value.maxValue, value.start + delta), + ); + setProject("timeline", "maskSegments", i(), "start", next); + setProject( + "timeline", + "maskSegments", + produce((items) => { + items.sort((a, b) => a.start - b.start); + }), + ); + }, + )} + /> + { + const original = { ...segment }; + const bounds = neighborBounds(i()); + const minDelta = bounds.prevEnd - original.start; + const maxDelta = bounds.nextStart - original.end; + return { + original, + minDelta, + maxDelta, + }; + }, + (e, value, initialMouseX) => { + const delta = (e.clientX - initialMouseX) * secsPerPixel(); + const lowerBound = Math.min(value.minDelta, value.maxDelta); + const upperBound = Math.max(value.minDelta, value.maxDelta); + const clampedDelta = Math.min( + upperBound, + Math.max(lowerBound, delta), + ); + setProject("timeline", "maskSegments", i(), { + ...value.original, + start: value.original.start + clampedDelta, + end: value.original.end + clampedDelta, + }); + setProject( + "timeline", + "maskSegments", + produce((items) => { + items.sort((a, b) => a.start - b.start); + }), + ); + }, + )} + > + {(() => { + return ( +
+ Mask +
+ {contentLabel()} +
+
+ ); + })()} + + { + const bounds = neighborBounds(i()); + const end = segment.end; + const minValue = segment.start + minDuration(); + const maxValue = Math.max(minValue, bounds.nextStart); + return { end, minValue, maxValue }; + }, + (e, value, initialMouseX) => { + const delta = (e.clientX - initialMouseX) * secsPerPixel(); + const next = Math.max( + value.minValue, + Math.min(value.maxValue, value.end + delta), + ); + setProject("timeline", "maskSegments", i(), "end", next); + setProject( + "timeline", + "maskSegments", + produce((items) => { + items.sort((a, b) => a.start - b.start); + }), + ); + }, + )} + /> + + ); + }} + + + ); +} diff --git a/apps/desktop/src/routes/editor/Timeline/Track.tsx b/apps/desktop/src/routes/editor/Timeline/Track.tsx index c5581ecc94..85c954f327 100644 --- a/apps/desktop/src/routes/editor/Timeline/Track.tsx +++ b/apps/desktop/src/routes/editor/Timeline/Track.tsx @@ -11,13 +11,19 @@ import { export function TrackRoot(props: ComponentProps<"div">) { const [ref, setRef] = createSignal(); + const height = "var(--track-height, 3.25rem)"; + const style = + typeof props.style === "string" + ? `${props.style};height:${height}` + : { height, ...(props.style ?? {}) }; return (
{props.children}
diff --git a/apps/desktop/src/routes/editor/Timeline/TrackManager.tsx b/apps/desktop/src/routes/editor/Timeline/TrackManager.tsx index 798f721785..e9bcd6f529 100644 --- a/apps/desktop/src/routes/editor/Timeline/TrackManager.tsx +++ b/apps/desktop/src/routes/editor/Timeline/TrackManager.tsx @@ -10,6 +10,7 @@ type TrackManagerOption = { icon: JSX.Element; active: boolean; available: boolean; + locked: boolean; }; export function TrackManager(props: { @@ -22,7 +23,7 @@ export function TrackManager(props: { try { const items = await Promise.all( props.options.map((option) => { - if (option.type === "scene") { + if (!option.locked) { return CheckMenuItem.new({ text: option.label, checked: option.active, @@ -33,7 +34,7 @@ export function TrackManager(props: { return CheckMenuItem.new({ text: option.label, - checked: true, + checked: option.active, enabled: false, }); }), diff --git a/apps/desktop/src/routes/editor/Timeline/index.tsx b/apps/desktop/src/routes/editor/Timeline/index.tsx index b53bc8df55..b209729d4a 100644 --- a/apps/desktop/src/routes/editor/Timeline/index.tsx +++ b/apps/desktop/src/routes/editor/Timeline/index.tsx @@ -21,6 +21,7 @@ import { FPS, type TimelineTrackType, useEditorContext } from "../context"; import { formatTime } from "../utils"; import { ClipTrack } from "./ClipTrack"; import { TimelineContextProvider, useTimelineContext } from "./context"; +import { type MaskSegmentDragState, MaskTrack } from "./MaskTrack"; import { type SceneSegmentDragState, SceneTrack } from "./SceneTrack"; import { TrackIcon, TrackManager } from "./TrackManager"; import { type ZoomSegmentDragState, ZoomTrack } from "./ZoomTrack"; @@ -28,10 +29,10 @@ import { type ZoomSegmentDragState, ZoomTrack } from "./ZoomTrack"; const TIMELINE_PADDING = 16; const TRACK_GUTTER = 64; const TIMELINE_HEADER_HEIGHT = 32; -const TRACK_MANAGER_BUTTON_SIZE = 56; const trackIcons: Record = { clip: , + mask: , zoom: , scene: , }; @@ -50,6 +51,12 @@ const trackDefinitions: TrackDefinition[] = [ icon: trackIcons.clip, locked: true, }, + { + type: "mask", + label: "Mask", + icon: trackIcons.mask, + locked: false, + }, { type: "zoom", label: "Zoom", @@ -91,14 +98,31 @@ export function Timeline() { const trackOptions = () => trackDefinitions.map((definition) => ({ ...definition, - active: definition.type === "scene" ? trackState().scene : true, + active: + definition.type === "scene" + ? trackState().scene + : definition.type === "mask" + ? trackState().mask + : true, available: definition.type === "scene" ? sceneAvailable() : true, })); const sceneTrackVisible = () => trackState().scene && sceneAvailable(); + const visibleTrackCount = () => + 2 + (trackState().mask ? 1 : 0) + (sceneTrackVisible() ? 1 : 0); + const trackHeight = () => (visibleTrackCount() > 2 ? "3rem" : "3.25rem"); function handleToggleTrack(type: TimelineTrackType, next: boolean) { - if (type !== "scene") return; - setEditorState("timeline", "tracks", "scene", next); + if (type === "scene") { + setEditorState("timeline", "tracks", "scene", next); + return; + } + + if (type === "mask") { + setEditorState("timeline", "tracks", "mask", next); + if (!next && editorState.timeline.selection?.type === "mask") { + setEditorState("timeline", "selection", null); + } + } } onMount(() => { @@ -112,6 +136,9 @@ export function Timeline() { end: duration(), }, ], + zoomSegments: [], + sceneSegments: [], + maskSegments: [], }); resume(); } @@ -148,19 +175,46 @@ export function Timeline() { }, ], zoomSegments: [], + sceneSegments: [], + maskSegments: [], + }; + project.timeline.sceneSegments ??= []; + project.timeline.maskSegments ??= []; + project.timeline.zoomSegments ??= []; + }), + ); + } + + if (!project.timeline?.maskSegments) { + setProject( + produce((project) => { + project.timeline ??= { + segments: [ + { + start: 0, + end: duration(), + timescale: 1, + }, + ], + zoomSegments: [], + sceneSegments: [], + maskSegments: [], }; + project.timeline.maskSegments ??= []; }), ); } let zoomSegmentDragState = { type: "idle" } as ZoomSegmentDragState; let sceneSegmentDragState = { type: "idle" } as SceneSegmentDragState; + let maskSegmentDragState = { type: "idle" } as MaskSegmentDragState; async function handleUpdatePlayhead(e: MouseEvent) { const { left } = timelineBounds; if ( zoomSegmentDragState.type !== "moving" && - sceneSegmentDragState.type !== "moving" + sceneSegmentDragState.type !== "moving" && + maskSegmentDragState.type !== "moving" ) { // Guard against missing bounds and clamp computed time to [0, totalDuration()] if (left == null) return; @@ -203,6 +257,8 @@ export function Timeline() { if (selection.type === "zoom") { projectActions.deleteZoomSegments(selection.indices); + } else if (selection.type === "mask") { + projectActions.deleteMaskSegments(selection.indices); } else if (selection.type === "clip") { // Delete all selected clips in reverse order [...selection.indices] @@ -283,12 +339,13 @@ export function Timeline() { timelineBounds={timelineBounds} >
{ createRoot((dispose) => { @@ -371,71 +428,94 @@ export function Timeline() {
- - {(time) => ( -
+
+ + {(time) => (
+ style={{ + left: `${TIMELINE_PADDING + TRACK_GUTTER}px`, + transform: `translateX(${ + (time() - transform().position) / secsPerPixel() - 0.5 + }px)`, + top: "0px", + }} + > +
+
+ )} + +
+
+
+
{ + if (!e.ctrlKey && Math.abs(e.deltaY) > Math.abs(e.deltaX)) { + e.stopPropagation(); + } + }} + > +
+ + + + + + { + maskSegmentDragState = v; + }} + handleUpdatePlayhead={handleUpdatePlayhead} + /> + + + + { + zoomSegmentDragState = v; + }} + handleUpdatePlayhead={handleUpdatePlayhead} + /> + + + + { + sceneSegmentDragState = v; + }} + handleUpdatePlayhead={handleUpdatePlayhead} + /> + +
- )} - -
-
+
- - - - - { - zoomSegmentDragState = v; - }} - handleUpdatePlayhead={handleUpdatePlayhead} - /> - - - - { - sceneSegmentDragState = v; - }} - handleUpdatePlayhead={handleUpdatePlayhead} - /> - -
); diff --git a/apps/desktop/src/routes/editor/context.ts b/apps/desktop/src/routes/editor/context.ts index 15c17e0919..21a60f6e5d 100644 --- a/apps/desktop/src/routes/editor/context.ts +++ b/apps/desktop/src/routes/editor/context.ts @@ -28,10 +28,15 @@ import { type MultipleSegments, type ProjectConfiguration, type RecordingMeta, + type SceneSegment, type SerializedEditorInstance, type SingleSegment, + type TimelineConfiguration, + type TimelineSegment, type XY, + type ZoomSegment, } from "~/utils/tauri"; +import type { MaskSegment } from "./masks"; import { createProgressBar } from "./utils"; export type CurrentDialog = @@ -67,7 +72,7 @@ export const getPreviewResolution = (quality: PreviewQuality): XY => { return { x: width, y: height }; }; -export type TimelineTrackType = "clip" | "zoom" | "scene"; +export type TimelineTrackType = "clip" | "zoom" | "scene" | "mask"; export const MAX_ZOOM_IN = 3; const PROJECT_SAVE_DEBOUNCE_MS = 250; @@ -85,12 +90,21 @@ export type CornerRoundingType = "rounded" | "squircle"; type WithCornerStyle = T & { roundingType: CornerRoundingType }; +type EditorTimelineConfiguration = Omit< + TimelineConfiguration, + "sceneSegments" +> & { + sceneSegments?: SceneSegment[]; + maskSegments: MaskSegment[]; +}; + export type EditorProjectConfiguration = Omit< ProjectConfiguration, - "background" | "camera" + "background" | "camera" | "timeline" > & { background: WithCornerStyle; camera: WithCornerStyle; + timeline?: EditorTimelineConfiguration | null; }; function withCornerDefaults< @@ -109,8 +123,22 @@ function withCornerDefaults< export function normalizeProject( config: ProjectConfiguration, ): EditorProjectConfiguration { + const timeline = config.timeline + ? { + ...config.timeline, + sceneSegments: config.timeline.sceneSegments ?? [], + maskSegments: + ( + config.timeline as TimelineConfiguration & { + maskSegments?: MaskSegment[]; + } + ).maskSegments ?? [], + } + : undefined; + return { ...config, + timeline, background: withCornerDefaults(config.background), camera: withCornerDefaults(config.camera), }; @@ -124,8 +152,16 @@ export function serializeProjectConfiguration( background; const { roundingType: cameraRoundingType, ...cameraRest } = camera; + const timeline = project.timeline + ? { + ...project.timeline, + maskSegments: project.timeline.maskSegments ?? [], + } + : project.timeline; + return { ...rest, + timeline: timeline as unknown as ProjectConfiguration["timeline"], background: { ...backgroundRest, roundingType: backgroundRoundingType, @@ -242,6 +278,45 @@ export const [EditorContextProvider, useEditorContext] = createContextProvider( setEditorState("timeline", "selection", null); }); }, + splitMaskSegment: (index: number, time: number) => { + setProject( + "timeline", + "maskSegments", + produce((segments) => { + const segment = segments?.[index]; + if (!segment) return; + + const duration = segment.end - segment.start; + const remaining = duration - time; + if (time < 1 || remaining < 1) return; + + segments.splice(index + 1, 0, { + ...segment, + start: segment.start + time, + end: segment.end, + }); + segments[index].end = segment.start + time; + }), + ); + }, + deleteMaskSegments: (segmentIndices: number[]) => { + batch(() => { + setProject( + "timeline", + "maskSegments", + produce((segments) => { + if (!segments) return; + const sorted = [...new Set(segmentIndices)] + .filter( + (i) => Number.isInteger(i) && i >= 0 && i < segments.length, + ) + .sort((a, b) => b - a); + for (const i of sorted) segments.splice(i, 1); + }), + ); + setEditorState("timeline", "selection", null); + }); + }, splitSceneSegment: (index: number, time: number) => { setProject( "timeline", @@ -309,6 +384,11 @@ export const [EditorContextProvider, useEditorContext] = createContextProvider( zoomSegment.end += diff(zoomSegment.end); } + for (const maskSegment of timeline.maskSegments) { + maskSegment.start += diff(maskSegment.start); + maskSegment.end += diff(maskSegment.end); + } + segment.timescale = timescale; }), ); @@ -455,6 +535,9 @@ export const [EditorContextProvider, useEditorContext] = createContextProvider( }; } + const initialMaskTrackEnabled = + (project.timeline?.maskSegments?.length ?? 0) > 0; + const [editorState, setEditorState] = createStore({ previewTime: null as number | null, playbackTime: 0, @@ -465,7 +548,8 @@ export const [EditorContextProvider, useEditorContext] = createContextProvider( | null | { type: "zoom"; indices: number[] } | { type: "clip"; indices: number[] } - | { type: "scene"; indices: number[] }, + | { type: "scene"; indices: number[] } + | { type: "mask"; indices: number[] }, transform: { // visible seconds zoom: zoomOutLimit(), @@ -506,6 +590,7 @@ export const [EditorContextProvider, useEditorContext] = createContextProvider( clip: true, zoom: true, scene: true, + mask: initialMaskTrackEnabled, }, hoveredTrack: null as null | TimelineTrackType, }, diff --git a/apps/desktop/src/routes/editor/masks.ts b/apps/desktop/src/routes/editor/masks.ts new file mode 100644 index 0000000000..8b7935151b --- /dev/null +++ b/apps/desktop/src/routes/editor/masks.ts @@ -0,0 +1,117 @@ +import type { XY } from "~/utils/tauri"; + +export type MaskKind = "sensitive" | "highlight"; + +export type MaskScalarKeyframe = { + time: number; + value: number; +}; + +export type MaskVectorKeyframe = { + time: number; + x: number; + y: number; +}; + +export type MaskKeyframes = { + position: MaskVectorKeyframe[]; + size: MaskVectorKeyframe[]; + intensity: MaskScalarKeyframe[]; +}; + +export type MaskSegment = { + start: number; + end: number; + enabled: boolean; + maskType: MaskKind; + center: XY; + size: XY; + feather: number; + opacity: number; + pixelation: number; + darkness: number; + keyframes: MaskKeyframes; +}; + +export type MaskState = { + position: XY; + size: XY; + intensity: number; +}; + +export const defaultMaskSegment = ( + start: number, + end: number, +): MaskSegment => ({ + start, + end, + enabled: true, + maskType: "sensitive", + center: { x: 0.5, y: 0.5 }, + size: { x: 0.35, y: 0.35 }, + feather: 0.1, + opacity: 1, + pixelation: 18, + darkness: 0.5, + keyframes: { position: [], size: [], intensity: [] }, +}); + +export const evaluateMask = ( + segment: MaskSegment, + _time?: number, +): MaskState => { + const position = { + x: Math.min(Math.max(segment.center.x, 0), 1), + y: Math.min(Math.max(segment.center.y, 0), 1), + }; + const size = { + x: Math.min(Math.max(segment.size.x, 0.01), 2), + y: Math.min(Math.max(segment.size.y, 0.01), 2), + }; + const intensity = Math.min(Math.max(segment.opacity, 0), 1); + + return { position, size, intensity }; +}; + +const sortByTime = (items: T[]) => + [...items].sort((a, b) => a.time - b.time); + +const timeMatch = (a: number, b: number) => Math.abs(a - b) < 1e-3; + +export const upsertVectorKeyframe = ( + keyframes: MaskVectorKeyframe[], + time: number, + value: XY, +) => { + const existingIndex = keyframes.findIndex((k) => timeMatch(k.time, time)); + if (existingIndex >= 0) { + const next = [...keyframes]; + next[existingIndex] = { + ...next[existingIndex], + time, + x: value.x, + y: value.y, + }; + return sortByTime(next); + } + return sortByTime([...keyframes, { time, x: value.x, y: value.y }]); +}; + +export const upsertScalarKeyframe = ( + keyframes: MaskScalarKeyframe[], + time: number, + value: number, +) => { + const existingIndex = keyframes.findIndex((k) => timeMatch(k.time, time)); + if (existingIndex >= 0) { + const next = [...keyframes]; + next[existingIndex] = { ...next[existingIndex], time, value }; + return sortByTime(next); + } + return sortByTime([...keyframes, { time, value }]); +}; + +export const removeKeyframeAt = ( + items: T[], + time: number, +) => items.filter((k) => !timeMatch(k.time, time)); diff --git a/apps/desktop/src/utils/tauri.ts b/apps/desktop/src/utils/tauri.ts index d47dea60b1..8b076dc2a7 100644 --- a/apps/desktop/src/utils/tauri.ts +++ b/apps/desktop/src/utils/tauri.ts @@ -441,7 +441,12 @@ export type LogicalBounds = { position: LogicalPosition; size: LogicalSize } export type LogicalPosition = { x: number; y: number } export type LogicalSize = { width: number; height: number } export type MainWindowRecordingStartBehaviour = "close" | "minimise" +export type MaskKeyframes = { position?: MaskVectorKeyframe[]; size?: MaskVectorKeyframe[]; intensity?: MaskScalarKeyframe[] } +export type MaskKind = "sensitive" | "highlight" +export type MaskScalarKeyframe = { time: number; value: number } +export type MaskSegment = { start: number; end: number; enabled?: boolean; maskType: MaskKind; center: XY; size: XY; feather?: number; opacity?: number; pixelation?: number; darkness?: number; keyframes?: MaskKeyframes } export type MaskType = "blur" | "pixelate" +export type MaskVectorKeyframe = { time: number; x: number; y: number } export type ModelIDType = string export type Mp4ExportSettings = { fps: number; resolution_base: XY; compression: ExportCompression } export type MultipleSegment = { display: VideoMeta; camera?: VideoMeta | null; mic?: AudioMeta | null; system_audio?: AudioMeta | null; cursor?: string | null } @@ -498,7 +503,7 @@ export type StereoMode = "stereo" | "monoL" | "monoR" export type StudioRecordingMeta = { segment: SingleSegment } | { inner: MultipleSegments } export type StudioRecordingStatus = { status: "InProgress" } | { status: "Failed"; error: string } | { status: "Complete" } export type TargetUnderCursor = { display_id: DisplayId | null; window: WindowUnderCursor | null } -export type TimelineConfiguration = { segments: TimelineSegment[]; zoomSegments: ZoomSegment[]; sceneSegments?: SceneSegment[] } +export type TimelineConfiguration = { segments: TimelineSegment[]; zoomSegments: ZoomSegment[]; sceneSegments?: SceneSegment[]; maskSegments?: MaskSegment[] } export type TimelineSegment = { recordingSegment?: number; timescale: number; start: number; end: number } export type UploadMeta = { state: "MultipartUpload"; video_id: string; file_path: string; pre_created_video: VideoUploadInfo; recording_dir: string } | { state: "SinglePartUpload"; video_id: string; recording_dir: string; file_path: string; screenshot_path: string } | { state: "Failed"; error: string } | { state: "Complete" } export type UploadMode = { Initial: { pre_created_video: VideoUploadInfo | null } } | "Reupload" diff --git a/crates/project/src/configuration.rs b/crates/project/src/configuration.rs index 9920f689c5..76ee3cfc23 100644 --- a/crates/project/src/configuration.rs +++ b/crates/project/src/configuration.rs @@ -548,6 +548,71 @@ pub enum ZoomMode { Manual { x: f32, y: f32 }, } +#[derive(Type, Serialize, Deserialize, Clone, Copy, Debug)] +#[serde(rename_all = "camelCase")] +pub enum MaskKind { + Sensitive, + Highlight, +} + +#[derive(Type, Serialize, Deserialize, Clone, Debug, Default)] +#[serde(rename_all = "camelCase")] +pub struct MaskScalarKeyframe { + pub time: f64, + pub value: f64, +} + +#[derive(Type, Serialize, Deserialize, Clone, Debug, Default)] +#[serde(rename_all = "camelCase")] +pub struct MaskVectorKeyframe { + pub time: f64, + pub x: f64, + pub y: f64, +} + +#[derive(Type, Serialize, Deserialize, Clone, Debug, Default)] +#[serde(rename_all = "camelCase")] +pub struct MaskKeyframes { + #[serde(default)] + pub position: Vec, + #[serde(default)] + pub size: Vec, + #[serde(default)] + pub intensity: Vec, +} + +#[derive(Type, Serialize, Deserialize, Clone, Debug)] +#[serde(rename_all = "camelCase")] +pub struct MaskSegment { + pub start: f64, + pub end: f64, + #[serde(default = "MaskSegment::default_enabled")] + pub enabled: bool, + pub mask_type: MaskKind, + pub center: XY, + pub size: XY, + #[serde(default)] + pub feather: f64, + #[serde(default = "MaskSegment::default_opacity")] + pub opacity: f64, + #[serde(default)] + pub pixelation: f64, + #[serde(default)] + pub darkness: f64, + #[serde(default)] + pub keyframes: MaskKeyframes, +} + +impl MaskSegment { + fn default_enabled() -> bool { + true + } + + fn default_opacity() -> f64 { + 1.0 + } +} + #[derive(Type, Serialize, Deserialize, Clone, Copy, Debug, Default)] #[serde(rename_all = "camelCase")] pub enum SceneMode { @@ -573,6 +638,8 @@ pub struct TimelineConfiguration { pub zoom_segments: Vec, #[serde(default)] pub scene_segments: Vec, + #[serde(default)] + pub mask_segments: Vec, } impl TimelineConfiguration { diff --git a/crates/rendering/src/layers/mask.rs b/crates/rendering/src/layers/mask.rs new file mode 100644 index 0000000000..daf62fac40 --- /dev/null +++ b/crates/rendering/src/layers/mask.rs @@ -0,0 +1,235 @@ +use bytemuck::{Pod, Zeroable}; +use wgpu::util::DeviceExt; + +use crate::{PreparedMask, RenderSession}; + +pub struct MaskLayer { + sampler: wgpu::Sampler, + uniforms_buffer: wgpu::Buffer, + pipeline: MaskPipeline, +} + +impl MaskLayer { + pub fn new(device: &wgpu::Device) -> Self { + Self { + sampler: device.create_sampler(&wgpu::SamplerDescriptor { + address_mode_u: wgpu::AddressMode::ClampToEdge, + address_mode_v: wgpu::AddressMode::ClampToEdge, + address_mode_w: wgpu::AddressMode::ClampToEdge, + mag_filter: wgpu::FilterMode::Nearest, + min_filter: wgpu::FilterMode::Nearest, + mipmap_filter: wgpu::FilterMode::Nearest, + ..Default::default() + }), + uniforms_buffer: device.create_buffer_init(&wgpu::util::BufferInitDescriptor { + label: Some("Mask Uniform Buffer"), + contents: bytemuck::cast_slice(&[MaskUniforms::default()]), + usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST, + }), + pipeline: MaskPipeline::new(device), + } + } + + pub fn render( + &self, + device: &wgpu::Device, + queue: &wgpu::Queue, + session: &mut RenderSession, + encoder: &mut wgpu::CommandEncoder, + mask: &PreparedMask, + ) { + let uniforms = MaskUniforms::from_mask(mask); + queue.write_buffer(&self.uniforms_buffer, 0, bytemuck::cast_slice(&[uniforms])); + + let bind_group = self.pipeline.bind_group( + device, + &self.uniforms_buffer, + session.current_texture_view(), + &self.sampler, + ); + + let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("Mask Pass"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: session.other_texture_view(), + resolve_target: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Clear(wgpu::Color::TRANSPARENT), + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: None, + timestamp_writes: None, + occlusion_query_set: None, + }); + + pass.set_pipeline(&self.pipeline.render_pipeline); + pass.set_bind_group(0, &bind_group, &[]); + pass.draw(0..3, 0..1); + + drop(pass); + session.swap_textures(); + } +} + +#[repr(C)] +#[derive(Debug, Clone, Copy, Pod, Zeroable, PartialEq)] +struct MaskUniforms { + rect_center: [f32; 2], + rect_size: [f32; 2], + feather: f32, + opacity: f32, + pixel_size: f32, + darkness: f32, + mode: u32, + padding0: u32, + output_size: [f32; 2], + padding1: [f32; 2], +} + +impl Default for MaskUniforms { + fn default() -> Self { + Self::zeroed() + } +} + +impl MaskUniforms { + fn from_mask(mask: &PreparedMask) -> Self { + Self { + rect_center: [mask.center.x, mask.center.y], + rect_size: [mask.size.x, mask.size.y], + feather: mask.feather, + opacity: mask.opacity, + pixel_size: mask.pixel_size, + darkness: mask.darkness, + mode: mask.mode_value(), + padding0: 0, + output_size: [mask.output_size.x as f32, mask.output_size.y as f32], + padding1: [0.0; 2], + } + } +} + +pub struct MaskPipeline { + bind_group_layout: wgpu::BindGroupLayout, + render_pipeline: wgpu::RenderPipeline, +} + +impl MaskPipeline { + pub fn new(device: &wgpu::Device) -> Self { + let bind_group_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("Mask Bind Group Layout"), + entries: &[ + wgpu::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu::ShaderStages::VERTEX_FRAGMENT, + ty: wgpu::BindingType::Buffer { + ty: wgpu::BufferBindingType::Uniform, + has_dynamic_offset: false, + min_binding_size: None, + }, + count: None, + }, + wgpu::BindGroupLayoutEntry { + binding: 1, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Texture { + sample_type: wgpu::TextureSampleType::Float { filterable: true }, + view_dimension: wgpu::TextureViewDimension::D2, + multisampled: false, + }, + count: None, + }, + wgpu::BindGroupLayoutEntry { + binding: 2, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering), + count: None, + }, + ], + }); + + let shader = device.create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some("Mask Shader"), + source: wgpu::ShaderSource::Wgsl(include_str!("../shaders/mask.wgsl").into()), + }); + + let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some("Mask Pipeline Layout"), + bind_group_layouts: &[&bind_group_layout], + push_constant_ranges: &[], + }); + + let render_pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some("Mask Pipeline"), + layout: Some(&pipeline_layout), + vertex: wgpu::VertexState { + module: &shader, + entry_point: Some("vs_main"), + buffers: &[], + compilation_options: wgpu::PipelineCompilationOptions { + constants: &[], + zero_initialize_workgroup_memory: false, + }, + }, + fragment: Some(wgpu::FragmentState { + module: &shader, + entry_point: Some("fs_main"), + targets: &[Some(wgpu::ColorTargetState { + format: wgpu::TextureFormat::Rgba8UnormSrgb, + blend: Some(wgpu::BlendState::REPLACE), + write_mask: wgpu::ColorWrites::ALL, + })], + compilation_options: wgpu::PipelineCompilationOptions { + constants: &[], + zero_initialize_workgroup_memory: false, + }, + }), + primitive: wgpu::PrimitiveState { + topology: wgpu::PrimitiveTopology::TriangleList, + strip_index_format: None, + front_face: wgpu::FrontFace::Ccw, + cull_mode: Some(wgpu::Face::Back), + polygon_mode: wgpu::PolygonMode::Fill, + unclipped_depth: false, + conservative: false, + }, + depth_stencil: None, + multisample: wgpu::MultisampleState::default(), + multiview: None, + cache: None, + }); + + Self { + bind_group_layout, + render_pipeline, + } + } + + pub fn bind_group( + &self, + device: &wgpu::Device, + uniform_buffer: &wgpu::Buffer, + texture_view: &wgpu::TextureView, + sampler: &wgpu::Sampler, + ) -> wgpu::BindGroup { + device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("Mask Bind Group"), + layout: &self.bind_group_layout, + entries: &[ + wgpu::BindGroupEntry { + binding: 0, + resource: uniform_buffer.as_entire_binding(), + }, + wgpu::BindGroupEntry { + binding: 1, + resource: wgpu::BindingResource::TextureView(texture_view), + }, + wgpu::BindGroupEntry { + binding: 2, + resource: wgpu::BindingResource::Sampler(sampler), + }, + ], + }) + } +} diff --git a/crates/rendering/src/layers/mod.rs b/crates/rendering/src/layers/mod.rs index 1469690f2a..7b14a87955 100644 --- a/crates/rendering/src/layers/mod.rs +++ b/crates/rendering/src/layers/mod.rs @@ -4,6 +4,7 @@ mod camera; mod captions; mod cursor; mod display; +mod mask; pub use background::*; pub use blur::*; @@ -11,3 +12,4 @@ pub use camera::*; pub use captions::*; pub use cursor::*; pub use display::*; +pub use mask::*; diff --git a/crates/rendering/src/lib.rs b/crates/rendering/src/lib.rs index aa8fb3b583..b9cd8175b0 100644 --- a/crates/rendering/src/lib.rs +++ b/crates/rendering/src/lib.rs @@ -1,7 +1,8 @@ use anyhow::Result; use cap_project::{ AspectRatio, CameraShape, CameraXPosition, CameraYPosition, ClipOffsets, CornerStyle, Crop, - CursorEvents, ProjectConfiguration, RecordingMeta, StudioRecordingMeta, XY, + CursorEvents, MaskKind, MaskSegment, ProjectConfiguration, RecordingMeta, StudioRecordingMeta, + XY, }; use composite_frame::CompositeVideoFrameUniforms; use core::f64; @@ -12,6 +13,7 @@ use futures::FutureExt; use futures::future::OptionFuture; use layers::{ Background, BackgroundLayer, BlurLayer, CameraLayer, CaptionsLayer, CursorLayer, DisplayLayer, + MaskLayer, }; use specta::Type; use spring_mass_damper::SpringMassDamperSimulationConfig; @@ -26,6 +28,7 @@ mod cursor_interpolation; pub mod decoder; mod frame_pipeline; mod layers; +mod mask; mod project_recordings; mod scene; mod spring_mass_damper; @@ -36,6 +39,7 @@ pub use decoder::DecodedFrame; pub use frame_pipeline::RenderedFrame; pub use project_recordings::{ProjectRecordingsMeta, SegmentRecordings}; +use mask::interpolate_masks; use scene::*; use zoom::*; @@ -54,6 +58,42 @@ pub struct RenderOptions { pub screen_size: XY, } +#[derive(Debug, Clone, Copy)] +pub enum MaskRenderMode { + Sensitive, + Highlight, +} + +impl MaskRenderMode { + fn from_kind(kind: MaskKind) -> Self { + match kind { + MaskKind::Sensitive => MaskRenderMode::Sensitive, + MaskKind::Highlight => MaskRenderMode::Highlight, + } + } +} + +#[derive(Debug, Clone)] +pub struct PreparedMask { + pub center: XY, + pub size: XY, + pub feather: f32, + pub opacity: f32, + pub pixel_size: f32, + pub darkness: f32, + pub mode: MaskRenderMode, + pub output_size: XY, +} + +impl PreparedMask { + fn mode_value(&self) -> u32 { + match self.mode { + MaskRenderMode::Sensitive => 0, + MaskRenderMode::Highlight => 1, + } + } +} + #[derive(Clone)] pub struct RecordingSegmentDecoders { screen: AsyncVideoDecoderHandle, @@ -382,6 +422,7 @@ pub struct ProjectUniforms { pub resolution_base: XY, pub display_parent_motion_px: XY, pub motion_blur_amount: f32, + pub masks: Vec, } #[derive(Debug, Clone)] @@ -1421,6 +1462,18 @@ impl ProjectUniforms { } }); + let masks = project + .timeline + .as_ref() + .map(|timeline| { + interpolate_masks( + XY::new(output_size.0, output_size.1), + frame_time as f64, + &timeline.mask_segments, + ) + }) + .unwrap_or_default(); + Self { output_size, cursor_size: project.cursor.size as f32, @@ -1436,6 +1489,7 @@ impl ProjectUniforms { prev_cursor: prev_interpolated_cursor, display_parent_motion_px: display_motion_parent, motion_blur_amount: user_motion_blur, + masks, } } } @@ -1500,6 +1554,7 @@ pub struct RendererLayers { cursor: CursorLayer, camera: CameraLayer, camera_only: CameraLayer, + mask: MaskLayer, #[allow(unused)] captions: CaptionsLayer, } @@ -1513,6 +1568,7 @@ impl RendererLayers { cursor: CursorLayer::new(device), camera: CameraLayer::new(device), camera_only: CameraLayer::new(device), + mask: MaskLayer::new(device), captions: CaptionsLayer::new(device, queue), } } @@ -1583,6 +1639,7 @@ impl RendererLayers { pub fn render( &self, device: &wgpu::Device, + queue: &wgpu::Queue, encoder: &mut wgpu::CommandEncoder, session: &mut RenderSession, uniforms: &ProjectUniforms, @@ -1645,6 +1702,12 @@ impl RendererLayers { let mut pass = render_pass!(session.current_texture_view(), wgpu::LoadOp::Load); self.camera.render(&mut pass); } + + if !uniforms.masks.is_empty() { + for mask in &uniforms.masks { + self.mask.render(device, queue, session, encoder, mask); + } + } } } @@ -1807,7 +1870,13 @@ async fn produce_frame( }), ); - layers.render(&constants.device, &mut encoder, session, &uniforms); + layers.render( + &constants.device, + &constants.queue, + &mut encoder, + session, + &uniforms, + ); finish_encoder( session, diff --git a/crates/rendering/src/mask.rs b/crates/rendering/src/mask.rs new file mode 100644 index 0000000000..b89e89c1cc --- /dev/null +++ b/crates/rendering/src/mask.rs @@ -0,0 +1,126 @@ +use cap_project::{MaskKind, MaskScalarKeyframe, MaskSegment, MaskVectorKeyframe, XY}; + +use crate::{MaskRenderMode, PreparedMask}; + +fn interpolate_vector(base: XY, keys: &[MaskVectorKeyframe], time: f64) -> XY { + if keys.is_empty() { + return base; + } + + let mut sorted = keys.to_vec(); + sorted.sort_by(|a, b| { + a.time + .partial_cmp(&b.time) + .unwrap_or(std::cmp::Ordering::Equal) + }); + + if time <= sorted[0].time { + return XY::new(sorted[0].x, sorted[0].y); + } + + for window in sorted.windows(2) { + let prev = &window[0]; + let next = &window[1]; + if time <= next.time { + let span = (next.time - prev.time).max(1e-6); + let t = ((time - prev.time) / span).clamp(0.0, 1.0); + let x = prev.x + (next.x - prev.x) * t; + let y = prev.y + (next.y - prev.y) * t; + return XY::new(x, y); + } + } + + let last = sorted.last().unwrap(); + XY::new(last.x, last.y) +} + +fn interpolate_scalar(base: f64, keys: &[MaskScalarKeyframe], time: f64) -> f64 { + if keys.is_empty() { + return base; + } + + let mut sorted = keys.to_vec(); + sorted.sort_by(|a, b| { + a.time + .partial_cmp(&b.time) + .unwrap_or(std::cmp::Ordering::Equal) + }); + + if time <= sorted[0].time { + return sorted[0].value; + } + + for window in sorted.windows(2) { + let prev = &window[0]; + let next = &window[1]; + if time <= next.time { + let span = (next.time - prev.time).max(1e-6); + let t = ((time - prev.time) / span).clamp(0.0, 1.0); + return prev.value + (next.value - prev.value) * t; + } + } + + sorted.last().map(|k| k.value).unwrap_or(base) +} + +pub fn interpolate_masks( + output_size: XY, + frame_time: f64, + segments: &[MaskSegment], +) -> Vec { + let mut prepared = Vec::new(); + + for segment in segments.iter().filter(|s| s.enabled) { + if frame_time < segment.start || frame_time > segment.end { + continue; + } + + let relative_time = (frame_time - segment.start).max(0.0); + + let position = + interpolate_vector(segment.center, &segment.keyframes.position, relative_time); + let size = interpolate_vector(segment.size, &segment.keyframes.size, relative_time); + let mut intensity = + interpolate_scalar(segment.opacity, &segment.keyframes.intensity, relative_time); + + if let MaskKind::Highlight = segment.mask_type { + let fade_duration = 0.15; + let time_since_start = (frame_time - segment.start).max(0.0); + let time_until_end = (segment.end - frame_time).max(0.0); + + let fade_in = (time_since_start / fade_duration).min(1.0); + let fade_out = (time_until_end / fade_duration).min(1.0); + + intensity *= fade_in * fade_out; + } + + let clamped_size = XY::new(size.x.clamp(0.01, 2.0), size.y.clamp(0.01, 2.0)); + + let min_axis = clamped_size.x.min(clamped_size.y).abs(); + let segment_feather = if let MaskKind::Highlight = segment.mask_type { + 0.0 + } else { + segment.feather + }; + let feather = (min_axis * 0.5 * segment_feather.max(0.0)).max(0.0001) as f32; + + prepared.push(PreparedMask { + center: XY::new( + position.x.clamp(0.0, 1.0) as f32, + position.y.clamp(0.0, 1.0) as f32, + ), + size: XY::new( + clamped_size.x.clamp(0.0, 2.0) as f32, + clamped_size.y.clamp(0.0, 2.0) as f32, + ), + feather, + opacity: intensity.clamp(0.0, 1.0) as f32, + pixel_size: segment.pixelation.max(1.0) as f32, + darkness: segment.darkness.clamp(0.0, 1.0) as f32, + mode: MaskRenderMode::from_kind(segment.mask_type), + output_size, + }); + } + + prepared +} diff --git a/crates/rendering/src/shaders/mask.wgsl b/crates/rendering/src/shaders/mask.wgsl new file mode 100644 index 0000000000..5269801cc7 --- /dev/null +++ b/crates/rendering/src/shaders/mask.wgsl @@ -0,0 +1,71 @@ +struct Uniforms { + rect_center: vec2, + rect_size: vec2, + feather: f32, + opacity: f32, + pixel_size: f32, + darkness: f32, + mode: u32, + padding0: u32, + output_size: vec2, + padding1: vec2, +} + +@group(0) @binding(0) var uniforms: Uniforms; +@group(0) @binding(1) var source_texture: texture_2d; +@group(0) @binding(2) var source_sampler: sampler; + +struct VertexOutput { + @builtin(position) position: vec4, + @location(0) uv: vec2, +} + +@vertex +fn vs_main(@builtin(vertex_index) vertex_index: u32) -> VertexOutput { + var positions = array, 3>( + vec2(-1.0, -1.0), + vec2(3.0, -1.0), + vec2(-1.0, 3.0), + ); + + let pos = positions[vertex_index]; + var out: VertexOutput; + out.position = vec4(pos, 0.0, 1.0); + out.uv = vec2(pos.x * 0.5 + 0.5, 1.0 - (pos.y * 0.5 + 0.5)); + return out; +} + +fn rect_mask(uv: vec2) -> f32 { + let half_size = uniforms.rect_size * 0.5; + let delta = abs(uv - uniforms.rect_center) - half_size; + let outside = max(delta, vec2(0.0)); + let outside_dist = length(outside); + let inside_dist = min(max(delta.x, delta.y), 0.0); + let sdf = outside_dist + inside_dist; + let edge = max(uniforms.feather, 1e-4); + return clamp(smoothstep(0.0, edge, -sdf), 0.0, 1.0); +} + +fn pixelate_sample(uv: vec2) -> vec4 { + let px_size = max(uniforms.pixel_size, 1.0); + let cell = px_size / uniforms.output_size; + let snapped = floor(uv / cell) * cell + cell * 0.5; + return textureSample(source_texture, source_sampler, snapped); +} + +@fragment +fn fs_main(@location(0) uv: vec2) -> @location(0) vec4 { + let base = textureSample(source_texture, source_sampler, uv); + let mask = rect_mask(uv); + + if uniforms.mode == 0u { + let pixelated = pixelate_sample(uv); + let mix_amount = clamp(uniforms.opacity, 0.0, 1.0); + let effect = mix(base, pixelated, mix_amount); + return mix(base, effect, mask * mix_amount); + } + + let darkness = clamp(uniforms.darkness * uniforms.opacity, 0.0, 1.0); + let outside = vec4(base.rgb * (1.0 - darkness), base.a); + return mix(outside, base, mask); +} diff --git a/packages/ui-solid/src/auto-imports.d.ts b/packages/ui-solid/src/auto-imports.d.ts index c10b785a8b..c19129bd88 100644 --- a/packages/ui-solid/src/auto-imports.d.ts +++ b/packages/ui-solid/src/auto-imports.d.ts @@ -67,6 +67,7 @@ declare global { const IconHugeiconsEaseCurveControlPoints: typeof import('~icons/hugeicons/ease-curve-control-points.jsx')['default'] const IconLucideAlertTriangle: typeof import('~icons/lucide/alert-triangle.jsx')['default'] const IconLucideBell: typeof import('~icons/lucide/bell.jsx')['default'] + const IconLucideBoxSelect: typeof import('~icons/lucide/box-select.jsx')['default'] const IconLucideBug: typeof import('~icons/lucide/bug.jsx')['default'] const IconLucideCheck: typeof import('~icons/lucide/check.jsx')['default'] const IconLucideClapperboard: typeof import('~icons/lucide/clapperboard.jsx')['default'] @@ -92,6 +93,7 @@ declare global { const IconLucideSave: typeof import('~icons/lucide/save.jsx')['default'] const IconLucideSearch: typeof import('~icons/lucide/search.jsx')['default'] const IconLucideSquarePlay: typeof import('~icons/lucide/square-play.jsx')['default'] + const IconLucideTimer: typeof import('~icons/lucide/timer.jsx')['default'] const IconLucideUnplug: typeof import('~icons/lucide/unplug.jsx')['default'] const IconLucideVideo: typeof import('~icons/lucide/video.jsx')['default'] const IconLucideVolume2: typeof import('~icons/lucide/volume2.jsx')['default'] From 2dba2de99a96444a611fba83bedab0f05df89cc3 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Mon, 1 Dec 2025 23:06:23 +0800 Subject: [PATCH 30/46] feat: Implement captions support --- apps/desktop/src-tauri/src/captions.rs | 326 +++++++------ .../desktop/src/routes/editor/CaptionsTab.tsx | 388 ++++++++------- crates/project/src/configuration.rs | 110 ++++- crates/rendering/src/layers/captions.rs | 448 +++++++++++------- 4 files changed, 752 insertions(+), 520 deletions(-) diff --git a/apps/desktop/src-tauri/src/captions.rs b/apps/desktop/src-tauri/src/captions.rs index 05678af591..cfb75f9145 100644 --- a/apps/desktop/src-tauri/src/captions.rs +++ b/apps/desktop/src-tauri/src/captions.rs @@ -18,12 +18,10 @@ use tokio::sync::Mutex; use tracing::instrument; use whisper_rs::{FullParams, SamplingStrategy, WhisperContext, WhisperContextParameters}; -// Re-export caption types from cap_project -pub use cap_project::{CaptionSegment, CaptionSettings}; +pub use cap_project::{CaptionSegment, CaptionSettings, CaptionWord}; use crate::http_client; -// Convert the project type's float precision from f32 to f64 for compatibility #[derive(Debug, Serialize, Deserialize, Type, Clone)] pub struct CaptionData { pub segments: Vec, @@ -39,15 +37,12 @@ impl Default for CaptionData { } } -// Model context is shared and cached lazy_static::lazy_static! { - static ref WHISPER_CONTEXT: Arc>> = Arc::new(Mutex::new(None)); + static ref WHISPER_CONTEXT: Arc>>> = Arc::new(Mutex::new(None)); } -// Constants const WHISPER_SAMPLE_RATE: u32 = 16000; -/// Function to handle creating directories for the model #[tauri::command] #[specta::specta] #[instrument] @@ -55,7 +50,6 @@ pub async fn create_dir(path: String, _recursive: bool) -> Result<(), String> { std::fs::create_dir_all(path).map_err(|e| format!("Failed to create directory: {e}")) } -/// Function to save the model file #[tauri::command] #[specta::specta] #[instrument] @@ -63,15 +57,12 @@ pub async fn save_model_file(path: String, data: Vec) -> Result<(), String> std::fs::write(&path, &data).map_err(|e| format!("Failed to write model file: {e}")) } -/// Extract audio from a video file and save it as a temporary WAV file async fn extract_audio_from_video(video_path: &str, output_path: &PathBuf) -> Result<(), String> { log::info!("Attempting to extract audio from: {video_path}"); - // Check if this is a .cap directory if video_path.ends_with(".cap") { log::info!("Detected .cap project directory"); - // Read the recording metadata let meta_path = std::path::Path::new(video_path).join("recording-meta.json"); let meta_content = std::fs::read_to_string(&meta_path) .map_err(|e| format!("Failed to read recording metadata: {e}"))?; @@ -79,21 +70,23 @@ async fn extract_audio_from_video(video_path: &str, output_path: &PathBuf) -> Re let meta: serde_json::Value = serde_json::from_str(&meta_content) .map_err(|e| format!("Failed to parse recording metadata: {e}"))?; - // Get paths for both audio sources let base_path = std::path::Path::new(video_path); let mut audio_sources = Vec::new(); if let Some(segments) = meta["segments"].as_array() { for segment in segments { - // Add system audio if available - if let Some(system_audio) = segment["system_audio"]["path"].as_str() { - audio_sources.push(base_path.join(system_audio)); - } + let mut push_source = |path: Option<&str>| { + if let Some(path) = path { + let full_path = base_path.join(path); + if !audio_sources.contains(&full_path) { + audio_sources.push(full_path); + } + } + }; - // Add microphone audio if available - if let Some(audio) = segment["audio"]["path"].as_str() { - audio_sources.push(base_path.join(audio)); - } + push_source(segment["system_audio"]["path"].as_str()); + push_source(segment["mic"]["path"].as_str()); + push_source(segment["audio"]["path"].as_str()); } } @@ -103,7 +96,6 @@ async fn extract_audio_from_video(video_path: &str, output_path: &PathBuf) -> Re log::info!("Found {} audio sources", audio_sources.len()); - // Process each audio source using AudioData let mut mixed_samples = Vec::new(); let mut channel_count = 0; @@ -121,7 +113,6 @@ async fn extract_audio_from_video(video_path: &str, output_path: &PathBuf) -> Re mixed_samples = audio.samples().to_vec(); channel_count = audio.channels() as usize; } else { - // Handle potential different channel counts by mixing to mono first if needed if audio.channels() as usize != channel_count { log::info!( "Channel count mismatch: {} vs {}, mixing to mono", @@ -129,24 +120,20 @@ async fn extract_audio_from_video(video_path: &str, output_path: &PathBuf) -> Re audio.channels() ); - // If we have mixed samples with multiple channels, convert to mono if channel_count > 1 { let mono_samples = convert_to_mono(&mixed_samples, channel_count); mixed_samples = mono_samples; channel_count = 1; } - // Convert the new audio to mono too if it has multiple channels let samples = if audio.channels() > 1 { convert_to_mono(audio.samples(), audio.channels() as usize) } else { audio.samples().to_vec() }; - // Mix mono samples mix_samples(&mut mixed_samples, &samples); } else { - // Same channel count, simple mix mix_samples(&mut mixed_samples, audio.samples()); } } @@ -158,7 +145,6 @@ async fn extract_audio_from_video(video_path: &str, output_path: &PathBuf) -> Re } } - // No matter what, ensure we have mono audio for Whisper if channel_count > 1 { log::info!("Converting final mixed audio from {channel_count} channels to mono"); mixed_samples = convert_to_mono(&mixed_samples, channel_count); @@ -169,7 +155,6 @@ async fn extract_audio_from_video(video_path: &str, output_path: &PathBuf) -> Re return Err("Failed to process any audio sources".to_string()); } - // Convert to WAV format with desired sample rate let mut output = avformat::output(&output_path) .map_err(|e| format!("Failed to create output file: {e}"))?; @@ -199,7 +184,6 @@ async fn extract_audio_from_video(video_path: &str, output_path: &PathBuf) -> Re .write_header() .map_err(|e| format!("Failed to write header: {e}"))?; - // Create resampler for sample rate conversion let mut resampler = resampling::Context::get( avformat::Sample::F32(avformat::sample::Type::Packed), channel_layout, @@ -210,9 +194,7 @@ async fn extract_audio_from_video(video_path: &str, output_path: &PathBuf) -> Re ) .map_err(|e| format!("Failed to create resampler: {e}"))?; - // Process audio in chunks let frame_size = encoder.frame_size() as usize; - // Check if frame_size is zero and use a fallback let frame_size = if frame_size == 0 { 1024 } else { frame_size }; log::info!( @@ -229,15 +211,12 @@ async fn extract_audio_from_video(video_path: &str, output_path: &PathBuf) -> Re ); frame.set_rate(WHISPER_SAMPLE_RATE); - // Make sure we have samples and a valid chunk size if !mixed_samples.is_empty() && frame_size * channel_count > 0 { - // Process chunks of audio for (chunk_idx, chunk) in mixed_samples.chunks(frame_size * channel_count).enumerate() { if chunk_idx % 100 == 0 { log::info!("Processing chunk {}, size: {}", chunk_idx, chunk.len()); } - // Create a new input frame with actual data from the chunk let mut input_frame = ffmpeg::frame::Audio::new( avformat::Sample::F32(avformat::sample::Type::Packed), chunk.len() / channel_count, @@ -245,7 +224,6 @@ async fn extract_audio_from_video(video_path: &str, output_path: &PathBuf) -> Re ); input_frame.set_rate(AudioData::SAMPLE_RATE); - // Copy data from chunk to frame let bytes = unsafe { std::slice::from_raw_parts( chunk.as_ptr() as *const u8, @@ -254,7 +232,6 @@ async fn extract_audio_from_video(video_path: &str, output_path: &PathBuf) -> Re }; input_frame.data_mut(0)[0..bytes.len()].copy_from_slice(bytes); - // Create output frame for resampled data let mut output_frame = ffmpeg::frame::Audio::new( avformat::Sample::I16(avformat::sample::Type::Packed), frame_size, @@ -262,7 +239,6 @@ async fn extract_audio_from_video(video_path: &str, output_path: &PathBuf) -> Re ); output_frame.set_rate(WHISPER_SAMPLE_RATE); - // Use the input frame with actual data instead of the empty frame match resampler.run(&input_frame, &mut output_frame) { Ok(_) => { if chunk_idx % 100 == 0 { @@ -284,7 +260,6 @@ async fn extract_audio_from_video(video_path: &str, output_path: &PathBuf) -> Re continue; } - // Process each encoded packet loop { let mut packet = ffmpeg::Packet::empty(); match encoder.receive_packet(&mut packet) { @@ -299,12 +274,10 @@ async fn extract_audio_from_video(video_path: &str, output_path: &PathBuf) -> Re } } - // Flush the encoder encoder .send_eof() .map_err(|e| format!("Failed to send EOF: {e}"))?; - // Process final packets in a loop with limited borrow scope loop { let mut packet = ffmpeg::Packet::empty(); let received = encoder.receive_packet(&mut packet); @@ -313,7 +286,6 @@ async fn extract_audio_from_video(video_path: &str, output_path: &PathBuf) -> Re break; } - // Use a block to limit the scope of the output borrow { if let Err(e) = packet.write_interleaved(&mut output) { return Err(format!("Failed to write final packet: {e}")); @@ -327,7 +299,6 @@ async fn extract_audio_from_video(video_path: &str, output_path: &PathBuf) -> Re Ok(()) } else { - // Handle regular video file let mut input = avformat::input(&video_path).map_err(|e| format!("Failed to open video file: {e}"))?; @@ -338,25 +309,20 @@ async fn extract_audio_from_video(video_path: &str, output_path: &PathBuf) -> Re let codec_params = stream.parameters(); - // Get decoder parameters first let decoder_ctx = avcodec::Context::from_parameters(codec_params.clone()) .map_err(|e| format!("Failed to create decoder context: {e}"))?; - // Create and open the decoder let mut decoder = decoder_ctx .decoder() .audio() .map_err(|e| format!("Failed to create decoder: {e}"))?; - // Now we can access audio-specific methods let decoder_format = decoder.format(); let decoder_channel_layout = decoder.channel_layout(); let decoder_rate = decoder.rate(); - // Set up and prepare encoder and output separately to avoid multiple borrows let channel_layout = ChannelLayout::MONO; - // Create encoder first let mut encoder_ctx = avcodec::Context::new() .encoder() .audio() @@ -373,11 +339,9 @@ async fn extract_audio_from_video(video_path: &str, output_path: &PathBuf) -> Re .open_as(codec) .map_err(|e| format!("Failed to open encoder: {e}"))?; - // Create output context separately let mut output = avformat::output(&output_path) .map_err(|e| format!("Failed to create output file: {e}"))?; - // Add stream and get parameters in a block to limit the borrow let stream_params = { let mut output_stream = output .add_stream(codec) @@ -385,16 +349,13 @@ async fn extract_audio_from_video(video_path: &str, output_path: &PathBuf) -> Re output_stream.set_parameters(&encoder); - // Store the stream parameters we need for later (output_stream.index(), output_stream.id()) }; - // Write header output .write_header() .map_err(|e| format!("Failed to write header: {e}"))?; - // Create resampler let mut resampler = resampling::Context::get( decoder_format, decoder_channel_layout, @@ -405,7 +366,6 @@ async fn extract_audio_from_video(video_path: &str, output_path: &PathBuf) -> Re ) .map_err(|e| format!("Failed to create resampler: {e}"))?; - // Create frames let mut decoded_frame = ffmpeg::frame::Audio::empty(); let mut resampled_frame = ffmpeg::frame::Audio::new( avformat::Sample::I16(avformat::sample::Type::Packed), @@ -413,22 +373,15 @@ async fn extract_audio_from_video(video_path: &str, output_path: &PathBuf) -> Re channel_layout, ); - // Save the stream index from the original stream (not the output stream) let input_stream_index = stream.index(); - // Process packets one at a time, cloning what we need from input packets let mut packet_queue = Vec::new(); - // First collect all the packets we need by cloning the data { - // Use a separate block to limit the immutable borrow lifetime for (stream_idx, packet) in input.packets() { if stream_idx.index() == input_stream_index { - // Clone the packet data to avoid borrowing input if let Some(data) = packet.data() { - // Copy the packet data to a new packet let mut cloned_packet = ffmpeg::Packet::copy(data); - // Copy timing information if let Some(pts) = packet.pts() { cloned_packet.set_pts(Some(pts)); } @@ -441,14 +394,12 @@ async fn extract_audio_from_video(video_path: &str, output_path: &PathBuf) -> Re } } - // Then process each cloned packet for packet_res in packet_queue { if let Err(e) = decoder.send_packet(&packet_res) { log::warn!("Failed to send packet to decoder: {e}"); continue; } - // Process decoded frames while decoder.receive_frame(&mut decoded_frame).is_ok() { if let Err(e) = resampler.run(&decoded_frame, &mut resampled_frame) { log::warn!("Failed to resample audio: {e}"); @@ -460,12 +411,10 @@ async fn extract_audio_from_video(video_path: &str, output_path: &PathBuf) -> Re continue; } - // Process encoded packets loop { let mut packet = ffmpeg::Packet::empty(); match encoder.receive_packet(&mut packet) { Ok(_) => { - // Set the stream for the output packet packet.set_stream(stream_params.0); if let Err(e) = packet.write_interleaved(&mut output) { @@ -478,7 +427,6 @@ async fn extract_audio_from_video(video_path: &str, output_path: &PathBuf) -> Re } } - // Flush the decoder decoder .send_eof() .map_err(|e| format!("Failed to send EOF to decoder: {e}"))?; @@ -492,7 +440,6 @@ async fn extract_audio_from_video(video_path: &str, output_path: &PathBuf) -> Re .send_frame(&resampled_frame) .map_err(|e| format!("Failed to send final frame: {e}"))?; - // Process final encoded packets loop { let mut packet = ffmpeg::Packet::empty(); let received = encoder.receive_packet(&mut packet); @@ -507,7 +454,6 @@ async fn extract_audio_from_video(video_path: &str, output_path: &PathBuf) -> Re } } - // Close the output file with trailer output .write_trailer() .map_err(|e| format!("Failed to write trailer: {e}"))?; @@ -516,24 +462,32 @@ async fn extract_audio_from_video(video_path: &str, output_path: &PathBuf) -> Re } } -/// Load or initialize the WhisperContext async fn get_whisper_context(model_path: &str) -> Result, String> { let mut context_guard = WHISPER_CONTEXT.lock().await; - // Always create a new context to avoid issues with multiple uses log::info!("Initializing Whisper context with model: {model_path}"); let ctx = WhisperContext::new_with_params(model_path, WhisperContextParameters::default()) .map_err(|e| format!("Failed to load Whisper model: {e}"))?; - *context_guard = Some(ctx); + let ctx_arc = Arc::new(ctx); + *context_guard = Some(ctx_arc.clone()); - // Get a reference to the context and wrap it in an Arc - let context_ref = context_guard.as_ref().unwrap(); - let context_arc = unsafe { Arc::new(std::ptr::read(context_ref)) }; - Ok(context_arc) + Ok(ctx_arc) +} + +fn is_special_token(token_text: &str) -> bool { + let trimmed = token_text.trim(); + if trimmed.is_empty() { + return true; + } + + trimmed.contains('[') + || trimmed.contains(']') + || trimmed.contains("_TT_") + || trimmed.contains("_BEG_") + || trimmed.contains("<|") } -/// Process audio file with Whisper for transcription fn process_with_whisper( audio_path: &PathBuf, context: Arc, @@ -541,19 +495,16 @@ fn process_with_whisper( ) -> Result { log::info!("Processing audio file: {audio_path:?}"); - // Set up parameters for Whisper let mut params = FullParams::new(SamplingStrategy::Greedy { best_of: 1 }); - // Configure parameters for better caption quality params.set_translate(false); params.set_print_special(false); params.set_print_progress(false); params.set_print_realtime(false); - params.set_token_timestamps(true); // Enable timestamps for captions - params.set_language(Some(if language == "auto" { "auto" } else { language })); // Use selected language or auto-detect - params.set_max_len(i32::MAX); // No max length for transcription + params.set_token_timestamps(true); + params.set_language(Some(if language == "auto" { "auto" } else { language })); + params.set_max_len(i32::MAX); - // Load audio file let mut audio_file = File::open(audio_path) .map_err(|e| format!("Failed to open audio file: {e} at path: {audio_path:?}"))?; let mut audio_data = Vec::new(); @@ -563,7 +514,6 @@ fn process_with_whisper( log::info!("Processing audio file of size: {} bytes", audio_data.len()); - // Convert audio data to the required format (16-bit mono PCM) let mut audio_data_f32 = Vec::new(); for i in (0..audio_data.len()).step_by(2) { if i + 1 < audio_data.len() { @@ -574,14 +524,12 @@ fn process_with_whisper( log::info!("Converted {} samples to f32 format", audio_data_f32.len()); - // Log sample data statistics for debugging if !audio_data_f32.is_empty() { let min_sample = audio_data_f32.iter().fold(f32::MAX, |a, &b| a.min(b)); let max_sample = audio_data_f32.iter().fold(f32::MIN, |a, &b| a.max(b)); let avg_sample = audio_data_f32.iter().sum::() / audio_data_f32.len() as f32; log::info!("Audio samples - min: {min_sample}, max: {max_sample}, avg: {avg_sample}"); - // Sample a few values let sample_count = audio_data_f32.len().min(10); for i in 0..sample_count { let idx = i * audio_data_f32.len() / sample_count; @@ -589,7 +537,6 @@ fn process_with_whisper( } } - // Run the transcription let mut state = context .create_state() .map_err(|e| format!("Failed to create Whisper state: {e}"))?; @@ -598,7 +545,6 @@ fn process_with_whisper( .full(params, &audio_data_f32[..]) .map_err(|e| format!("Failed to run Whisper transcription: {e}"))?; - // Process results: convert Whisper segments to CaptionSegment let num_segments = state .full_n_segments() .map_err(|e| format!("Failed to get number of segments: {e}"))?; @@ -608,11 +554,10 @@ fn process_with_whisper( let mut segments = Vec::new(); for i in 0..num_segments { - let text = state + let raw_text = state .full_get_segment_text(i) .map_err(|e| format!("Failed to get segment text: {e}"))?; - // Properly unwrap the Result first, then convert i64 to f64 let start_i64 = state .full_get_segment_t0(i) .map_err(|e| format!("Failed to get segment start time: {e}"))?; @@ -620,27 +565,91 @@ fn process_with_whisper( .full_get_segment_t1(i) .map_err(|e| format!("Failed to get segment end time: {e}"))?; - // Convert timestamps from centiseconds to seconds (as f32 for CaptionSegment) let start_time = (start_i64 as f32) / 100.0; let end_time = (end_i64 as f32) / 100.0; - // Add debug logging for timestamps log::info!( "Segment {}: start={}, end={}, text='{}'", i, start_time, end_time, - text.trim() + raw_text.trim() ); - if !text.trim().is_empty() { - segments.push(CaptionSegment { - id: format!("segment-{i}"), - start: start_time, - end: end_time, - text: text.trim().to_string(), - }); + let mut words = Vec::new(); + let num_tokens = state + .full_n_tokens(i) + .map_err(|e| format!("Failed to get token count: {e}"))?; + + let mut current_word = String::new(); + let mut word_start: Option = None; + let mut word_end: f32 = start_time; + + for t in 0..num_tokens { + let token_text = state.full_get_token_text(i, t).unwrap_or_default(); + + if is_special_token(&token_text) { + continue; + } + + let token_data = state.full_get_token_data(i, t).ok(); + + if let Some(data) = token_data { + let token_start = (data.t0 as f32) / 100.0; + let token_end = (data.t1 as f32) / 100.0; + + if token_text.starts_with(' ') || token_text.starts_with('\n') { + if !current_word.is_empty() { + if let Some(ws) = word_start { + words.push(CaptionWord { + text: current_word.trim().to_string(), + start: ws, + end: word_end, + }); + } + } + current_word = token_text.trim().to_string(); + word_start = None; + } else { + if word_start.is_none() { + word_start = Some(token_start); + } + current_word.push_str(&token_text); + } + word_end = token_end; + } + } + + if !current_word.trim().is_empty() { + if let Some(ws) = word_start { + words.push(CaptionWord { + text: current_word.trim().to_string(), + start: ws, + end: word_end, + }); + } } + + if words.is_empty() { + continue; + } + + let segment_text = words + .iter() + .map(|word| word.text.clone()) + .collect::>() + .join(" "); + + let segment_start = words.first().map(|word| word.start).unwrap_or(start_time); + let segment_end = words.last().map(|word| word.end).unwrap_or(end_time); + + segments.push(CaptionSegment { + id: format!("segment-{i}"), + start: segment_start, + end: segment_end, + text: segment_text, + words, + }); } log::info!("Successfully processed {} segments", segments.len()); @@ -651,7 +660,6 @@ fn process_with_whisper( }) } -/// Function to transcribe audio from a video file using Whisper #[tauri::command] #[specta::specta] #[instrument] @@ -660,7 +668,6 @@ pub async fn transcribe_audio( model_path: String, language: String, ) -> Result { - // Check if files exist with detailed error messages if !std::path::Path::new(&video_path).exists() { return Err(format!("Video file not found at path: {video_path}")); } @@ -669,11 +676,9 @@ pub async fn transcribe_audio( return Err(format!("Model file not found at path: {model_path}")); } - // Create temp dir with better error handling let temp_dir = tempdir().map_err(|e| format!("Failed to create temporary directory: {e}"))?; let audio_path = temp_dir.path().join("audio.wav"); - // First try the ffmpeg implementation match extract_audio_from_video(&video_path, &audio_path).await { Ok(_) => log::info!("Successfully extracted audio to {audio_path:?}"), Err(e) => { @@ -682,14 +687,12 @@ pub async fn transcribe_audio( } } - // Verify the audio file was created if !audio_path.exists() { return Err("Failed to create audio file for transcription".to_string()); } log::info!("Audio file created at: {audio_path:?}"); - // Get or initialize Whisper context with detailed error handling let context = match get_whisper_context(&model_path).await { Ok(ctx) => ctx, Err(e) => { @@ -698,8 +701,15 @@ pub async fn transcribe_audio( } }; - // Process with Whisper and handle errors - match process_with_whisper(&audio_path, context, &language) { + let audio_path_clone = audio_path.clone(); + let language_clone = language.clone(); + let whisper_result = tokio::task::spawn_blocking(move || { + process_with_whisper(&audio_path_clone, context, &language_clone) + }) + .await + .map_err(|e| format!("Whisper task panicked: {e}"))?; + + match whisper_result { Ok(captions) => { if captions.segments.is_empty() { log::warn!("No caption segments were generated"); @@ -714,7 +724,6 @@ pub async fn transcribe_audio( } } -/// Function to save caption data to a file #[tauri::command] #[specta::specta] #[instrument(skip(app))] @@ -739,13 +748,10 @@ pub async fn save_captions( tracing::info!("Writing captions to: {:?}", captions_path); - // Ensure settings are included with default values if not provided let settings = captions.settings.unwrap_or_default(); - // Create a JSON structure manually to ensure field naming consistency let mut json_obj = serde_json::Map::new(); - // Add segments array let segments_array = serde_json::to_value( captions .segments @@ -769,6 +775,18 @@ pub async fn save_captions( "text".to_string(), serde_json::Value::String(seg.text.clone()), ); + let words_array: Vec = seg + .words + .iter() + .map(|w| { + serde_json::json!({ + "text": w.text, + "start": w.start, + "end": w.end + }) + }) + .collect(); + segment.insert("words".to_string(), serde_json::Value::Array(words_array)); segment }) .collect::>(), @@ -780,7 +798,6 @@ pub async fn save_captions( json_obj.insert("segments".to_string(), segments_array); - // Add settings object with camelCase naming let mut settings_obj = serde_json::Map::new(); settings_obj.insert( "enabled".to_string(), @@ -827,13 +844,22 @@ pub async fn save_captions( "exportWithSubtitles".to_string(), serde_json::Value::Bool(settings.export_with_subtitles), ); + settings_obj.insert( + "highlightColor".to_string(), + serde_json::Value::String(settings.highlight_color.clone()), + ); + settings_obj.insert( + "fadeDuration".to_string(), + serde_json::Value::Number( + serde_json::Number::from_f64(settings.fade_duration as f64).unwrap(), + ), + ); json_obj.insert( "settings".to_string(), serde_json::Value::Object(settings_obj), ); - // Convert to pretty JSON string let json = serde_json::to_string_pretty(&json_obj).map_err(|e| { tracing::error!("Failed to serialize captions: {}", e); format!("Failed to serialize captions: {e}") @@ -848,16 +874,12 @@ pub async fn save_captions( Ok(()) } -/// Helper function to parse captions from a JSON string -/// This can be used by other modules to parse captions without duplicating code pub fn parse_captions_json(json: &str) -> Result { - // Use a more flexible parsing approach match serde_json::from_str::(json) { Ok(json_value) => { if let Some(segments_array) = json_value.get("segments").and_then(|v| v.as_array()) { let mut segments = Vec::new(); - // Process each segment for segment in segments_array { if let (Some(id), Some(start), Some(end), Some(text)) = ( segment.get("id").and_then(|v| v.as_str()), @@ -865,18 +887,33 @@ pub fn parse_captions_json(json: &str) -> Result Result Result Result Result Result { tracing::info!("Getting captions directory for video_id: {}", video_id); - // Get the app data directory let app_dir = app .path() .app_data_dir() .map_err(|_| "Failed to get app data directory".to_string())?; - // Create a dedicated captions directory - // Strip .cap extension if present in video_id let clean_video_id = video_id.trim_end_matches(".cap"); let captions_dir = app_dir.join("captions").join(clean_video_id); @@ -1036,7 +1080,6 @@ fn app_captions_dir(app: &AppHandle, video_id: &str) -> Result Ok(captions_dir) } -// Add new type for download progress #[derive(Debug, Serialize, Type, tauri_specta::Event, Clone)] pub struct DownloadProgress { pub progress: f64, @@ -1047,7 +1090,6 @@ impl DownloadProgress { const EVENT_NAME: &'static str = "download-progress"; } -/// Helper function to download a Whisper model from Hugging Face Hub #[tauri::command] #[specta::specta] #[instrument(skip(window))] @@ -1057,7 +1099,6 @@ pub async fn download_whisper_model( model_name: String, output_path: String, ) -> Result<(), String> { - // Define model URLs based on model names let model_url = match model_name.as_str() { "tiny" => "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-tiny.bin", "base" => "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.bin", @@ -1065,10 +1106,9 @@ pub async fn download_whisper_model( "medium" => "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-medium.bin", "large" => "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-large-v3.bin", "large-v3" => "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-large-v3.bin", - _ => "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-tiny.bin", // Default to tiny + _ => "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-tiny.bin", }; - // Create the client and download the model let response = app .state::() .get(model_url) @@ -1083,10 +1123,8 @@ pub async fn download_whisper_model( )); } - // Get the total size for progress calculation let total_size = response.content_length().unwrap_or(0); - // Create a file to write to if let Some(parent) = std::path::Path::new(&output_path).parent() { std::fs::create_dir_all(parent) .map_err(|e| format!("Failed to create parent directories: {e}"))?; @@ -1095,15 +1133,13 @@ pub async fn download_whisper_model( .await .map_err(|e| format!("Failed to create file: {e}"))?; - // Download and write in chunks let mut downloaded = 0; let mut bytes = response .bytes() .await .map_err(|e| format!("Failed to get response bytes: {e}"))?; - // Write the bytes in chunks to show progress - const CHUNK_SIZE: usize = 1024 * 1024; // 1MB chunks + const CHUNK_SIZE: usize = 1024 * 1024; while !bytes.is_empty() { let chunk_size = std::cmp::min(CHUNK_SIZE, bytes.len()); let chunk = bytes.split_to(chunk_size); @@ -1114,7 +1150,6 @@ pub async fn download_whisper_model( downloaded += chunk_size as u64; - // Calculate and emit progress let progress = if total_size > 0 { (downloaded as f64 / total_size as f64) * 100.0 } else { @@ -1132,7 +1167,6 @@ pub async fn download_whisper_model( .map_err(|e| format!("Failed to emit progress: {e}"))?; } - // Ensure file is properly written file.flush() .await .map_err(|e| format!("Failed to flush file: {e}"))?; @@ -1140,7 +1174,6 @@ pub async fn download_whisper_model( Ok(()) } -/// Function to check if a model file exists #[tauri::command] #[specta::specta] #[instrument] @@ -1148,7 +1181,6 @@ pub async fn check_model_exists(model_path: String) -> Result { Ok(std::path::Path::new(&model_path).exists()) } -/// Function to delete a downloaded model #[tauri::command] #[specta::specta] #[instrument] @@ -1164,15 +1196,12 @@ pub async fn delete_whisper_model(model_path: String) -> Result<(), String> { Ok(()) } -/// Convert caption segments to SRT format fn captions_to_srt(captions: &CaptionData) -> String { let mut srt = String::new(); for (i, segment) in captions.segments.iter().enumerate() { - // Convert start and end times from seconds to HH:MM:SS,mmm format let start_time = format_srt_time(f64::from(segment.start)); let end_time = format_srt_time(f64::from(segment.end)); - // Write SRT entry srt.push_str(&format!( "{}\n{} --> {}\n{}\n\n", i + 1, @@ -1184,7 +1213,6 @@ fn captions_to_srt(captions: &CaptionData) -> String { srt } -/// Format time in seconds to SRT time format (HH:MM:SS,mmm) fn format_srt_time(seconds: f64) -> String { let hours = (seconds / 3600.0) as i32; let minutes = ((seconds % 3600.0) / 60.0) as i32; @@ -1193,7 +1221,6 @@ fn format_srt_time(seconds: f64) -> String { format!("{hours:02}:{minutes:02}:{secs:02},{millis:03}") } -/// Export captions to an SRT file #[tauri::command] #[specta::specta] #[instrument(skip(app))] @@ -1203,7 +1230,6 @@ pub async fn export_captions_srt( ) -> Result, String> { tracing::info!("Starting SRT export for video_id: {}", video_id); - // Load captions let captions = match load_captions(app.clone(), video_id.clone()).await? { Some(c) => { tracing::info!("Found {} caption segments to export", c.segments.len()); @@ -1215,8 +1241,6 @@ pub async fn export_captions_srt( } }; - // Ensure we have settings (this should already be handled by load_captions, - // but we add this check for extra safety) let captions_with_settings = CaptionData { segments: captions.segments, settings: captions @@ -1224,16 +1248,13 @@ pub async fn export_captions_srt( .or_else(|| Some(CaptionSettings::default())), }; - // Convert to SRT format tracing::info!("Converting captions to SRT format"); let srt_content = captions_to_srt(&captions_with_settings); - // Get path for SRT file let captions_dir = app_captions_dir(&app, &video_id)?; let srt_path = captions_dir.join("captions.srt"); tracing::info!("Will write SRT file to: {:?}", srt_path); - // Write SRT file match std::fs::write(&srt_path, srt_content) { Ok(_) => { tracing::info!("Successfully wrote SRT file to: {:?}", srt_path); @@ -1246,7 +1267,6 @@ pub async fn export_captions_srt( } } -// Helper function to convert multi-channel audio to mono fn convert_to_mono(samples: &[f32], channels: usize) -> Vec { if channels == 1 { return samples.to_vec(); @@ -1266,11 +1286,9 @@ fn convert_to_mono(samples: &[f32], channels: usize) -> Vec { mono_samples } -// Helper function to mix two sample arrays together fn mix_samples(dest: &mut [f32], source: &[f32]) -> usize { let length = dest.len().min(source.len()); for i in 0..length { - // Simple mix with equal weight (0.5) to prevent clipping dest[i] = (dest[i] + source[i]) * 0.5; } length diff --git a/apps/desktop/src/routes/editor/CaptionsTab.tsx b/apps/desktop/src/routes/editor/CaptionsTab.tsx index d6677d16b4..bc1db3a3d0 100644 --- a/apps/desktop/src/routes/editor/CaptionsTab.tsx +++ b/apps/desktop/src/routes/editor/CaptionsTab.tsx @@ -4,12 +4,22 @@ import { createWritableMemo } from "@solid-primitives/memo"; import { createElementSize } from "@solid-primitives/resize-observer"; import { appLocalDataDir, join } from "@tauri-apps/api/path"; import { exists } from "@tauri-apps/plugin-fs"; -import { batch, createEffect, createSignal, onMount, Show } from "solid-js"; +import { cx } from "cva"; +import { + batch, + createEffect, + createSignal, + For, + onMount, + Show, +} from "solid-js"; import { createStore } from "solid-js/store"; import toast from "solid-toast"; import { Toggle } from "~/components/Toggle"; import type { CaptionSegment, CaptionSettings } from "~/utils/tauri"; import { commands, events } from "~/utils/tauri"; +import IconLucideCheck from "~icons/lucide/check"; +import IconLucideDownload from "~icons/lucide/download"; import { FPS, useEditorContext } from "./context"; import { TextInput } from "./TextInput"; import { @@ -23,10 +33,11 @@ import { topLeftAnimateClasses, } from "./ui"; -// Model information interface ModelOption { name: string; label: string; + size: string; + description: string; } interface LanguageOption { @@ -40,11 +51,18 @@ interface FontOption { } const MODEL_OPTIONS: ModelOption[] = [ - { name: "tiny", label: "Tiny (75MB) - Fastest, less accurate" }, - { name: "base", label: "Base (142MB) - Fast, decent accuracy" }, - { name: "small", label: "Small (466MB) - Balanced speed/accuracy" }, - { name: "medium", label: "Medium (1.5GB) - Slower, more accurate" }, - { name: "large-v3", label: "Large (3GB) - Slowest, most accurate" }, + { + name: "small", + label: "Small", + size: "466MB", + description: "Balanced speed/accuracy", + }, + { + name: "medium", + label: "Medium", + size: "1.5GB", + description: "Slower, more accurate", + }, ]; const LANGUAGE_OPTIONS: LanguageOption[] = [ @@ -64,7 +82,21 @@ const LANGUAGE_OPTIONS: LanguageOption[] = [ { code: "zh", label: "Chinese" }, ]; -const DEFAULT_MODEL = "tiny"; +interface PositionOption { + value: string; + label: string; +} + +const POSITION_OPTIONS: PositionOption[] = [ + { value: "top-left", label: "Top Left" }, + { value: "top-center", label: "Top Center" }, + { value: "top-right", label: "Top Right" }, + { value: "bottom-left", label: "Bottom Left" }, + { value: "bottom-center", label: "Bottom Center" }, + { value: "bottom-right", label: "Bottom Right" }, +]; + +const DEFAULT_MODEL = "small"; const MODEL_FOLDER = "transcription_models"; // Custom flat button component since we can't import it @@ -184,21 +216,22 @@ export function CaptionsTab() { // Track container size changes const size = createElementSize(() => scrollContainerRef); - // Create a local store for caption settings to avoid direct project mutations const [captionSettings, setCaptionSettings] = createStore( project?.captions?.settings || { enabled: false, - font: "Arial", + font: "System Sans-Serif", size: 24, color: "#FFFFFF", backgroundColor: "#000000", backgroundOpacity: 80, - position: "bottom", + position: "bottom-center", bold: true, italic: false, outline: true, outlineColor: "#000000", exportWithSubtitles: false, + highlightColor: "#FFFF00", + fadeDuration: 0.15, }, ); @@ -293,22 +326,23 @@ export function CaptionsTab() { if (!project || !editorInstance) return; if (!project.captions) { - // Initialize captions with default settings setProject("captions", { segments: [], settings: { enabled: false, - font: "Arial", + font: "System Sans-Serif", size: 24, color: "#FFFFFF", backgroundColor: "#000000", backgroundOpacity: 80, - position: "bottom", + position: "bottom-center", bold: true, italic: false, outline: true, outlineColor: "#000000", exportWithSubtitles: false, + highlightColor: "#FFFF00", + fadeDuration: 0.15, }, }); } @@ -344,6 +378,13 @@ export function CaptionsTab() { setModelExists(await checkModelExists(selectedModel())); } + // Restore previously selected model + const savedModel = localStorage.getItem("selectedTranscriptionModel"); + if (savedModel && MODEL_OPTIONS.some((m) => m.name === savedModel)) { + setSelectedModel(savedModel); + setModelExists(await checkModelExists(savedModel)); + } + // Check if the video has audio if (editorInstance && editorInstance.recordings) { const hasAudioTrack = editorInstance.recordings.segments.some( @@ -384,6 +425,14 @@ export function CaptionsTab() { } }); + // Save selected model when it changes + createEffect(() => { + const model = selectedModel(); + if (model) { + localStorage.setItem("selectedTranscriptionModel", model); + } + }); + // Effect to update current caption based on playback time createEffect(() => { if (!project?.captions?.segments || editorState.playbackTime === undefined) @@ -606,165 +655,113 @@ export function CaptionsTab() {
- {/* Model Selection and Download Section */}
-
- - - options={MODEL_OPTIONS.filter((m) => - downloadedModels().includes(m.name), - ).map((m) => m.name)} - value={selectedModel()} - onChange={(value: string | null) => { - if (value) { - batch(() => { - setSelectedModel(value); - setModelExists(downloadedModels().includes(value)); - }); - } - }} - itemComponent={(props) => ( - - as={KSelect.Item} - item={props.item} - > - - { - MODEL_OPTIONS.find( - (m) => m.name === props.item.rawValue, - )?.label - } - - - )} - > - - class="flex-1 text-left truncate"> - {(state) => { - const model = MODEL_OPTIONS.find( - (m) => m.name === state.selectedOption(), - ); - return ( - {model?.label || "Select a model"} - ); - }} - - - - - - - - as={KSelect.Content} - class={topLeftAnimateClasses} - > - - class="max-h-48 overflow-y-auto" - as={KSelect.Listbox} - /> - - - -
-
- - options={MODEL_OPTIONS.map((m) => m.name)} - value={selectedModel()} - onChange={(value: string | null) => { - if (value) setSelectedModel(value); - }} - disabled={isDownloading()} - itemComponent={(props) => ( - - as={KSelect.Item} - item={props.item} - > - - { - MODEL_OPTIONS.find( - (m) => m.name === props.item.rawValue, - )?.label - } - {downloadedModels().includes(props.item.rawValue) - ? " (Downloaded)" - : ""} - - - )} +
+ + {(model) => { + const isDownloaded = () => + downloadedModels().includes(model.name); + const isSelected = () => + selectedModel() === model.name; + + return ( + + ); + }} + +
+
+ +
+ + + +
+
+
+ +
+ } > - - class="flex-1 text-left truncate"> - {(state) => { - const model = MODEL_OPTIONS.find( - (m) => m.name === state.selectedOption(), - ); - return ( - {model?.label || "Select a model"} - ); - }} - - - - - - - - as={KSelect.Content} - class={topLeftAnimateClasses} + + + +
- - - Download{" "} - { - MODEL_OPTIONS.find((m) => m.name === selectedModel()) - ?.label - } - - } - > -
-
-
-
-

- Downloading{" "} - { - MODEL_OPTIONS.find( - (m) => m.name === downloadingModel(), - )?.label - } - : {Math.round(downloadProgress())}% -

-
-
- {/* Language Selection */} options={LANGUAGE_OPTIONS.map((l) => l.code)} @@ -818,17 +815,6 @@ export function CaptionsTab() { - {/* Generate Captions Button */} - - - - {/* Font Settings */} }>
@@ -943,8 +929,8 @@ export function CaptionsTab() { {/* Position Settings */} }> - options={["top", "bottom"]} - value={captionSettings.position || "bottom"} + options={POSITION_OPTIONS.map((p) => p.value)} + value={captionSettings.position || "bottom-center"} onChange={(value) => { if (value === null) return; updateCaptionSetting("position", value); @@ -954,8 +940,12 @@ export function CaptionsTab() { as={KSelect.Item} item={props.item} > - - {props.item.rawValue} + + { + POSITION_OPTIONS.find( + (p) => p.value === props.item.rawValue, + )?.label + } )} @@ -963,8 +953,12 @@ export function CaptionsTab() { > {(state) => ( - - {state.selectedOption()} + + { + POSITION_OPTIONS.find( + (p) => p.value === state.selectedOption(), + )?.label + } )} @@ -985,6 +979,38 @@ export function CaptionsTab() { + {/* Highlight & Animation */} + }> +
+
+ Highlight Color + + updateCaptionSetting("highlightColor", value) + } + /> +
+
+ + Fade Duration (seconds) + + + updateCaptionSetting("fadeDuration", v[0] / 100) + } + minValue={0} + maxValue={50} + step={1} + /> + + {(captionSettings.fadeDuration || 0.15).toFixed(2)}s + +
+
+ + {/* Style Options */} }>
diff --git a/crates/project/src/configuration.rs b/crates/project/src/configuration.rs index 76ee3cfc23..c273110ad3 100644 --- a/crates/project/src/configuration.rs +++ b/crates/project/src/configuration.rs @@ -613,6 +613,65 @@ impl MaskSegment { } } +#[derive(Type, Serialize, Deserialize, Clone, Debug)] +#[serde(rename_all = "camelCase")] +pub struct TextSegment { + pub start: f64, + pub end: f64, + #[serde(default = "TextSegment::default_enabled")] + pub enabled: bool, + #[serde(default = "TextSegment::default_content")] + pub content: String, + #[serde(default = "TextSegment::default_center")] + pub center: XY, + #[serde(default = "TextSegment::default_size")] + pub size: XY, + #[serde(default = "TextSegment::default_font_family")] + pub font_family: String, + #[serde(default = "TextSegment::default_font_size")] + pub font_size: f32, + #[serde(default = "TextSegment::default_font_weight")] + pub font_weight: f32, + #[serde(default)] + pub italic: bool, + #[serde(default = "TextSegment::default_color")] + pub color: String, +} + +impl TextSegment { + fn default_enabled() -> bool { + true + } + + fn default_content() -> String { + "Text".to_string() + } + + fn default_center() -> XY { + XY::new(0.5, 0.5) + } + + fn default_size() -> XY { + XY::new(0.35, 0.2) + } + + fn default_font_family() -> String { + "sans-serif".to_string() + } + + fn default_font_size() -> f32 { + 48.0 + } + + fn default_font_weight() -> f32 { + 700.0 + } + + fn default_color() -> String { + "#ffffff".to_string() + } +} + #[derive(Type, Serialize, Deserialize, Clone, Copy, Debug, Default)] #[serde(rename_all = "camelCase")] pub enum SceneMode { @@ -640,6 +699,8 @@ pub struct TimelineConfiguration { pub scene_segments: Vec, #[serde(default)] pub mask_segments: Vec, + #[serde(default)] + pub text_segments: Vec, } impl TimelineConfiguration { @@ -666,6 +727,14 @@ impl TimelineConfiguration { pub const WALLPAPERS_PATH: &str = "assets/backgrounds/macOS"; +#[derive(Type, Serialize, Deserialize, Clone, Debug, Default)] +#[serde(rename_all = "camelCase")] +pub struct CaptionWord { + pub text: String, + pub start: f32, + pub end: f32, +} + #[derive(Type, Serialize, Deserialize, Clone, Debug, Default)] #[serde(rename_all = "camelCase")] pub struct CaptionSegment { @@ -673,6 +742,20 @@ pub struct CaptionSegment { pub start: f32, pub end: f32, pub text: String, + #[serde(default)] + pub words: Vec, +} + +#[derive(Type, Serialize, Deserialize, Clone, Copy, Debug, Default, PartialEq)] +#[serde(rename_all = "kebab-case")] +pub enum CaptionPosition { + TopLeft, + TopCenter, + TopRight, + #[default] + BottomLeft, + BottomCenter, + BottomRight, } #[derive(Type, Serialize, Deserialize, Clone, Debug)] @@ -686,6 +769,7 @@ pub struct CaptionSettings { pub background_color: String, #[serde(alias = "backgroundOpacity")] pub background_opacity: u32, + #[serde(default)] pub position: String, pub bold: bool, pub italic: bool, @@ -694,6 +778,26 @@ pub struct CaptionSettings { pub outline_color: String, #[serde(alias = "exportWithSubtitles")] pub export_with_subtitles: bool, + #[serde( + alias = "highlightColor", + default = "CaptionSettings::default_highlight_color" + )] + pub highlight_color: String, + #[serde( + alias = "fadeDuration", + default = "CaptionSettings::default_fade_duration" + )] + pub fade_duration: f32, +} + +impl CaptionSettings { + fn default_highlight_color() -> String { + "#FFFF00".to_string() + } + + fn default_fade_duration() -> f32 { + 0.15 + } } impl Default for CaptionSettings { @@ -705,12 +809,14 @@ impl Default for CaptionSettings { color: "#FFFFFF".to_string(), background_color: "#000000".to_string(), background_opacity: 80, - position: "bottom".to_string(), + position: "bottom-center".to_string(), bold: true, italic: false, outline: true, outline_color: "#000000".to_string(), export_with_subtitles: false, + highlight_color: Self::default_highlight_color(), + fade_duration: Self::default_fade_duration(), } } } @@ -878,6 +984,8 @@ pub struct ProjectConfiguration { pub clips: Vec, #[serde(default)] pub annotations: Vec, + #[serde(default, skip_serializing)] + pub hidden_text_segments: Vec, } fn camera_config_needs_migration(value: &Value) -> bool { diff --git a/crates/rendering/src/layers/captions.rs b/crates/rendering/src/layers/captions.rs index 5e1f6bcb85..fa16fa9edd 100644 --- a/crates/rendering/src/layers/captions.rs +++ b/crates/rendering/src/layers/captions.rs @@ -1,38 +1,42 @@ -#![allow(unused)] // TODO: This module is still being implemented - use bytemuck::{Pod, Zeroable}; use cap_project::XY; use glyphon::{ Attrs, Buffer, Cache, Color, Family, FontSystem, Metrics, Resolution, Shaping, SwashCache, - TextArea, TextAtlas, TextBounds, TextRenderer, Viewport, + TextArea, TextAtlas, TextBounds, TextRenderer, Viewport, Weight, }; -use log::{debug, info, warn}; +use log::{debug, warn}; use wgpu::{Device, Queue, util::DeviceExt}; use crate::{DecodedSegmentFrames, ProjectUniforms, RenderVideoConstants, parse_color_component}; -/// Represents a caption segment with timing and text +#[derive(Debug, Clone)] +pub struct CaptionWord { + pub text: String, + pub start: f32, + pub end: f32, +} + #[derive(Debug, Clone)] pub struct CaptionSegment { pub id: String, pub start: f32, pub end: f32, pub text: String, + pub words: Vec, } -/// Settings for caption rendering #[repr(C)] #[derive(Copy, Clone, Pod, Zeroable, Debug)] pub struct CaptionSettings { - pub enabled: u32, // 0 = disabled, 1 = enabled + pub enabled: u32, pub font_size: f32, pub color: [f32; 4], pub background_color: [f32; 4], - pub position: u32, // 0 = top, 1 = middle, 2 = bottom - pub outline: u32, // 0 = disabled, 1 = enabled + pub position: u32, + pub outline: u32, pub outline_color: [f32; 4], - pub font: u32, // 0 = SansSerif, 1 = Serif, 2 = Monospace - pub _padding: [f32; 1], // for alignment + pub font: u32, + pub _padding: [f32; 1], } impl Default for CaptionSettings { @@ -40,18 +44,55 @@ impl Default for CaptionSettings { Self { enabled: 1, font_size: 24.0, - color: [1.0, 1.0, 1.0, 1.0], // white - background_color: [0.0, 0.0, 0.0, 0.8], // 80% black - position: 2, // bottom - outline: 1, // enabled - outline_color: [0.0, 0.0, 0.0, 1.0], // black - font: 0, // SansSerif + color: [1.0, 1.0, 1.0, 1.0], + background_color: [0.0, 0.0, 0.0, 0.8], + position: 5, + outline: 1, + outline_color: [0.0, 0.0, 0.0, 1.0], + font: 0, _padding: [0.0], } } } -/// Caption layer that renders text using GPU +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum CaptionPosition { + TopLeft, + TopCenter, + TopRight, + BottomLeft, + BottomCenter, + BottomRight, +} + +impl CaptionPosition { + fn from_str(s: &str) -> Self { + match s { + "top-left" => Self::TopLeft, + "top-center" | "top" => Self::TopCenter, + "top-right" => Self::TopRight, + "bottom-left" => Self::BottomLeft, + "bottom-right" => Self::BottomRight, + _ => Self::BottomCenter, + } + } + + fn y_factor(&self) -> f32 { + match self { + Self::TopLeft | Self::TopCenter | Self::TopRight => 0.08, + Self::BottomLeft | Self::BottomCenter | Self::BottomRight => 0.85, + } + } + + fn x_alignment(&self) -> f32 { + match self { + Self::TopLeft | Self::BottomLeft => 0.05, + Self::TopCenter | Self::BottomCenter => 0.5, + Self::TopRight | Self::BottomRight => 0.95, + } + } +} + pub struct CaptionsLayer { settings_buffer: wgpu::Buffer, font_system: FontSystem, @@ -60,13 +101,13 @@ pub struct CaptionsLayer { text_renderer: TextRenderer, text_buffer: Buffer, current_text: Option, - current_segment_time: f32, + current_segment_start: f32, + current_segment_end: f32, viewport: Viewport, } impl CaptionsLayer { pub fn new(device: &Device, queue: &Queue) -> Self { - // Create default settings buffer let settings = CaptionSettings::default(); let settings_buffer = device.create_buffer_init(&wgpu::util::BufferInitDescriptor { label: Some("Caption Settings Buffer"), @@ -74,7 +115,6 @@ impl CaptionsLayer { usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST, }); - // Initialize glyphon text rendering components let font_system = FontSystem::new(); let swash_cache = SwashCache::new(); let cache = Cache::new(device); @@ -88,8 +128,7 @@ impl CaptionsLayer { None, ); - // Create an empty buffer with default metrics - let metrics = Metrics::new(24.0, 24.0 * 1.2); // Default font size and line height + let metrics = Metrics::new(24.0, 24.0 * 1.2); let text_buffer = Buffer::new_empty(metrics); Self { @@ -100,35 +139,35 @@ impl CaptionsLayer { text_renderer, text_buffer, current_text: None, - current_segment_time: 0.0, + current_segment_start: 0.0, + current_segment_end: 0.0, viewport, } } - /// Update the settings for caption rendering pub fn update_settings(&mut self, queue: &Queue, settings: CaptionSettings) { queue.write_buffer(&self.settings_buffer, 0, bytemuck::cast_slice(&[settings])); } - /// Update the current caption text and timing - pub fn update_caption(&mut self, text: Option, time: f32) { - debug!("Updating caption - Text: {text:?}, Time: {time}"); - if self.current_text != text { - if let Some(content) = &text { - info!("Setting new caption text: {content}"); - // Update the text buffer with new content - let metrics = Metrics::new(24.0, 24.0 * 1.2); - self.text_buffer = Buffer::new_empty(metrics); - self.text_buffer.set_text( - &mut self.font_system, - content, - &Attrs::new(), - Shaping::Advanced, - ); - } - self.current_text = text; + pub fn update_caption(&mut self, text: Option, start: f32, end: f32) { + debug!("Updating caption - Text: {text:?}, Start: {start}, End: {end}"); + self.current_text = text; + self.current_segment_start = start; + self.current_segment_end = end; + } + + fn calculate_fade_opacity(&self, current_time: f32, fade_duration: f32) -> f32 { + if fade_duration <= 0.0 { + return 1.0; } - self.current_segment_time = time; + + let time_from_start = current_time - self.current_segment_start; + let time_to_end = self.current_segment_end - current_time; + + let fade_in = (time_from_start / fade_duration).min(1.0); + let fade_out = (time_to_end / fade_duration).min(1.0); + + fade_in.min(fade_out).max(0.0) } pub fn prepare( @@ -138,182 +177,213 @@ impl CaptionsLayer { output_size: XY, constants: &RenderVideoConstants, ) { - // Render captions if there are any caption segments to display - if let Some(caption_data) = &uniforms.project.captions - && caption_data.settings.enabled - { - // Find the current caption for this time + if let Some(caption_data) = &uniforms.project.captions { + if !caption_data.settings.enabled { + return; + } + let current_time = segment_frames.segment_time; + let fade_duration = caption_data.settings.fade_duration; if let Some(current_caption) = find_caption_at_time_project(current_time, &caption_data.segments) { - // Get caption text and time for use in rendering let caption_text = current_caption.text.clone(); + let caption_words = current_caption.words.clone(); + self.update_caption( + Some(caption_text.clone()), + current_caption.start, + current_caption.end, + ); - // Create settings for the caption - let settings = CaptionSettings { - enabled: 1, - font_size: caption_data.settings.size as f32, - color: [ - parse_color_component(&caption_data.settings.color, 0), - parse_color_component(&caption_data.settings.color, 1), - parse_color_component(&caption_data.settings.color, 2), - 1.0, - ], - background_color: [ - parse_color_component(&caption_data.settings.background_color, 0), - parse_color_component(&caption_data.settings.background_color, 1), - parse_color_component(&caption_data.settings.background_color, 2), - caption_data.settings.background_opacity as f32 / 100.0, - ], - position: match caption_data.settings.position.as_str() { - "top" => 0, - "middle" => 1, - _ => 2, // default to bottom - }, - outline: if caption_data.settings.outline { 1 } else { 0 }, - outline_color: [ - parse_color_component(&caption_data.settings.outline_color, 0), - parse_color_component(&caption_data.settings.outline_color, 1), - parse_color_component(&caption_data.settings.outline_color, 2), - 1.0, - ], - font: match caption_data.settings.font.as_str() { - "System Serif" => 1, - "System Monospace" => 2, - _ => 0, // Default to SansSerif for "System Sans-Serif" and any other value - }, - _padding: [0.0], - }; - - self.update_caption(Some(caption_text), current_time); - - if settings.enabled == 0 { - return; - } - - if self.current_text.is_none() { - return; - } + let fade_opacity = self.calculate_fade_opacity(current_time, fade_duration); if let Some(text) = &self.current_text { let (width, height) = (output_size.x, output_size.y); - - // Access device and queue from the pipeline's constants let device = &constants.device; let queue = &constants.queue; - // Find caption position based on settings - let y_position = match settings.position { - 0 => height as f32 * 0.1, // top - 1 => height as f32 * 0.5, // middle - _ => height as f32 * 0.85, // bottom (default) - }; + let position = CaptionPosition::from_str(&caption_data.settings.position); + let y_position = height as f32 * position.y_factor(); - // Set up caption appearance - let color = Color::rgb( - (settings.color[0] * 255.0) as u8, - (settings.color[1] * 255.0) as u8, - (settings.color[2] * 255.0) as u8, - ); + let base_color = [ + parse_color_component(&caption_data.settings.color, 0), + parse_color_component(&caption_data.settings.color, 1), + parse_color_component(&caption_data.settings.color, 2), + ]; - // Get outline color if needed - let outline_color = Color::rgb( - (settings.outline_color[0] * 255.0) as u8, - (settings.outline_color[1] * 255.0) as u8, - (settings.outline_color[2] * 255.0) as u8, - ); + let highlight_color_rgb = [ + parse_color_component(&caption_data.settings.highlight_color, 0), + parse_color_component(&caption_data.settings.highlight_color, 1), + parse_color_component(&caption_data.settings.highlight_color, 2), + ]; + + let outline_color_rgb = [ + parse_color_component(&caption_data.settings.outline_color, 0), + parse_color_component(&caption_data.settings.outline_color, 1), + parse_color_component(&caption_data.settings.outline_color, 2), + ]; + + let _bg_opacity = + (caption_data.settings.background_opacity as f32 / 100.0) * fade_opacity; - // Calculate text bounds - let font_size = settings.font_size * (height as f32 / 1080.0); // Scale font size based on resolution - let metrics = Metrics::new(font_size, font_size * 1.2); // 1.2 line height + let font_size = caption_data.settings.size as f32 * (height as f32 / 1080.0); + let metrics = Metrics::new(font_size, font_size * 1.2); - // Create a new buffer with explicit size for this frame let mut updated_buffer = Buffer::new(&mut self.font_system, metrics); - // Set explicit width to enable proper text wrapping and centering - // Set width to 90% of screen width for better appearance let text_width = width as f32 * 0.9; updated_buffer.set_size(&mut self.font_system, Some(text_width), None); updated_buffer.set_wrap(&mut self.font_system, glyphon::Wrap::Word); - // Position text in the center horizontally - // The bounds dictate the rendering area + let x_offset = match position.x_alignment() { + x if x < 0.3 => (width as f32 * 0.05) as i32, + x if x > 0.7 => (width as f32 * 0.05) as i32, + _ => ((width as f32 - text_width) / 2.0) as i32, + }; + let bounds = TextBounds { - left: ((width as f32 - text_width) / 2.0) as i32, // Center the text horizontally + left: x_offset, top: y_position as i32, - right: ((width as f32 + text_width) / 2.0) as i32, // Center + width - bottom: (y_position + font_size * 4.0) as i32, // Increased height for better visibility + right: (x_offset as f32 + text_width) as i32, + bottom: (y_position + font_size * 4.0) as i32, }; - // Apply text styling directly when setting the text - // Create text attributes with or without outline - let font_family = match settings.font { - 0 => Family::SansSerif, - 1 => Family::Serif, - 2 => Family::Monospace, - _ => Family::SansSerif, // Default to SansSerif for any other value + let font_family = match caption_data.settings.font.as_str() { + "System Serif" => Family::Serif, + "System Monospace" => Family::Monospace, + _ => Family::SansSerif, }; - let attrs = Attrs::new().family(font_family).color(color); - // Apply text to buffer - updated_buffer.set_text(&mut self.font_system, text, &attrs, Shaping::Advanced); + let weight = if caption_data.settings.bold { + Weight::BOLD + } else { + Weight::NORMAL + }; - // Replace the existing buffer - self.text_buffer = updated_buffer; + if !caption_words.is_empty() { + let current_word_idx = caption_words + .iter() + .position(|w| current_time >= w.start && current_time < w.end); + + let mut rich_text: Vec<(&str, Attrs)> = Vec::new(); + let full_text = text.as_str(); + let mut last_end = 0usize; + + for (idx, word) in caption_words.iter().enumerate() { + if let Some(start_pos) = full_text[last_end..].find(&word.text) { + let abs_start = last_end + start_pos; + + if abs_start > last_end { + let space = &full_text[last_end..abs_start]; + rich_text.push(( + space, + Attrs::new().family(font_family).weight(weight).color( + Color::rgba( + (base_color[0] * 255.0) as u8, + (base_color[1] * 255.0) as u8, + (base_color[2] * 255.0) as u8, + (fade_opacity * 255.0) as u8, + ), + ), + )); + } + + let is_current = Some(idx) == current_word_idx; + let word_color = if is_current { + Color::rgba( + (highlight_color_rgb[0] * 255.0) as u8, + (highlight_color_rgb[1] * 255.0) as u8, + (highlight_color_rgb[2] * 255.0) as u8, + (fade_opacity * 255.0) as u8, + ) + } else { + Color::rgba( + (base_color[0] * 255.0) as u8, + (base_color[1] * 255.0) as u8, + (base_color[2] * 255.0) as u8, + (fade_opacity * 255.0) as u8, + ) + }; + + let word_end = abs_start + word.text.len(); + rich_text.push(( + &full_text[abs_start..word_end], + Attrs::new() + .family(font_family) + .weight(weight) + .color(word_color), + )); + last_end = word_end; + } + } - // Update the viewport with explicit resolution - self.viewport.update(queue, Resolution { width, height }); + if last_end < full_text.len() { + rich_text.push(( + &full_text[last_end..], + Attrs::new() + .family(font_family) + .weight(weight) + .color(Color::rgba( + (base_color[0] * 255.0) as u8, + (base_color[1] * 255.0) as u8, + (base_color[2] * 255.0) as u8, + (fade_opacity * 255.0) as u8, + )), + )); + } - // Background color - let bg_color = if settings.background_color[3] > 0.01 { - // Create a new text area with background color - Color::rgba( - (settings.background_color[0] * 255.0) as u8, - (settings.background_color[1] * 255.0) as u8, - (settings.background_color[2] * 255.0) as u8, - (settings.background_color[3] * 255.0) as u8, - ) + updated_buffer.set_rich_text( + &mut self.font_system, + rich_text, + &Attrs::new().family(font_family).weight(weight), + Shaping::Advanced, + None, + ); } else { - Color::rgba(0, 0, 0, 0) - }; + let color = Color::rgba( + (base_color[0] * 255.0) as u8, + (base_color[1] * 255.0) as u8, + (base_color[2] * 255.0) as u8, + (fade_opacity * 255.0) as u8, + ); + let attrs = Attrs::new().family(font_family).weight(weight).color(color); + updated_buffer.set_text( + &mut self.font_system, + text, + &attrs, + Shaping::Advanced, + ); + } + + self.text_buffer = updated_buffer; + self.viewport.update(queue, Resolution { width, height }); - // Prepare text areas for rendering let mut text_areas = Vec::new(); - // Add background if enabled - if settings.background_color[3] > 0.01 { - text_areas.push(TextArea { - buffer: &self.text_buffer, - left: bounds.left as f32, // Match the bounds left for positioning - top: y_position, - scale: 1.0, - bounds, - default_color: bg_color, - custom_glyphs: &[], - }); - } + let outline_color = Color::rgba( + (outline_color_rgb[0] * 255.0) as u8, + (outline_color_rgb[1] * 255.0) as u8, + (outline_color_rgb[2] * 255.0) as u8, + (fade_opacity * 255.0) as u8, + ); - // Add outline if enabled (by rendering the text multiple times with slight offsets in different positions) - if settings.outline == 1 { - info!("Rendering with outline"); - // Outline is created by drawing the text multiple times with small offsets in different directions + if caption_data.settings.outline { let outline_offsets = [ - (-1.0, -1.0), - (0.0, -1.0), - (1.0, -1.0), - (-1.0, 0.0), - (1.0, 0.0), - (-1.0, 1.0), - (0.0, 1.0), - (1.0, 1.0), + (-1.5, -1.5), + (0.0, -1.5), + (1.5, -1.5), + (-1.5, 0.0), + (1.5, 0.0), + (-1.5, 1.5), + (0.0, 1.5), + (1.5, 1.5), ]; for (offset_x, offset_y) in outline_offsets.iter() { text_areas.push(TextArea { buffer: &self.text_buffer, - left: bounds.left as f32 + offset_x, // Match bounds with small offset for outline + left: bounds.left as f32 + offset_x, top: y_position + offset_y, scale: 1.0, bounds, @@ -323,18 +393,23 @@ impl CaptionsLayer { } } - // Add main text (rendered last, on top of everything) + let default_color = Color::rgba( + (base_color[0] * 255.0) as u8, + (base_color[1] * 255.0) as u8, + (base_color[2] * 255.0) as u8, + (fade_opacity * 255.0) as u8, + ); + text_areas.push(TextArea { buffer: &self.text_buffer, - left: bounds.left as f32, // Match the bounds left for positioning + left: bounds.left as f32, top: y_position, scale: 1.0, bounds, - default_color: color, + default_color, custom_glyphs: &[], }); - // Prepare text rendering match self.text_renderer.prepare( device, queue, @@ -352,7 +427,6 @@ impl CaptionsLayer { } } - /// Render the current caption to the frame pub fn render<'a>(&'a self, pass: &mut wgpu::RenderPass<'a>) { match self .text_renderer @@ -364,15 +438,12 @@ impl CaptionsLayer { } } -/// Function to find the current caption segment based on playback time pub fn find_caption_at_time(time: f32, segments: &[CaptionSegment]) -> Option<&CaptionSegment> { segments .iter() .find(|segment| time >= segment.start && time < segment.end) } -// Adding a new version that accepts cap_project::CaptionSegment -/// Function to find the current caption segment from cap_project::CaptionSegment based on playback time pub fn find_caption_at_time_project( time: f32, segments: &[cap_project::CaptionSegment], @@ -385,5 +456,14 @@ pub fn find_caption_at_time_project( start: segment.start, end: segment.end, text: segment.text.clone(), + words: segment + .words + .iter() + .map(|w| CaptionWord { + text: w.text.clone(), + start: w.start, + end: w.end, + }) + .collect(), }) } From 5ca1132a84f3b856f6be3c12f919d65f0a4dcd99 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Mon, 1 Dec 2025 23:07:16 +0800 Subject: [PATCH 31/46] Add text overlay and timeline track to editor --- apps/desktop/src-tauri/src/recording.rs | 1 + .../src/routes/editor/ConfigSidebar.tsx | 386 +++++++++++++++++- .../src/routes/editor/ExportDialog.tsx | 207 ++++++---- apps/desktop/src/routes/editor/Player.tsx | 2 + .../desktop/src/routes/editor/TextOverlay.tsx | 372 +++++++++++++++++ .../src/routes/editor/Timeline/TextTrack.tsx | 371 +++++++++++++++++ .../src/routes/editor/Timeline/index.tsx | 75 ++-- apps/desktop/src/routes/editor/context.ts | 68 ++- apps/desktop/src/routes/editor/text.ts | 32 ++ apps/desktop/src/utils/export.ts | 27 +- apps/desktop/src/utils/tauri.ts | 38 +- crates/export/src/mp4.rs | 53 ++- crates/rendering/src/layers/mod.rs | 2 + crates/rendering/src/layers/text.rs | 162 ++++++++ crates/rendering/src/lib.rs | 36 +- crates/rendering/src/text.rs | 85 ++++ packages/ui-solid/src/auto-imports.d.ts | 1 + 17 files changed, 1748 insertions(+), 170 deletions(-) create mode 100644 apps/desktop/src/routes/editor/TextOverlay.tsx create mode 100644 apps/desktop/src/routes/editor/Timeline/TextTrack.tsx create mode 100644 apps/desktop/src/routes/editor/text.ts create mode 100644 crates/rendering/src/layers/text.rs create mode 100644 crates/rendering/src/text.rs diff --git a/apps/desktop/src-tauri/src/recording.rs b/apps/desktop/src-tauri/src/recording.rs index af9b13617d..2794eefc05 100644 --- a/apps/desktop/src-tauri/src/recording.rs +++ b/apps/desktop/src-tauri/src/recording.rs @@ -1706,6 +1706,7 @@ fn project_config_from_recording( zoom_segments, scene_segments: Vec::new(), mask_segments: Vec::new(), + text_segments: Vec::new(), }); config diff --git a/apps/desktop/src/routes/editor/ConfigSidebar.tsx b/apps/desktop/src/routes/editor/ConfigSidebar.tsx index e2b1d74cfe..e2e24332f5 100644 --- a/apps/desktop/src/routes/editor/ConfigSidebar.tsx +++ b/apps/desktop/src/routes/editor/ConfigSidebar.tsx @@ -58,10 +58,12 @@ import IconLucideGrid from "~icons/lucide/grid"; import IconLucideMonitor from "~icons/lucide/monitor"; import IconLucideMoon from "~icons/lucide/moon"; import IconLucideMove from "~icons/lucide/move"; +import IconLucidePalette from "~icons/lucide/palette"; import IconLucideRabbit from "~icons/lucide/rabbit"; import IconLucideScan from "~icons/lucide/scan"; import IconLucideSparkles from "~icons/lucide/sparkles"; import IconLucideTimer from "~icons/lucide/timer"; +import IconLucideType from "~icons/lucide/type"; import IconLucideWind from "~icons/lucide/wind"; import { CaptionsTab } from "./CaptionsTab"; import { type CornerRoundingType, useEditorContext } from "./context"; @@ -73,6 +75,7 @@ import { } from "./projectConfig"; import ShadowSettings from "./ShadowSettings"; import { TextInput } from "./TextInput"; +import type { TextSegment } from "./text"; import { ComingSoonTooltip, EditorButton, @@ -384,7 +387,7 @@ export function ConfigSidebar() { meta().type === "multiple" && (meta() as any).segments[0].cursor ), }, - window.FLAGS.captions && { + { id: "captions" as const, icon: IconCapMessageBubble, }, @@ -790,6 +793,73 @@ export function ConfigSidebar() { {(selection) => ( + { + const textSelection = selection(); + if (textSelection.type !== "text") return; + + const segments = textSelection.indices + .map((index) => ({ + index, + segment: project.timeline?.textSegments?.[index], + })) + .filter( + (item): item is { index: number; segment: TextSegment } => + item.segment !== undefined, + ); + + if (segments.length === 0) { + setEditorState("timeline", "selection", null); + return; + } + return { selection: textSelection, segments }; + })()} + > + {(value) => ( +
+
+
+ + setEditorState("timeline", "selection", null) + } + leftIcon={} + > + Done + + + {value().segments.length} text{" "} + {value().segments.length === 1 + ? "segment" + : "segments"}{" "} + selected + +
+ + projectActions.deleteTextSegments( + value().segments.map((s) => s.index), + ) + } + leftIcon={} + > + Delete + +
+ + {(item) => ( +
+ +
+ )} + +
+ )} + { const maskSelection = selection(); @@ -2380,6 +2450,320 @@ function CornerStyleSelect(props: { ); } +const TEXT_FONT_OPTIONS = [ + { value: "sans-serif", label: "Sans" }, + { value: "serif", label: "Serif" }, + { value: "monospace", label: "Monospace" }, + { value: "Inter", label: "Inter" }, + { value: "Geist Sans", label: "Geist Sans" }, +]; + +const normalizeHexInput = (value: string, fallback: string) => { + const trimmed = value.trim(); + const withHash = trimmed.startsWith("#") ? trimmed : `#${trimmed}`; + const shortMatch = /^#[0-9A-Fa-f]{3}$/.test(withHash); + if (shortMatch) { + const [, r, g, b] = withHash; + return `#${r}${r}${g}${g}${b}${b}`.toLowerCase(); + } + const fullMatch = /^#[0-9A-Fa-f]{6}$/.test(withHash); + if (fullMatch) return withHash.toLowerCase(); + return fallback; +}; + +function HexColorInput(props: { + value: string; + onChange: (value: string) => void; +}) { + const [text, setText] = createWritableMemo(() => props.value); + let prevColor = props.value; + let colorInput: HTMLInputElement | undefined; + + return ( +
+
+ ); +} + +function TextSegmentConfig(props: { + segmentIndex: number; + segment: TextSegment; +}) { + const { setProject } = useEditorContext(); + const clampNumber = (value: number, min: number, max: number) => + Math.min(Math.max(Number.isFinite(value) ? value : min), max); + const textFontOptions = createMemo(() => { + const font = props.segment.fontFamily; + if (!font) return TEXT_FONT_OPTIONS; + const exists = TEXT_FONT_OPTIONS.some((option) => option.value === font); + return exists + ? TEXT_FONT_OPTIONS + : [...TEXT_FONT_OPTIONS, { value: font, label: font }]; + }); + + const selectedFont = createMemo( + () => + textFontOptions().find( + (option) => option.value === props.segment.fontFamily, + ) ?? textFontOptions()[0], + ); + + const updateSegment = (fn: (segment: TextSegment) => void) => { + setProject( + "timeline", + "textSegments", + produce((segments) => { + const target = segments?.[props.segmentIndex]; + if (!target) return; + fn(target); + }), + ); + }; + + return ( +
+ } + > +
+