diff --git a/apps/desktop/src-tauri/src/general_settings.rs b/apps/desktop/src-tauri/src/general_settings.rs index 8ea2ce2fd9c..7b3d7456621 100644 --- a/apps/desktop/src-tauri/src/general_settings.rs +++ b/apps/desktop/src-tauri/src/general_settings.rs @@ -247,6 +247,10 @@ pub struct GeneralSettingsStore { pub camera_blur_disabled_by_crash: Option, #[serde(default)] pub update_channel: UpdateChannel, + #[serde(default)] + pub ocr_keep_screenshot: bool, + #[serde(default)] + pub ocr_show_notification: bool, } fn default_enable_native_camera_preview() -> bool { @@ -350,6 +354,8 @@ impl Default for GeneralSettingsStore { previous_recordings_paths: Vec::new(), camera_blur_disabled_by_crash: None, update_channel: UpdateChannel::Stable, + ocr_keep_screenshot: false, + ocr_show_notification: false, } } } diff --git a/apps/desktop/src-tauri/src/hotkeys.rs b/apps/desktop/src-tauri/src/hotkeys.rs index 2c9a6eed18c..cb8aea45b5c 100644 --- a/apps/desktop/src-tauri/src/hotkeys.rs +++ b/apps/desktop/src-tauri/src/hotkeys.rs @@ -66,6 +66,7 @@ pub enum HotkeyAction { ScreenshotDisplay, ScreenshotWindow, ScreenshotArea, + OcrArea, #[serde(other)] Other, } @@ -73,6 +74,8 @@ pub enum HotkeyAction { #[derive(Serialize, Deserialize, Type, Default)] pub struct HotkeysStore { hotkeys: HashMap, + #[serde(default)] + seeded: Vec, } impl HotkeysStore { @@ -208,15 +211,21 @@ pub fn init(app: &AppHandle) { ) .unwrap(); - let store = match HotkeysStore::get(app) { - Ok(Some(s)) => s, - Ok(None) => HotkeysStore::default(), + let mut store = match HotkeysStore::get(app) { + Ok(Some(s)) => Some(s), + Ok(None) => Some(HotkeysStore::default()), Err(e) => { eprintln!("Failed to load hotkeys store: {e}"); - HotkeysStore::default() + None } }; + if let Some(store) = &mut store { + seed_default_hotkeys(app, store); + } + + let store = store.unwrap_or_default(); + let global_shortcut = app.global_shortcut(); for hotkey in store.hotkeys.values() { global_shortcut.register(Shortcut::from(*hotkey)).ok(); @@ -225,6 +234,46 @@ pub fn init(app: &AppHandle) { app.manage(Mutex::new(store)); } +fn default_hotkey(action: HotkeyAction) -> Option { + match action { + HotkeyAction::OcrArea => Some(Hotkey { + code: Code::KeyT, + meta: cfg!(target_os = "macos"), + ctrl: !cfg!(target_os = "macos"), + alt: false, + shift: true, + }), + _ => None, + } +} + +fn seed_default_hotkeys(app: &AppHandle, store: &mut HotkeysStore) { + let mut changed = false; + + for action in [HotkeyAction::OcrArea] { + if store.seeded.contains(&action) || store.hotkeys.contains_key(&action) { + continue; + } + + let Some(hotkey) = default_hotkey(action) else { + continue; + }; + + if !store.hotkeys.values().any(|h| h == &hotkey) { + store.hotkeys.insert(action, hotkey); + } + store.seeded.push(action); + changed = true; + } + + if changed && let Ok(s) = app.store("store") { + s.set("hotkeys", serde_json::json!(&*store)); + if let Err(e) = s.save() { + eprintln!("Failed to save hotkeys store: {e}"); + } + } +} + async fn handle_hotkey(app: AppHandle, action: HotkeyAction) -> Result<(), String> { match action { HotkeyAction::StartStudioRecording => { @@ -332,6 +381,13 @@ async fn handle_hotkey(app: AppHandle, action: HotkeyAction) -> Result<(), Strin .emit(&app); Ok(()) } + HotkeyAction::OcrArea => { + let _ = RequestOpenRecordingPicker { + target_mode: Some(RecordingTargetMode::Ocr), + } + .emit(&app); + Ok(()) + } HotkeyAction::Other => Ok(()), } } diff --git a/apps/desktop/src-tauri/src/lib.rs b/apps/desktop/src-tauri/src/lib.rs index 6a57a662496..1cfc9739ac5 100644 --- a/apps/desktop/src-tauri/src/lib.rs +++ b/apps/desktop/src-tauri/src/lib.rs @@ -4853,6 +4853,7 @@ pub async fn run(recording_logging_handle: LoggingHandle, logs_dir: PathBuf) { recording::restart_recording, recording::delete_recording, recording::take_screenshot, + recording::capture_ocr_text, recording::import_current_desktop_background, recording::list_cameras, recording::get_camera_formats, @@ -4935,6 +4936,7 @@ pub async fn run(recording_logging_handle: LoggingHandle, logs_dir: PathBuf) { clip_thumbnails::get_clip_thumbnail, windows::position_traffic_lights, windows::set_theme, + windows::show_window_without_activating, windows::set_teleprompter_window_level, windows::set_teleprompter_window_opacity, windows::apply_macos_liquid_glass_background, diff --git a/apps/desktop/src-tauri/src/recording.rs b/apps/desktop/src-tauri/src/recording.rs index a12f05431d6..b9b121f099d 100644 --- a/apps/desktop/src-tauri/src/recording.rs +++ b/apps/desktop/src-tauri/src/recording.rs @@ -2803,26 +2803,71 @@ pub async fn take_screenshot( app: AppHandle, target: ScreenCaptureTarget, ) -> Result { - use crate::NewScreenshotAdded; - use crate::notifications; - use crate::{PendingScreenshot, PendingScreenshots}; - use cap_recording::screenshot::capture_screenshot; - use image::ImageEncoder; - use std::time::Instant; + let image = capture_screen_image(&app, target.clone()).await?; - let general_settings = GeneralSettingsStore::get(&app).ok().flatten(); - let general_settings = general_settings.as_ref(); + AppSounds::Notification.play(); - let project_name = format_project_name( - general_settings - .and_then(|s| s.default_project_name_template.clone()) - .as_deref(), - target.title().as_deref().unwrap_or("Unknown"), - target.kind_str(), - RecordingMode::Screenshot, - None, - ); + save_screenshot_project(&app, image, &target, true) +} + +#[tauri::command(async)] +#[specta::specta] +#[tracing::instrument(name = "capture_ocr_text", skip(app))] +pub async fn capture_ocr_text( + app: AppHandle, + target: ScreenCaptureTarget, +) -> Result { + use tauri_plugin_clipboard_manager::ClipboardExt; + + let settings = GeneralSettingsStore::get(&app) + .ok() + .flatten() + .unwrap_or_default(); + + let image = capture_screen_image(&app, target.clone()).await?; + + let text = crate::screenshot_editor::recognize_text_from_dynamic_image(&image).await?; + let text = text.trim().to_string(); + + if text.is_empty() { + return Err("No text was found in the selected area".to_string()); + } + + app.clipboard() + .write_text(text.clone()) + .map_err(|e| format!("Failed to copy text to clipboard: {e}"))?; + + AppSounds::Notification.play(); + + if settings.ocr_keep_screenshot + && let Err(e) = save_screenshot_project(&app, image, &target, false) + { + error!("Failed to save OCR screenshot: {e}"); + } + + if settings.enable_notifications && settings.ocr_show_notification { + use tauri_plugin_notification::NotificationExt; + + let preview: String = text.chars().take(120).collect(); + app.notification() + .builder() + .title("Text copied to clipboard") + .body(preview) + .show() + .ok(); + } + + Ok(text) +} +async fn capture_screen_image( + app: &AppHandle, + target: ScreenCaptureTarget, +) -> Result { + use crate::windows::show_overlay; + use cap_recording::screenshot::capture_screenshot; + + let mut hidden_windows = Vec::new(); let mut hid_any = false; for (label, window) in app.webview_windows() { if let Ok(id) = CapWindowId::from_str(&label) @@ -2835,8 +2880,18 @@ pub async fn take_screenshot( | CapWindowId::RecordingsOverlay ) { + let was_visible = window.is_visible().unwrap_or(false); hide_overlay(&window); hid_any = true; + // The target-select overlay's lifecycle is owned by the frontend, + // which hides it before invoking a capture and closes or restores + // it afterwards; re-showing it here would fight that. The occluder + // must keep ignoring cursor events, so it is re-shown without the + // show_overlay cursor-event reset. + if was_visible && !matches!(id, CapWindowId::TargetSelectOverlay { .. }) { + let ignores_cursor = matches!(id, CapWindowId::WindowCaptureOccluder { .. }); + hidden_windows.push((window, ignores_cursor)); + } } } @@ -2844,13 +2899,51 @@ pub async fn take_screenshot( tokio::time::sleep(std::time::Duration::from_millis(150)).await; } - let automation_target = target.clone(); - - let image = capture_screenshot(target) + let result = capture_screenshot(target) .await - .map_err(|e| format!("Failed to capture screenshot: {e}"))?; + .map_err(|e| format!("Failed to capture screenshot: {e}")); - AppSounds::Notification.play(); + for (window, ignores_cursor) in hidden_windows { + if ignores_cursor { + let _ = window.show(); + } else { + show_overlay(&window); + } + } + + result +} + +/// `run_side_effects` controls whether screenshot automations and the +/// "screenshot saved" notification fire once the project is written. OCR +/// captures skip them: an automation like copy-to-clipboard would overwrite +/// the text that was just copied. +fn save_screenshot_project( + app: &AppHandle, + image: image::DynamicImage, + target: &ScreenCaptureTarget, + run_side_effects: bool, +) -> Result { + use crate::NewScreenshotAdded; + use crate::notifications; + use crate::{PendingScreenshot, PendingScreenshots}; + use image::ImageEncoder; + use std::time::Instant; + + let general_settings = GeneralSettingsStore::get(app).ok().flatten(); + let general_settings = general_settings.as_ref(); + + let project_name = format_project_name( + general_settings + .and_then(|s| s.default_project_name_template.clone()) + .as_deref(), + target.title().as_deref().unwrap_or("Unknown"), + target.kind_str(), + RecordingMode::Screenshot, + None, + ); + + let automation_target = target.clone(); let image_width = image.width(); let image_height = image.height(); @@ -2975,16 +3068,18 @@ pub async fn take_screenshot( } .emit(&app_handle); - crate::automation::run_screenshot_automations( - app_handle.clone(), - image_path_for_emit.clone(), - &automation_target, - ); + if run_side_effects { + crate::automation::run_screenshot_automations( + app_handle.clone(), + image_path_for_emit.clone(), + &automation_target, + ); - notifications::send_notification( - &app_handle, - notifications::NotificationType::ScreenshotSaved, - ); + notifications::send_notification( + &app_handle, + notifications::NotificationType::ScreenshotSaved, + ); + } } Ok(Err(e)) => { error!("Failed to encode PNG: {e}"); diff --git a/apps/desktop/src-tauri/src/recording_settings.rs b/apps/desktop/src-tauri/src/recording_settings.rs index c8935289818..d6c9dc4c7c0 100644 --- a/apps/desktop/src-tauri/src/recording_settings.rs +++ b/apps/desktop/src-tauri/src/recording_settings.rs @@ -19,6 +19,7 @@ pub enum RecordingTargetMode { Window, Area, Camera, + Ocr, } #[derive(serde::Serialize, serde::Deserialize, specta::Type, Debug, Clone, Default)] diff --git a/apps/desktop/src-tauri/src/screenshot_editor.rs b/apps/desktop/src-tauri/src/screenshot_editor.rs index 222da938b07..47d62302b6a 100644 --- a/apps/desktop/src-tauri/src/screenshot_editor.rs +++ b/apps/desktop/src-tauri/src/screenshot_editor.rs @@ -986,6 +986,12 @@ pub async fn recognize_screenshot_text( pub async fn recognize_text_from_image_path(path: &std::path::Path) -> Result { let dynamic = image::open(path).map_err(|e| format!("Failed to open image for OCR: {e}"))?; + recognize_text_from_dynamic_image(&dynamic).await +} + +pub async fn recognize_text_from_dynamic_image( + dynamic: &image::DynamicImage, +) -> Result { let rgba = dynamic.to_rgba8(); let width = rgba.width(); let height = rgba.height(); diff --git a/apps/desktop/src-tauri/src/windows.rs b/apps/desktop/src-tauri/src/windows.rs index 4dec972b3ac..e1e3b16e81c 100644 --- a/apps/desktop/src-tauri/src/windows.rs +++ b/apps/desktop/src-tauri/src/windows.rs @@ -130,6 +130,27 @@ pub fn show_overlay(window: &WebviewWindow) { let _ = window.show(); } +/// Shows the calling window without activating it, so the foreground app +/// keeps keyboard focus. +#[tauri::command] +#[specta::specta] +pub fn show_window_without_activating(window: WebviewWindow) { + #[cfg(windows)] + if let Ok(hwnd) = window.hwnd() { + use ::windows::Win32::UI::WindowsAndMessaging::{SW_SHOWNOACTIVATE, ShowWindow}; + + unsafe { + let _ = ShowWindow( + ::windows::Win32::Foundation::HWND(hwnd.0), + SW_SHOWNOACTIVATE, + ); + } + return; + } + + let _ = window.show(); +} + fn emit_app_event(app: &AppHandle, event: E) where E: Event + serde::Serialize + Clone, @@ -1707,6 +1728,7 @@ impl ShowCapWindow { Some(RecordingTargetMode::Window) => "&targetMode=window", Some(RecordingTargetMode::Area) => "&targetMode=area", Some(RecordingTargetMode::Camera) => "&targetMode=camera", + Some(RecordingTargetMode::Ocr) => "&targetMode=ocr", None => "", }; diff --git a/apps/desktop/src/routes/(window-chrome)/new-main/index.tsx b/apps/desktop/src/routes/(window-chrome)/new-main/index.tsx index cc270e2b049..96a06e1e3b9 100644 --- a/apps/desktop/src/routes/(window-chrome)/new-main/index.tsx +++ b/apps/desktop/src/routes/(window-chrome)/new-main/index.tsx @@ -2035,11 +2035,18 @@ function Page() { const dismissalReveals = dismissal === "cancelled" || dismissal === "screenshot" || + dismissal === "ocr" || dismissal === "recordingInstant"; if (shouldRevealMainWindow && dismissalReveals) { - const currentWindow = getCurrentWindow(); - void currentWindow.show(); - void currentWindow.setFocus(); + // An OCR capture is a background copy-to-clipboard gesture; bring + // the window back without stealing focus from the user's app. + if (dismissal === "ocr") { + void commands.showWindowWithoutActivating(); + } else { + const currentWindow = getCurrentWindow(); + void currentWindow.show(); + void currentWindow.setFocus(); + } } } }); diff --git a/apps/desktop/src/routes/(window-chrome)/settings/general.tsx b/apps/desktop/src/routes/(window-chrome)/settings/general.tsx index 46e7b205259..fade02a0acf 100644 --- a/apps/desktop/src/routes/(window-chrome)/settings/general.tsx +++ b/apps/desktop/src/routes/(window-chrome)/settings/general.tsx @@ -572,6 +572,35 @@ function Inner(props: { } /> +
+ + handleChange("ocrKeepScreenshot", value)} + /> + { + if (value) { + const permissionGranted = await isPermissionGranted(); + if (!permissionGranted) { + const permission = await requestPermission(); + if (permission !== "granted") return; + } + } + handleChange("ocrShowNotification", value); + }} + /> + +
+
(); const [options, setOptions] = useOptions(); const [areaSelectionPreferences, setAreaSelectionPreferences] = makePersisted( @@ -293,7 +293,16 @@ function Inner() { }); createEffect( - (prevMode: "display" | "window" | "area" | "camera" | null | undefined) => { + ( + prevMode: + | "display" + | "window" + | "area" + | "camera" + | "ocr" + | null + | undefined, + ) => { const mode = options.targetMode ?? null; if (prevMode === "area" && mode !== "area") { const target = pendingAreaTarget(); @@ -779,8 +788,16 @@ function Inner() { ); }} - + {(displayId) => { + const isOcr = () => options.targetMode === "ocr"; + const isImmediateCapture = () => + isOcr() || options.mode === "screenshot"; let controlsEl: HTMLDivElement | undefined; let cropperRef: CropperRef | undefined; @@ -812,19 +829,19 @@ function Inner() { const [screenshotSnapToRatio, setScreenshotSnapToRatio] = createSignal(true); const minSize = () => - options.mode === "screenshot" ? MIN_SCREENSHOT_SIZE : MIN_SIZE; + isImmediateCapture() ? MIN_SCREENSHOT_SIZE : MIN_SIZE; const currentAspect = () => - options.mode === "screenshot" + isImmediateCapture() ? screenshotAspect() : areaSelectionPreferences.aspectRatio; const currentSnapToRatio = () => - options.mode === "screenshot" + isImmediateCapture() ? screenshotSnapToRatio() : areaSelectionPreferences.snapToRatio; const effectiveInitialAreaBounds = createMemo(() => { const explicitBounds = initialAreaBounds(); if (explicitBounds) return explicitBounds; - if (options.mode === "screenshot") return undefined; + if (isImmediateCapture()) return undefined; return getLockedAreaBounds( areaSelectionPreferences, displayId(), @@ -855,7 +872,7 @@ function Inner() { }); const isSelectionLocked = createMemo( () => - options.mode !== "screenshot" && + !isImmediateCapture() && getLockedAreaBounds( areaSelectionPreferences, displayId(), @@ -864,7 +881,7 @@ function Inner() { ); function setAspect(aspect: Ratio | null) { - if (options.mode === "screenshot") { + if (isImmediateCapture()) { setScreenshotAspect(aspect); return; } @@ -875,7 +892,7 @@ function Inner() { } function setSnapToRatio(enabled: boolean) { - if (options.mode === "screenshot") { + if (isImmediateCapture()) { setScreenshotSnapToRatio(enabled); return; } @@ -884,7 +901,7 @@ function Inner() { function persistLockedSelection() { if ( - options.mode === "screenshot" || + isImmediateCapture() || !areaSelectionPreferences.locked || areaSelectionPreferences.screenId !== displayId() || !isValid() @@ -920,7 +937,7 @@ function Inner() { } if ( isInteracting() || - options.mode === "screenshot" || + isImmediateCapture() || !areaSelectionPreferences.locked || areaSelectionPreferences.screenId !== displayId() || !isValid() || @@ -989,7 +1006,7 @@ function Inner() { }); createEffect(async () => { - if (options.mode === "screenshot") return; + if (isImmediateCapture()) return; const bounds = crop(); const interacting = isInteracting(); const displayInfo = areaDisplayInfo.data; @@ -1252,6 +1269,51 @@ function Inner() { if (was && !interacting) { persistLockedSelection(); + if (isOcr() && isValid()) { + const cropBounds = crop(); + const target: ScreenCaptureTarget = { + variant: "area", + screen: displayId(), + bounds: { + position: { + x: cropBounds.x, + y: cropBounds.y, + }, + size: { + width: cropBounds.width, + height: cropBounds.height, + }, + }, + }; + + const overlayWindows = (await WebviewWindow.getAll()).filter( + (win) => win.label.startsWith("target-select-overlay-"), + ); + + try { + for (const win of overlayWindows) { + await win.setIgnoreCursorEvents(true); + await win.hide(); + } + await new Promise((resolve) => setTimeout(resolve, 50)); + + await commands.captureOcrText(target); + setOptions({ + targetMode: null, + targetModeDismissal: "ocr", + }); + await commands.closeTargetSelectOverlays(); + } catch (e) { + for (const win of overlayWindows) { + await win.setIgnoreCursorEvents(false); + await win.show(); + } + const message = e instanceof Error ? e.message : String(e); + toast.error(`Failed to copy text: ${message}`); + console.error("Failed to copy text", e); + } + return; + } if (options.mode === "screenshot" && isValid()) { const cropBounds = crop(); const displayInfo = areaDisplayInfo.data; @@ -1330,7 +1392,9 @@ function Inner() {
{isValid() ? `${Math.round(crop().width)} × ${Math.round(crop().height)}` - : "Draw an area"} + : isOcr() + ? "Draw an area to copy text" + : "Draw an area"}
@@ -1402,7 +1466,7 @@ function Inner() { > - +