Desktop: Add support for UI scaling (#3310)

* desktop support ui scaling

* fix some warnings

* use browser zoom if needed

* fix infinite footprint size

* fix web canvas scale

* always set zoom

* use only zoom for scaling

* prevent user zoom

* remove mouse position scaling

Author: timon-schelling

desktop/src/app.rs

@@ -8,6 +8,7 @@ use std::thread;
 use std::time::Duration;
 use std::time::Instant;
 use winit::application::ApplicationHandler;
+use winit::dpi::PhysicalSize;
 use winit::event::WindowEvent;
 use winit::event_loop::ActiveEventLoop;
 use winit::event_loop::ControlFlow;
@@ -25,8 +26,9 @@ use crate::wrapper::{DesktopWrapper, NodeGraphExecutionResult, WgpuContext, seri
 pub(crate) struct App {
 	cef_context: Box<dyn cef::CefContext>,
 	window: Option<Window>,
+	window_scale: f64,
 	cef_schedule: Option<Instant>,
-	cef_window_size_sender: Sender<cef::WindowSize>,
+	cef_view_info_sender: Sender<cef::ViewInfoUpdate>,
 	graphics_state: Option<GraphicsState>,
 	wgpu_context: WgpuContext,
 	app_event_receiver: Receiver<AppEvent>,
@@ -44,7 +46,7 @@ pub(crate) struct App {
 impl App {
 	pub(crate) fn new(
 		cef_context: Box<dyn cef::CefContext>,
-		window_size_sender: Sender<cef::WindowSize>,
+		cef_view_info_sender: Sender<cef::ViewInfoUpdate>,
 		wgpu_context: WgpuContext,
 		app_event_receiver: Receiver<AppEvent>,
 		app_event_scheduler: AppEventScheduler,
@@ -66,9 +68,10 @@ impl App {
 		Self {
 			cef_context,
 			window: None,
+			window_scale: 1.0,
 			cef_schedule: Some(Instant::now()),
 			graphics_state: None,
-			cef_window_size_sender: window_size_sender,
+			cef_view_info_sender,
 			wgpu_context,
 			app_event_receiver,
 			app_event_scheduler,
@@ -147,19 +150,19 @@ impl App {
 					}
 				});
 			}
-			DesktopFrontendMessage::UpdateViewportBounds { x, y, width, height } => {
+			DesktopFrontendMessage::UpdateViewportPhysicalBounds { x, y, width, height } => {
 				if let Some(graphics_state) = &mut self.graphics_state
 					&& let Some(window) = &self.window
 				{
 					let window_size = window.surface_size();
 
-					let viewport_offset_x = x / window_size.width as f32;
-					let viewport_offset_y = y / window_size.height as f32;
-					graphics_state.set_viewport_offset([viewport_offset_x, viewport_offset_y]);
+					let viewport_offset_x = x / window_size.width as f64;
+					let viewport_offset_y = y / window_size.height as f64;
+					graphics_state.set_viewport_offset([viewport_offset_x as f32, viewport_offset_y as f32]);
 
-					let viewport_scale_x = if width != 0.0 { window_size.width as f32 / width } else { 1.0 };
-					let viewport_scale_y = if height != 0.0 { window_size.height as f32 / height } else { 1.0 };
-					graphics_state.set_viewport_scale([viewport_scale_x, viewport_scale_y]);
+					let viewport_scale_x = if width != 0.0 { window_size.width as f64 / width } else { 1.0 };
+					let viewport_scale_y = if height != 0.0 { window_size.height as f64 / height } else { 1.0 };
+					graphics_state.set_viewport_scale([viewport_scale_x as f32, viewport_scale_y as f32]);
 				}
 			}
 			DesktopFrontendMessage::UpdateOverlays(scene) => {
@@ -179,7 +182,7 @@ impl App {
 			}
 			DesktopFrontendMessage::DragWindow => {
 				if let Some(window) = &self.window {
-					let _ = window.start_drag();
+					window.start_drag();
 				}
 			}
 			DesktopFrontendMessage::CloseWindow => {
@@ -352,14 +355,17 @@ impl App {
 impl ApplicationHandler for App {
 	fn can_create_surfaces(&mut self, event_loop: &dyn ActiveEventLoop) {
 		let window = Window::new(event_loop, self.app_event_scheduler.clone());
+
+		self.window_scale = window.scale_factor();
+		let _ = self.cef_view_info_sender.send(cef::ViewInfoUpdate::Scale(self.window_scale));
+		self.cef_context.notify_view_info_changed();
+
 		self.window = Some(window);
 
 		let graphics_state = GraphicsState::new(self.window.as_ref().unwrap(), self.wgpu_context.clone());
 
 		self.graphics_state = Some(graphics_state);
 
-		tracing::info!("Winit window created and ready");
-
 		self.desktop_wrapper.init(self.wgpu_context.clone());
 
 		#[cfg(target_os = "windows")]
@@ -384,14 +390,22 @@ impl ApplicationHandler for App {
 			WindowEvent::CloseRequested => {
 				self.app_event_scheduler.schedule(AppEvent::CloseWindow);
 			}
-			WindowEvent::SurfaceResized(size) => {
-				let _ = self.cef_window_size_sender.send(size.into());
-				self.cef_context.notify_of_resize();
+			WindowEvent::SurfaceResized(PhysicalSize { width, height }) => {
+				let _ = self.cef_view_info_sender.send(cef::ViewInfoUpdate::Size {
+					width: width as usize,
+					height: height as usize,
+				});
+				self.cef_context.notify_view_info_changed();
 				if let Some(window) = &self.window {
 					let maximized = window.is_maximized();
 					self.app_event_scheduler.schedule(AppEvent::DesktopWrapperMessage(DesktopWrapperMessage::UpdateMaximized { maximized }));
 				}
 			}
+			WindowEvent::ScaleFactorChanged { scale_factor, .. } => {
+				self.window_scale = scale_factor;
+				let _ = self.cef_view_info_sender.send(cef::ViewInfoUpdate::Scale(self.window_scale));
+				self.cef_context.notify_view_info_changed();
+			}
 			WindowEvent::RedrawRequested => {
 				let Some(ref mut graphics_state) = self.graphics_state else { return };
 				// Only rerender once we have a new UI texture to display

desktop/src/cef.rs

@@ -38,8 +38,8 @@ use texture_import::SharedTextureHandle;
 
 pub(crate) use context::{CefContext, CefContextBuilder, InitError};
 
-pub(crate) trait CefEventHandler: Clone + Send + Sync + 'static {
-	fn window_size(&self) -> WindowSize;
+pub(crate) trait CefEventHandler: Send + Sync + 'static {
+	fn view_info(&self) -> ViewInfo;
 	fn draw<'a>(&self, frame_buffer: FrameBufferRef<'a>);
 	#[cfg(feature = "accelerated_paint")]
 	fn draw_gpu(&self, shared_texture: SharedTextureHandle);
@@ -50,24 +50,54 @@ pub(crate) trait CefEventHandler: Clone + Send + Sync + 'static {
 	fn schedule_cef_message_loop_work(&self, scheduled_time: Instant);
 	fn initialized_web_communication(&self);
 	fn receive_web_message(&self, message: &[u8]);
+	fn duplicate(&self) -> Self
+	where
+		Self: Sized;
 }
 
 #[derive(Clone, Copy)]
-pub(crate) struct WindowSize {
-	pub(crate) width: usize,
-	pub(crate) height: usize,
+pub(crate) struct ViewInfo {
+	width: usize,
+	height: usize,
+	scale: f64,
 }
-impl WindowSize {
-	pub(crate) fn new(width: usize, height: usize) -> Self {
-		Self { width, height }
+impl ViewInfo {
+	pub(crate) fn new() -> Self {
+		Self { width: 1, height: 1, scale: 1. }
+	}
+	pub(crate) fn apply_update(&mut self, update: ViewInfoUpdate) {
+		match update {
+			ViewInfoUpdate::Size { width, height } if width > 0 && height > 0 => {
+				self.width = width;
+				self.height = height;
+			}
+			ViewInfoUpdate::Scale(scale) if scale > 0. => {
+				self.scale = scale;
+			}
+			_ => {}
+		}
+	}
+	pub(crate) fn zoom(&self) -> f64 {
+		self.scale.ln() / 1.2_f64.ln()
+	}
+	pub(crate) fn width(&self) -> usize {
+		self.width
+	}
+	pub(crate) fn height(&self) -> usize {
+		self.height
 	}
 }
-impl From<winit::dpi::PhysicalSize<u32>> for WindowSize {
-	fn from(size: winit::dpi::PhysicalSize<u32>) -> Self {
-		Self::new(size.width as usize, size.height as usize)
+impl Default for ViewInfo {
+	fn default() -> Self {
+		Self::new()
 	}
 }
 
+pub(crate) enum ViewInfoUpdate {
+	Size { width: usize, height: usize },
+	Scale(f64),
+}
+
 #[derive(Clone)]
 pub(crate) struct Resource {
 	pub(crate) reader: ResourceReader,
@@ -89,34 +119,33 @@ impl Read for ResourceReader {
 	}
 }
 
-#[derive(Clone)]
 pub(crate) struct CefHandler {
 	wgpu_context: WgpuContext,
 	app_event_scheduler: AppEventScheduler,
-	window_size_receiver: Arc<Mutex<WindowSizeReceiver>>,
+	view_info_receiver: Arc<Mutex<ViewInfoReceiver>>,
 }
 
 impl CefHandler {
-	pub(crate) fn new(wgpu_context: WgpuContext, app_event_scheduler: AppEventScheduler, window_size_receiver: Receiver<WindowSize>) -> Self {
+	pub(crate) fn new(wgpu_context: WgpuContext, app_event_scheduler: AppEventScheduler, view_info_receiver: Receiver<ViewInfoUpdate>) -> Self {
 		Self {
 			wgpu_context,
 			app_event_scheduler,
-			window_size_receiver: Arc::new(Mutex::new(WindowSizeReceiver::new(window_size_receiver))),
+			view_info_receiver: Arc::new(Mutex::new(ViewInfoReceiver::new(view_info_receiver))),
 		}
 	}
 }
 
 impl CefEventHandler for CefHandler {
-	fn window_size(&self) -> WindowSize {
-		let Ok(mut guard) = self.window_size_receiver.lock() else {
-			tracing::error!("Failed to lock window_size_receiver");
-			return WindowSize::new(1, 1);
+	fn view_info(&self) -> ViewInfo {
+		let Ok(mut guard) = self.view_info_receiver.lock() else {
+			tracing::error!("Failed to lock view_info_receiver");
+			return ViewInfo::new();
 		};
-		let WindowSizeReceiver { receiver, window_size } = &mut *guard;
-		for new_window_size in receiver.try_iter() {
-			*window_size = new_window_size;
+		let ViewInfoReceiver { receiver, view_info } = &mut *guard;
+		for update in receiver.try_iter() {
+			view_info.apply_update(update);
 		}
-		*window_size
+		*view_info
 	}
 	fn draw<'a>(&self, frame_buffer: FrameBufferRef<'a>) {
 		let width = frame_buffer.width() as u32;
@@ -244,17 +273,25 @@ impl CefEventHandler for CefHandler {
 		};
 		self.app_event_scheduler.schedule(AppEvent::DesktopWrapperMessage(desktop_wrapper_message));
 	}
-}
 
-struct WindowSizeReceiver {
-	window_size: WindowSize,
-	receiver: Receiver<WindowSize>,
-}
-impl WindowSizeReceiver {
-	fn new(window_size_receiver: Receiver<WindowSize>) -> Self {
+	fn duplicate(&self) -> Self
+	where
+		Self: Sized,
+	{
 		Self {
-			window_size: WindowSize { width: 1, height: 1 },
-			receiver: window_size_receiver,
+			wgpu_context: self.wgpu_context.clone(),
+			app_event_scheduler: self.app_event_scheduler.clone(),
+			view_info_receiver: self.view_info_receiver.clone(),
 		}
 	}
 }
+
+struct ViewInfoReceiver {
+	view_info: ViewInfo,
+	receiver: Receiver<ViewInfoUpdate>,
+}
+impl ViewInfoReceiver {
+	fn new(receiver: Receiver<ViewInfoUpdate>) -> Self {
+		Self { view_info: ViewInfo::new(), receiver }
+	}
+}

desktop/src/cef/context.rs

@@ -10,7 +10,7 @@ pub(crate) trait CefContext {
 
 	fn handle_window_event(&mut self, event: &winit::event::WindowEvent);
 
-	fn notify_of_resize(&self);
+	fn notify_view_info_changed(&self);
 
 	fn send_web_message(&self, message: Vec<u8>);
 }

desktop/src/cef/context/builder.rs

@@ -3,7 +3,7 @@ use std::path::{Path, PathBuf};
 use cef::args::Args;
 use cef::sys::{CEF_API_VERSION_LAST, cef_resultcode_t};
 use cef::{
-	App, BrowserSettings, CefString, Client, DictionaryValue, ImplCommandLine, ImplRequestContext, RenderHandler, RequestContextSettings, SchemeHandlerFactory, Settings, WindowInfo, api_hash,
+	App, BrowserSettings, CefString, Client, DictionaryValue, ImplCommandLine, ImplRequestContext, RequestContextSettings, SchemeHandlerFactory, Settings, WindowInfo, api_hash,
 	browser_host_create_browser_sync, execute_process,
 };
 
@@ -13,7 +13,7 @@ use crate::cef::CefEventHandler;
 use crate::cef::consts::{RESOURCE_DOMAIN, RESOURCE_SCHEME};
 use crate::cef::dirs::create_instance_dir;
 use crate::cef::input::InputState;
-use crate::cef::internal::{BrowserProcessAppImpl, BrowserProcessClientImpl, RenderHandlerImpl, RenderProcessAppImpl, SchemeHandlerFactoryImpl};
+use crate::cef::internal::{BrowserProcessAppImpl, BrowserProcessClientImpl, RenderProcessAppImpl, SchemeHandlerFactoryImpl};
 
 pub(crate) struct CefContextBuilder<H: CefEventHandler> {
 	pub(crate) args: Args,
@@ -131,7 +131,7 @@ impl<H: CefEventHandler> CefContextBuilder<H> {
 
 	fn initialize_inner(self, event_handler: &H, settings: Settings) -> Result<(), InitError> {
 		// Attention! Wrapping this in an extra App is necessary, otherwise the program still compiles but segfaults
-		let mut cef_app = App::new(BrowserProcessAppImpl::new(event_handler.clone()));
+		let mut cef_app = App::new(BrowserProcessAppImpl::new(event_handler.duplicate()));
 
 		let result = cef::initialize(Some(self.args.as_main_args()), Some(&settings), Some(&mut cef_app), std::ptr::null_mut());
 		if result != 1 {
@@ -146,8 +146,7 @@ impl<H: CefEventHandler> CefContextBuilder<H> {
 }
 
 fn create_browser<H: CefEventHandler>(event_handler: H, instance_dir: PathBuf, disable_gpu_acceleration: bool) -> Result<SingleThreadedCefContext, InitError> {
-	let render_handler = RenderHandler::new(RenderHandlerImpl::new(event_handler.clone()));
-	let mut client = Client::new(BrowserProcessClientImpl::new(render_handler, event_handler.clone()));
+	let mut client = Client::new(BrowserProcessClientImpl::new(&event_handler));
 
 	#[cfg(feature = "accelerated_paint")]
 	let use_accelerated_paint = if disable_gpu_acceleration {
@@ -180,7 +179,7 @@ fn create_browser<H: CefEventHandler>(event_handler: H, instance_dir: PathBuf, d
 		return Err(InitError::RequestContextCreationFailed);
 	};
 
-	let mut scheme_handler_factory = SchemeHandlerFactory::new(SchemeHandlerFactoryImpl::new(event_handler.clone()));
+	let mut scheme_handler_factory = SchemeHandlerFactory::new(SchemeHandlerFactoryImpl::new(event_handler.duplicate()));
 	incognito_request_context.clear_scheme_handler_factories();
 	incognito_request_context.register_scheme_handler_factory(Some(&CefString::from(RESOURCE_SCHEME)), Some(&CefString::from(RESOURCE_DOMAIN)), Some(&mut scheme_handler_factory));
 
@@ -197,6 +196,7 @@ fn create_browser<H: CefEventHandler>(event_handler: H, instance_dir: PathBuf, d
 
 	if let Some(browser) = browser {
 		Ok(SingleThreadedCefContext {
+			event_handler: Box::new(event_handler),
 			browser,
 			input_state: InputState::default(),
 			instance_dir,

desktop/src/cef/context/multithreaded.rs

@@ -30,11 +30,11 @@ impl CefContext for MultiThreadedCefContextProxy {
 		});
 	}
 
-	fn notify_of_resize(&self) {
+	fn notify_view_info_changed(&self) {
 		run_on_ui_thread(move || {
 			CONTEXT.with(|b| {
 				if let Some(context) = b.borrow_mut().as_mut() {
-					context.notify_of_resize();
+					context.notify_view_info_changed();
 				}
 			});
 		});

desktop/src/cef/context/singlethreaded.rs

@@ -1,13 +1,14 @@
 use cef::{Browser, ImplBrowser, ImplBrowserHost};
 use winit::event::WindowEvent;
 
-use crate::cef::input;
 use crate::cef::input::InputState;
 use crate::cef::ipc::{MessageType, SendMessage};
+use crate::cef::{CefEventHandler, input};
 
 use super::CefContext;
 
 pub(super) struct SingleThreadedCefContext {
+	pub(super) event_handler: Box<dyn CefEventHandler>,
 	pub(super) browser: Browser,
 	pub(super) input_state: InputState,
 	pub(super) instance_dir: std::path::PathBuf,
@@ -19,11 +20,14 @@ impl CefContext for SingleThreadedCefContext {
 	}
 
 	fn handle_window_event(&mut self, event: &WindowEvent) {
-		input::handle_window_event(&self.browser, &mut self.input_state, event)
+		input::handle_window_event(&self.browser, &mut self.input_state, event);
 	}
 
-	fn notify_of_resize(&self) {
-		self.browser.host().unwrap().was_resized();
+	fn notify_view_info_changed(&self) {
+		let view_info = self.event_handler.view_info();
+		let host = self.browser.host().unwrap();
+		host.set_zoom_level(view_info.zoom());
+		host.was_resized();
 	}
 
 	fn send_web_message(&self, message: Vec<u8>) {