Desktop: Restructure UI input handling

desktop/src/app.rs

@@ -17,11 +17,11 @@ use winit::window::WindowId;
 
 use crate::dirs;
 use crate::event::{AppEvent, AppEventScheduler};
-use crate::input::{InputAction, InputState};
+use crate::input::InputState;
 use crate::persist;
 use crate::preferences;
 use crate::render::{RenderError, RenderState};
-use crate::ui::{UiCommand, UiInstance};
+use crate::ui::{InputEvent, UiCommand, UiInstance};
 use crate::window::Window;
 use crate::wrapper::messages::{DesktopFrontendMessage, DesktopWrapperMessage, Preferences};
 use crate::wrapper::{DesktopWrapper, MmapResourceStorage, NodeGraphExecutionResult, WgpuContext, serialize_frontend_messages};
@@ -547,20 +547,16 @@ impl ApplicationHandler for App {
 			if let Some(window) = &self.window {
 				window.end_pointer_lock();
 			}
-			self.ui.send(UiCommand::Input(WindowEvent::PointerMoved {
-				device_id: None,
-				position: pointer_lock_position,
-				primary: true,
-				source: winit::event::PointerSource::Mouse,
-			}));
+			self.ui.send(UiCommand::Input(
+				InputEvent::pointer().position(pointer_lock_position).moved().modifiers(self.input_state.modifiers()).build(),
+			));
 		}
 
-		for action in self.input_state.process(&event) {
-			match action {
-				InputAction::Ui(event) => self.ui.send(UiCommand::Input(event)),
-				InputAction::Editor(message) => self.app_event_scheduler.schedule(AppEvent::DesktopWrapperMessage(message)),
-			}
-		}
+		self.input_state.process(
+			&event,
+			|message| self.app_event_scheduler.schedule(AppEvent::DesktopWrapperMessage(DesktopWrapperMessage::Input(message))),
+			|input| self.ui.send(UiCommand::Input(input)),
+		);
 
 		match event {
 			WindowEvent::CloseRequested => {

desktop/src/input.rs

@@ -1,30 +1,20 @@
 use std::time::Instant;
 use winit::dpi::PhysicalPosition;
 use winit::event::{ButtonSource, ElementState, MouseButton, MouseScrollDelta, PointerSource, TabletToolData, TabletToolKind, WindowEvent};
+use winit::keyboard::ModifiersState;
 
-use crate::ui::{MULTICLICK_ALLOWED_TRAVEL, MULTICLICK_TIMEOUT, PINCH_ZOOM_SPEED, SCROLL_LINE_HEIGHT, SCROLL_LINE_WIDTH, SCROLL_SPEED_X, SCROLL_SPEED_Y};
-use crate::wrapper::messages::{DesktopWrapperMessage, InputMessage, ModifierKeys, MouseKeys, PointerState, ScrollDelta};
+use crate::ui::{InputEvent, MULTICLICK_ALLOWED_TRAVEL, MULTICLICK_TIMEOUT, PINCH_ZOOM_SPEED, SCROLL_LINE_HEIGHT, SCROLL_LINE_WIDTH, SCROLL_SPEED_X, SCROLL_SPEED_Y};
+use crate::wrapper::messages::{InputMessage, ModifierKeys, MouseKeys, PointerState, ScrollDelta};
 
 pub(crate) struct InputState {
 	start: Instant,
 	viewport_info: Option<ViewportInfo>,
 	pointer_lock_position: Option<PhysicalPosition<f64>>,
-	modifier_keys: ModifierKeys,
+	modifiers: ModifiersState,
 	pointer_position: PhysicalPosition<f64>,
 	pointer_keys: MouseKeys,
 	ui_capture: bool,
-	multiclick: Option<Multiclick>,
-}
-
-pub(crate) enum InputAction {
-	Ui(WindowEvent),
-	Editor(DesktopWrapperMessage),
-}
-
-impl InputAction {
-	fn editor(message: InputMessage) -> Self {
-		Self::Editor(DesktopWrapperMessage::Input(message))
-	}
+	click_tracker: ClickTracker,
 }
 
 impl InputState {
@@ -33,11 +23,11 @@ impl InputState {
 			start: Instant::now(),
 			viewport_info: None,
 			pointer_lock_position: None,
-			modifier_keys: ModifierKeys::empty(),
+			modifiers: ModifiersState::default(),
 			pointer_position: PhysicalPosition::default(),
 			pointer_keys: MouseKeys::empty(),
 			ui_capture: true,
-			multiclick: None,
+			click_tracker: ClickTracker::default(),
 		}
 	}
 
@@ -61,32 +51,43 @@ impl InputState {
 		self.pointer_lock_position.is_some()
 	}
 
-	pub(crate) fn process(&mut self, event: &WindowEvent) -> Vec<InputAction> {
+	pub(crate) fn modifiers(&self) -> ModifiersState {
+		self.modifiers
+	}
+
+	pub(crate) fn process(&mut self, event: &WindowEvent, mut editor_callback: impl FnMut(InputMessage), mut ui_callback: impl FnMut(InputEvent)) {
 		match event {
 			WindowEvent::PointerMoved { position, source, .. } => {
 				self.pointer_position = *position;
 
 				let PointerSource::TabletTool { kind, data } = source else {
-					return vec![InputAction::Ui(event.clone())];
+					ui_callback(InputEvent::pointer().position(*position).moved().modifiers(self.modifiers).build());
+					return;
 				};
 				let ui_capture = if self.pointer_keys.is_empty() {
 					self.pointer_locked() || !self.in_viewport(*position)
 				} else {
 					self.ui_capture
 				};
 				if ui_capture {
-					return vec![InputAction::Ui(event.clone())];
+					ui_callback(InputEvent::pointer().position(*position).moved().modifiers(self.modifiers).build());
+					return;
 				}
 
-				vec![InputAction::editor(InputMessage::PointerMove {
+				editor_callback(InputMessage::PointerMove {
 					editor_mouse_state: self.tablet_pointer_state(kind, data),
-					modifier_keys: self.modifier_keys,
-				})]
+					modifier_keys: self.modifier_keys(),
+				});
+			}
+			WindowEvent::PointerEntered { position, .. } => {
+				self.pointer_position = *position;
+				ui_callback(InputEvent::pointer().position(*position).entered().modifiers(self.modifiers).build())
 			}
-			WindowEvent::PointerEntered { position, .. } | WindowEvent::PointerLeft { position: Some(position), .. } => {
+			WindowEvent::PointerLeft { position: Some(position), .. } => {
 				self.pointer_position = *position;
-				vec![InputAction::Ui(event.clone())]
+				ui_callback(InputEvent::pointer().position(*position).exited().modifiers(self.modifiers).build())
 			}
+			WindowEvent::PointerLeft { position: None, .. } => ui_callback(InputEvent::pointer().exited().modifiers(self.modifiers).build()),
 			WindowEvent::PointerButton { state, button, position, .. } => {
 				self.pointer_position = *position;
 
@@ -111,38 +112,47 @@ impl InputState {
 					ElementState::Released => self.pointer_keys.remove(keys),
 				}
 
+				let count = mouse_button.map_or(1, |button| self.click_tracker.input(*position, button, *state));
+
 				let back_or_forward = matches!(mouse_button, Some(MouseButton::Back | MouseButton::Forward));
 				if self.pointer_locked() || !(back_or_forward || (tablet && !self.ui_capture)) {
-					return vec![InputAction::Ui(event.clone())];
+					let pointer = InputEvent::pointer().position(*position);
+					let input = match state {
+						ElementState::Pressed => pointer.pressed(button.clone(), count),
+						ElementState::Released => pointer.released(button.clone(), count),
+					};
+					ui_callback(input.modifiers(self.modifiers).build());
+					return;
 				}
 
 				let editor_mouse_state = match button {
 					ButtonSource::TabletTool { kind, data, .. } => self.tablet_pointer_state(kind, data),
 					_ => self.pointer_state(),
 				};
-				let modifier_keys = self.modifier_keys;
+				let modifier_keys = self.modifier_keys();
 				match state {
-					ElementState::Pressed => vec![InputAction::editor(InputMessage::PointerDown { editor_mouse_state, modifier_keys })],
-					ElementState::Released => {
-						let mut actions = vec![InputAction::editor(InputMessage::PointerUp { editor_mouse_state, modifier_keys })];
-						if let Some(mouse_button) = mouse_button
-							&& self.track_multiclick(mouse_button, *position)
-						{
-							actions.push(InputAction::editor(InputMessage::DoubleClick {
-								editor_mouse_state: PointerState {
-									mouse_keys: keys,
-									..editor_mouse_state
-								},
-								modifier_keys,
-							}));
-						}
-						actions
+					ElementState::Pressed => editor_callback(InputMessage::PointerDown { editor_mouse_state, modifier_keys }),
+					ElementState::Released if count % 2 == 0 => {
+						editor_callback(InputMessage::PointerUp { editor_mouse_state, modifier_keys });
+						editor_callback(InputMessage::DoubleClick {
+							editor_mouse_state: PointerState {
+								mouse_keys: keys,
+								..editor_mouse_state
+							},
+							modifier_keys,
+						});
 					}
+					ElementState::Released => editor_callback(InputMessage::PointerUp { editor_mouse_state, modifier_keys }),
 				}
 			}
 			WindowEvent::MouseWheel { delta, .. } => {
 				if self.pointer_locked() || !self.in_viewport(self.pointer_position) {
-					return vec![InputAction::Ui(event.clone())];
+					let input = match delta {
+						MouseScrollDelta::LineDelta(x, y) => InputEvent::pointer().scrolled_lines(f64::from(*x), f64::from(*y)),
+						MouseScrollDelta::PixelDelta(position) => InputEvent::pointer().scrolled_pixels(position.x, position.y),
+					};
+					ui_callback(input.modifiers(self.modifiers).build());
+					return;
 				}
 
 				let (x, y) = match delta {
@@ -152,34 +162,29 @@ impl InputState {
 
 				let scroll_delta = ScrollDelta::new(-x * SCROLL_SPEED_X, -y * SCROLL_SPEED_Y, 0.);
 
-				vec![InputAction::editor(InputMessage::WheelScroll {
+				editor_callback(InputMessage::WheelScroll {
 					editor_mouse_state: PointerState { scroll_delta, ..self.pointer_state() },
-					modifier_keys: self.modifier_keys,
-				})]
+					modifier_keys: self.modifier_keys(),
+				});
 			}
 			WindowEvent::PinchGesture { delta, .. } => {
 				if self.pointer_locked() || !self.in_viewport(self.pointer_position) || !delta.is_normal() {
-					return vec![InputAction::Ui(event.clone())];
+					ui_callback(InputEvent::pointer().zoomed(*delta).modifiers(self.modifiers).build());
+					return;
 				}
 
 				// TODO: This is a temporary solution to handle pinch gestures, we should handle pinch gestures editor-side instead.
 				let scroll_delta = ScrollDelta::new(0., -delta * PINCH_ZOOM_SPEED, 0.);
-				vec![InputAction::editor(InputMessage::WheelScroll {
+				editor_callback(InputMessage::WheelScroll {
 					editor_mouse_state: PointerState { scroll_delta, ..self.pointer_state() },
-					modifier_keys: self.modifier_keys | ModifierKeys::CONTROL,
-				})]
+					modifier_keys: self.modifier_keys() | ModifierKeys::CONTROL,
+				});
 			}
 			WindowEvent::ModifiersChanged(modifiers) => {
-				let modifiers = modifiers.state();
-				self.modifier_keys = ModifierKeys::empty();
-				self.modifier_keys.set(ModifierKeys::SHIFT, modifiers.shift_key());
-				self.modifier_keys.set(ModifierKeys::CONTROL, modifiers.control_key());
-				self.modifier_keys.set(ModifierKeys::ALT, modifiers.alt_key());
-				self.modifier_keys.set(ModifierKeys::META_OR_COMMAND, modifiers.meta_key());
-
-				vec![InputAction::Ui(event.clone())]
+				self.modifiers = modifiers.state();
 			}
-			_ => vec![InputAction::Ui(event.clone())],
+			WindowEvent::KeyboardInput { event, .. } => ui_callback(InputEvent::key(event).modifiers(self.modifiers).build()),
+			_ => {}
 		}
 	}
 
@@ -211,16 +216,13 @@ impl InputState {
 		}
 	}
 
-	fn track_multiclick(&mut self, button: MouseButton, position: PhysicalPosition<f64>) -> bool {
-		let now = Instant::now();
-		let travel = MULTICLICK_ALLOWED_TRAVEL as f64;
-		let double = self.multiclick.take().is_some_and(|click| {
-			click.button == button && now.duration_since(click.time) <= MULTICLICK_TIMEOUT && (position.x - click.position.x).abs() <= travel && (position.y - click.position.y).abs() <= travel
-		});
-		if !double {
-			self.multiclick = Some(Multiclick { button, time: now, position });
-		}
-		double
+	fn modifier_keys(&self) -> ModifierKeys {
+		let mut keys = ModifierKeys::empty();
+		keys.set(ModifierKeys::SHIFT, self.modifiers.shift_key());
+		keys.set(ModifierKeys::CONTROL, self.modifiers.control_key());
+		keys.set(ModifierKeys::ALT, self.modifiers.alt_key());
+		keys.set(ModifierKeys::META_OR_COMMAND, self.modifiers.meta_key());
+		keys
 	}
 }
 
@@ -238,8 +240,55 @@ impl ViewportInfo {
 	}
 }
 
-struct Multiclick {
-	button: MouseButton,
+#[derive(Default)]
+struct ClickTracker {
+	left: ClickChains,
+	right: ClickChains,
+	middle: ClickChains,
+	back: ClickChains,
+	forward: ClickChains,
+}
+
+impl ClickTracker {
+	fn input(&mut self, position: PhysicalPosition<f64>, button: MouseButton, state: ElementState) -> u32 {
+		let position = (position.x as i32, position.y as i32);
+		let clicks = match button {
+			MouseButton::Left => &mut self.left,
+			MouseButton::Right => &mut self.right,
+			MouseButton::Middle => &mut self.middle,
+			MouseButton::Back => &mut self.back,
+			MouseButton::Forward => &mut self.forward,
+			_ => return 1,
+		};
+		let chain = match state {
+			ElementState::Pressed => &mut clicks.down,
+			ElementState::Released => &mut clicks.up,
+		};
+
+		let now = Instant::now();
+		let count = match chain {
+			Some(previous) => {
+				let within_time = now.saturating_duration_since(previous.time) <= MULTICLICK_TIMEOUT;
+				let dx = position.0.abs_diff(previous.position.0) as usize;
+				let dy = position.1.abs_diff(previous.position.1) as usize;
+				let within_distance = dx <= MULTICLICK_ALLOWED_TRAVEL && dy <= MULTICLICK_ALLOWED_TRAVEL;
+				if within_time && within_distance { previous.count.saturating_add(1) } else { 1 }
+			}
+			None => 1,
+		};
+		*chain = Some(Click { time: now, position, count });
+		count
+	}
+}
+
+#[derive(Default)]
+struct ClickChains {
+	down: Option<Click>,
+	up: Option<Click>,
+}
+
+struct Click {
 	time: Instant,
-	position: PhysicalPosition<f64>,
+	position: (i32, i32),
+	count: u32,
 }

desktop/ui/src/context.rs

@@ -223,6 +223,7 @@ fn create_browser(delegate: BrowserDelegate, frames: FrameStreamer, view_info_se
 	.map(|browser| BrowserContext {
 		delegate,
 		browser,
+		input: input::InputState::default(),
 		view_info_sender,
 		_instance_dir: instance_dir,
 	})
@@ -252,12 +253,8 @@ pub(crate) enum InitError {
 pub(crate) struct CefContextHandle;
 
 impl CefContextHandle {
-	pub(crate) fn apply_input(&self, events: Vec<InputEvent>) {
-		with_context(move |context| {
-			for event in &events {
-				input::apply(&context.browser, event);
-			}
-		});
+	pub(crate) fn process_input(&self, event: InputEvent) {
+		with_context(move |context| input::process(&mut context.input, &context.browser, &event));
 	}
 
 	pub(crate) fn update_view_info(&self, update: ViewInfoUpdate) {
@@ -276,6 +273,7 @@ impl CefContextHandle {
 struct BrowserContext {
 	delegate: BrowserDelegate,
 	browser: Browser,
+	input: input::InputState,
 	view_info_sender: Sender<ViewInfoUpdate>,
 	_instance_dir: TempDir,
 }