diff --git a/openadapt_capture/__init__.py b/openadapt_capture/__init__.py index 79f28fd..58db2dc 100644 --- a/openadapt_capture/__init__.py +++ b/openadapt_capture/__init__.py @@ -44,6 +44,7 @@ Event, EventType, KeyDownEvent, + KeyShortcutEvent, KeyTypeEvent, KeyUpEvent, MouseButton, @@ -186,6 +187,7 @@ "KeyDownEvent", "KeyUpEvent", "KeyTypeEvent", + "KeyShortcutEvent", # Screen/audio events "ScreenFrameEvent", "AudioChunkEvent", diff --git a/openadapt_capture/capture.py b/openadapt_capture/capture.py index b4a229b..5bfcf51 100644 --- a/openadapt_capture/capture.py +++ b/openadapt_capture/capture.py @@ -28,6 +28,7 @@ ) from openadapt_capture.events import ( KeyDownEvent, + KeyShortcutEvent, KeyTypeEvent, KeyUpEvent, MouseDownEvent, @@ -351,6 +352,8 @@ def keys(self) -> list[str] | None: Returns list of key names like ['ctrl', 'space'] or ['enter']. """ + if isinstance(self.event, KeyShortcutEvent): + return [*self.event.modifiers, self.event.key] if isinstance(self.event, KeyTypeEvent): key_names = [] seen = set() diff --git a/openadapt_capture/events.py b/openadapt_capture/events.py index e22986f..57a7d3c 100644 --- a/openadapt_capture/events.py +++ b/openadapt_capture/events.py @@ -267,6 +267,27 @@ class KeyTypeEvent(ActionBaseEvent): ) +class KeyShortcutEvent(ActionBaseEvent): + """A modifier chord demonstrated by the operator. + + ``modifiers`` is canonicalized by the processing pipeline (``ctrl``, + ``alt``, ``shift``, ``meta``). ``key`` is the single non-modifier trigger. + The raw down/up sequence remains in ``children`` so conversion never has to + reconstruct the demonstrated chord from display text. + """ + + type: Literal[EventType.KEY_SHORTCUT] = EventType.KEY_SHORTCUT + modifiers: list[str] = Field( + min_length=1, + description="Canonical modifier keys held for the chord", + ) + key: str = Field(min_length=1, description="Non-modifier trigger key") + children: list[KeyDownEvent | KeyUpEvent] = Field( + default_factory=list, + description="Raw keyboard events retained for provenance", + ) + + # ============================================================================= # Union type for all events # ============================================================================= @@ -282,6 +303,7 @@ class KeyTypeEvent(ActionBaseEvent): | MouseDoubleClickEvent | MouseDragEvent | KeyTypeEvent + | KeyShortcutEvent ) ScreenEvent = ScreenFrameEvent diff --git a/openadapt_capture/processing.py b/openadapt_capture/processing.py index f30feb6..14ba9b1 100644 --- a/openadapt_capture/processing.py +++ b/openadapt_capture/processing.py @@ -12,6 +12,7 @@ ActionEvent, Event, KeyDownEvent, + KeyShortcutEvent, KeyTypeEvent, KeyUpEvent, MouseClickEvent, @@ -38,6 +39,45 @@ DOUBLE_CLICK_DISTANCE_PIXELS = 5.0 # default double-click distance KEY_TYPE_MERGE_INTERVAL_SECONDS = 0.5 # merge adjacent KeyTypeEvents within this interval +_MODIFIER_ALIASES = { + "alt": "alt", + "alt_gr": "alt", + "alt_l": "alt", + "alt_r": "alt", + "cmd": "meta", + "cmd_l": "meta", + "cmd_r": "meta", + "command": "meta", + "control": "ctrl", + "ctrl": "ctrl", + "ctrl_l": "ctrl", + "ctrl_r": "ctrl", + "meta": "meta", + "shift": "shift", + "shift_l": "shift", + "shift_r": "shift", +} +_MODIFIER_ORDER = {"ctrl": 0, "alt": 1, "shift": 2, "meta": 3} + + +def _keyboard_key(event: KeyDownEvent | KeyUpEvent) -> str: + """Return one bounded, stable identifier for a captured key event.""" + + value = ( + event.canonical_key_name + or event.key_name + or event.canonical_key_char + or event.key_char + or event.canonical_key_vk + or event.key_vk + or "" + ) + return str(value).strip().lower() + + +def _modifier_for(event: KeyDownEvent | KeyUpEvent) -> str | None: + return _MODIFIER_ALIASES.get(_keyboard_key(event)) + def _first_structural_observation( events: list[ActionEvent], @@ -157,19 +197,58 @@ def merge_consecutive_keyboard_events(events: list[ActionEvent]) -> list[ActionE pressed_keys: set[str] = set() def flush_buffer() -> None: - """Convert buffer to KeyTypeEvent and add to result.""" + """Convert one raw keyboard sequence without losing modifier intent.""" if not keyboard_buffer: return - # Extract typed characters from press events - chars = [] - for event in keyboard_buffer: - if isinstance(event, KeyDownEvent) and event.key_char: - chars.append(event.key_char) - - text = "".join(chars) - if text or keyboard_buffer: - first_event = keyboard_buffer[0] + down_events = [ + event for event in keyboard_buffer if isinstance(event, KeyDownEvent) + ] + modifiers = sorted( + { + modifier + for event in down_events + if (modifier := _modifier_for(event)) is not None + }, + key=_MODIFIER_ORDER.__getitem__, + ) + triggers = [event for event in down_events if _modifier_for(event) is None] + first_event = keyboard_buffer[0] + + # Shift plus a printable character is ordinary typing. Control, Alt, + # Meta, or Shift plus a named/non-printable key is a shortcut. A + # multi-trigger sequence remains a KeyTypeEvent with all raw children; + # downstream conversion can refuse it rather than inventing an order. + is_single_shortcut = ( + bool(modifiers) + and len(triggers) == 1 + and not ( + modifiers == ["shift"] + and bool( + triggers[0].canonical_key_char or triggers[0].key_char + ) + ) + ) + if is_single_shortcut: + trigger = _keyboard_key(triggers[0]) + if trigger: + result.append( + KeyShortcutEvent( + timestamp=first_event.timestamp, + modifiers=modifiers, + key=trigger, + children=list(keyboard_buffer), + structural_observation=_first_structural_observation( + list(keyboard_buffer) + ), + ) + ) + else: + text = "".join( + event.key_char + for event in down_events + if event.key_char and _modifier_for(event) is None + ) type_event = KeyTypeEvent( timestamp=first_event.timestamp, text=text, @@ -546,6 +625,7 @@ def get_action_events(events: list[Event]) -> list[ActionEvent]: MouseDoubleClickEvent, MouseDragEvent, KeyTypeEvent, + KeyShortcutEvent, ) return [e for e in events if isinstance(e, action_types)] diff --git a/openadapt_capture/storage.py b/openadapt_capture/storage.py index 3aa1208..c7e1524 100644 --- a/openadapt_capture/storage.py +++ b/openadapt_capture/storage.py @@ -19,6 +19,7 @@ Event, EventType, KeyDownEvent, + KeyShortcutEvent, KeyTypeEvent, KeyUpEvent, MouseClickEvent, @@ -140,6 +141,7 @@ class Capture(BaseModel): EventType.MOUSE_DOUBLECLICK.value: MouseDoubleClickEvent, EventType.MOUSE_DRAG.value: MouseDragEvent, EventType.KEY_TYPE.value: KeyTypeEvent, + EventType.KEY_SHORTCUT.value: KeyShortcutEvent, } diff --git a/openadapt_capture/storage/sqlite.py b/openadapt_capture/storage/sqlite.py index ab36af9..67a5c34 100644 --- a/openadapt_capture/storage/sqlite.py +++ b/openadapt_capture/storage/sqlite.py @@ -17,6 +17,7 @@ Event, EventType, KeyDownEvent, + KeyShortcutEvent, KeyTypeEvent, KeyUpEvent, MouseClickEvent, @@ -47,6 +48,7 @@ EventType.MOUSE_DOUBLECLICK.value: MouseDoubleClickEvent, EventType.MOUSE_DRAG.value: MouseDragEvent, EventType.KEY_TYPE.value: KeyTypeEvent, + EventType.KEY_SHORTCUT.value: KeyShortcutEvent, } diff --git a/openadapt_capture/storage_impl.py b/openadapt_capture/storage_impl.py index 3aa1208..c7e1524 100644 --- a/openadapt_capture/storage_impl.py +++ b/openadapt_capture/storage_impl.py @@ -19,6 +19,7 @@ Event, EventType, KeyDownEvent, + KeyShortcutEvent, KeyTypeEvent, KeyUpEvent, MouseClickEvent, @@ -140,6 +141,7 @@ class Capture(BaseModel): EventType.MOUSE_DOUBLECLICK.value: MouseDoubleClickEvent, EventType.MOUSE_DRAG.value: MouseDragEvent, EventType.KEY_TYPE.value: KeyTypeEvent, + EventType.KEY_SHORTCUT.value: KeyShortcutEvent, } diff --git a/tests/test_events.py b/tests/test_events.py index b6d9fdd..12844b7 100644 --- a/tests/test_events.py +++ b/tests/test_events.py @@ -5,6 +5,7 @@ AudioChunkEvent, EventType, KeyDownEvent, + KeyShortcutEvent, KeyTypeEvent, KeyUpEvent, MouseButton, @@ -168,6 +169,24 @@ def test_key_type_event(self): assert event.text == "hi" assert len(event.children) == 4 + def test_key_shortcut_event(self): + children = [ + KeyDownEvent(timestamp=1.0, key_name="ctrl"), + KeyDownEvent(timestamp=1.1, key_char="s"), + KeyUpEvent(timestamp=1.2, key_char="s"), + KeyUpEvent(timestamp=1.3, key_name="ctrl"), + ] + event = KeyShortcutEvent( + timestamp=1.0, + modifiers=["ctrl"], + key="s", + children=children, + ) + + assert event.model_dump()["type"] == "key.shortcut" + assert event.modifiers == ["ctrl"] + assert event.key == "s" + class TestScreenEvents: """Tests for screen event types.""" diff --git a/tests/test_processing.py b/tests/test_processing.py index bea7ea8..5ef9b88 100644 --- a/tests/test_processing.py +++ b/tests/test_processing.py @@ -3,6 +3,7 @@ from openadapt_capture.events import ( KeyDownEvent, + KeyShortcutEvent, KeyTypeEvent, KeyUpEvent, MouseButton, @@ -111,6 +112,37 @@ def test_preserves_non_keyboard_events(self): result = merge_consecutive_keyboard_events(events) assert len(result) == 3 # KeyTypeEvent("a"), MouseMove, KeyTypeEvent("b") + def test_preserves_modifier_chord_as_typed_shortcut(self): + events = [ + KeyDownEvent(timestamp=1.0, key_name="control"), + KeyDownEvent(timestamp=1.1, key_name="shift"), + KeyDownEvent(timestamp=1.2, key_char="s"), + KeyUpEvent(timestamp=1.3, key_char="s"), + KeyUpEvent(timestamp=1.4, key_name="shift"), + KeyUpEvent(timestamp=1.5, key_name="control"), + ] + + result = merge_consecutive_keyboard_events(events) + + assert len(result) == 1 + assert isinstance(result[0], KeyShortcutEvent) + assert result[0].modifiers == ["ctrl", "shift"] + assert result[0].key == "s" + + def test_shift_printable_character_remains_typed_text(self): + events = [ + KeyDownEvent(timestamp=1.0, key_name="shift"), + KeyDownEvent(timestamp=1.1, key_char="A"), + KeyUpEvent(timestamp=1.2, key_char="A"), + KeyUpEvent(timestamp=1.3, key_name="shift"), + ] + + result = merge_consecutive_keyboard_events(events) + + assert len(result) == 1 + assert isinstance(result[0], KeyTypeEvent) + assert result[0].text == "A" + class TestMergeConsecutiveMouseMoveEvents: """Tests for merge_consecutive_mouse_move_events.""" diff --git a/tests/test_storage.py b/tests/test_storage.py index 295f0e2..9262e3f 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -8,6 +8,8 @@ from openadapt_capture.events import ( EventType, KeyDownEvent, + KeyShortcutEvent, + KeyUpEvent, MouseButton, MouseDownEvent, MouseMoveEvent, @@ -116,6 +118,39 @@ def test_write_and_get_event(self, temp_db): assert events[0].x == 100.0 assert events[0].y == 200.0 + def test_shortcut_round_trips_with_raw_children(self, temp_db): + children = [ + KeyDownEvent(timestamp=1.0, key_name="ctrl"), + KeyDownEvent(timestamp=1.1, key_char="s"), + KeyUpEvent(timestamp=1.2, key_char="s"), + KeyUpEvent(timestamp=1.3, key_name="ctrl"), + ] + shortcut = KeyShortcutEvent( + timestamp=1.0, + modifiers=["ctrl"], + key="s", + children=children, + ) + + with CaptureStorage(temp_db) as storage: + storage.write_event(shortcut) + loaded = storage.get_events() + stored_rows = storage.get_events(include_children=True) + + assert len(loaded) == 1 + restored = loaded[0] + assert isinstance(restored, KeyShortcutEvent) + assert restored.modifiers == ["ctrl"] + assert restored.key == "s" + assert restored.children == [] + assert [event.type for event in stored_rows] == [ + EventType.KEY_SHORTCUT, + EventType.KEY_DOWN, + EventType.KEY_DOWN, + EventType.KEY_UP, + EventType.KEY_UP, + ] + def test_write_multiple_events(self, temp_db): """Test writing multiple events.""" with CaptureStorage(temp_db) as storage: