feat: preserve keyboard shortcuts as typed capture events

openadapt_capture/__init__.py

@@ -44,6 +44,7 @@
     Event,
     EventType,
     KeyDownEvent,
+    KeyShortcutEvent,
     KeyTypeEvent,
     KeyUpEvent,
     MouseButton,
@@ -186,6 +187,7 @@
     "KeyDownEvent",
     "KeyUpEvent",
     "KeyTypeEvent",
+    "KeyShortcutEvent",
     # Screen/audio events
     "ScreenFrameEvent",
     "AudioChunkEvent",

openadapt_capture/capture.py

@@ -28,6 +28,7 @@
 )
 from openadapt_capture.events import (
     KeyDownEvent,
+    KeyShortcutEvent,
     KeyTypeEvent,
     KeyUpEvent,
     MouseDownEvent,
@@ -351,6 +352,8 @@ def keys(self) -> list[str] | None:
 
         Returns list of key names like ['ctrl', 'space'] or ['enter'].
         """
+        if isinstance(self.event, KeyShortcutEvent):
+            return [*self.event.modifiers, self.event.key]
         if isinstance(self.event, KeyTypeEvent):
             key_names = []
             seen = set()

openadapt_capture/events.py

@@ -267,6 +267,27 @@ class KeyTypeEvent(ActionBaseEvent):
     )
 
 
+class KeyShortcutEvent(ActionBaseEvent):
+    """A modifier chord demonstrated by the operator.
+
+    ``modifiers`` is canonicalized by the processing pipeline (``ctrl``,
+    ``alt``, ``shift``, ``meta``). ``key`` is the single non-modifier trigger.
+    The raw down/up sequence remains in ``children`` so conversion never has to
+    reconstruct the demonstrated chord from display text.
+    """
+
+    type: Literal[EventType.KEY_SHORTCUT] = EventType.KEY_SHORTCUT
+    modifiers: list[str] = Field(
+        min_length=1,
+        description="Canonical modifier keys held for the chord",
+    )
+    key: str = Field(min_length=1, description="Non-modifier trigger key")
+    children: list[KeyDownEvent | KeyUpEvent] = Field(
+        default_factory=list,
+        description="Raw keyboard events retained for provenance",
+    )
+
+
 # =============================================================================
 # Union type for all events
 # =============================================================================
@@ -282,6 +303,7 @@ class KeyTypeEvent(ActionBaseEvent):
     | MouseDoubleClickEvent
     | MouseDragEvent
     | KeyTypeEvent
+    | KeyShortcutEvent
 )
 
 ScreenEvent = ScreenFrameEvent

openadapt_capture/processing.py

@@ -12,6 +12,7 @@
     ActionEvent,
     Event,
     KeyDownEvent,
+    KeyShortcutEvent,
     KeyTypeEvent,
     KeyUpEvent,
     MouseClickEvent,
@@ -38,6 +39,45 @@
 DOUBLE_CLICK_DISTANCE_PIXELS = 5.0  # default double-click distance
 KEY_TYPE_MERGE_INTERVAL_SECONDS = 0.5  # merge adjacent KeyTypeEvents within this interval
 
+_MODIFIER_ALIASES = {
+    "alt": "alt",
+    "alt_gr": "alt",
+    "alt_l": "alt",
+    "alt_r": "alt",
+    "cmd": "meta",
+    "cmd_l": "meta",
+    "cmd_r": "meta",
+    "command": "meta",
+    "control": "ctrl",
+    "ctrl": "ctrl",
+    "ctrl_l": "ctrl",
+    "ctrl_r": "ctrl",
+    "meta": "meta",
+    "shift": "shift",
+    "shift_l": "shift",
+    "shift_r": "shift",
+}
+_MODIFIER_ORDER = {"ctrl": 0, "alt": 1, "shift": 2, "meta": 3}
+
+
+def _keyboard_key(event: KeyDownEvent | KeyUpEvent) -> str:
+    """Return one bounded, stable identifier for a captured key event."""
+
+    value = (
+        event.canonical_key_name
+        or event.key_name
+        or event.canonical_key_char
+        or event.key_char
+        or event.canonical_key_vk
+        or event.key_vk
+        or ""
+    )
+    return str(value).strip().lower()
+
+
+def _modifier_for(event: KeyDownEvent | KeyUpEvent) -> str | None:
+    return _MODIFIER_ALIASES.get(_keyboard_key(event))
+
 
 def _first_structural_observation(
     events: list[ActionEvent],
@@ -157,19 +197,58 @@ def merge_consecutive_keyboard_events(events: list[ActionEvent]) -> list[ActionE
     pressed_keys: set[str] = set()
 
     def flush_buffer() -> None:
-        """Convert buffer to KeyTypeEvent and add to result."""
+        """Convert one raw keyboard sequence without losing modifier intent."""
         if not keyboard_buffer:
             return
 
-        # Extract typed characters from press events
-        chars = []
-        for event in keyboard_buffer:
-            if isinstance(event, KeyDownEvent) and event.key_char:
-                chars.append(event.key_char)
-
-        text = "".join(chars)
-        if text or keyboard_buffer:
-            first_event = keyboard_buffer[0]
+        down_events = [
+            event for event in keyboard_buffer if isinstance(event, KeyDownEvent)
+        ]
+        modifiers = sorted(
+            {
+                modifier
+                for event in down_events
+                if (modifier := _modifier_for(event)) is not None
+            },
+            key=_MODIFIER_ORDER.__getitem__,
+        )
+        triggers = [event for event in down_events if _modifier_for(event) is None]
+        first_event = keyboard_buffer[0]
+
+        # Shift plus a printable character is ordinary typing. Control, Alt,
+        # Meta, or Shift plus a named/non-printable key is a shortcut. A
+        # multi-trigger sequence remains a KeyTypeEvent with all raw children;
+        # downstream conversion can refuse it rather than inventing an order.
+        is_single_shortcut = (
+            bool(modifiers)
+            and len(triggers) == 1
+            and not (
+                modifiers == ["shift"]
+                and bool(
+                    triggers[0].canonical_key_char or triggers[0].key_char
+                )
+            )
+        )
+        if is_single_shortcut:
+            trigger = _keyboard_key(triggers[0])
+            if trigger:
+                result.append(
+                    KeyShortcutEvent(
+                        timestamp=first_event.timestamp,
+                        modifiers=modifiers,
+                        key=trigger,
+                        children=list(keyboard_buffer),
+                        structural_observation=_first_structural_observation(
+                            list(keyboard_buffer)
+                        ),
+                    )
+                )
+        else:
+            text = "".join(
+                event.key_char
+                for event in down_events
+                if event.key_char and _modifier_for(event) is None
+            )
             type_event = KeyTypeEvent(
                 timestamp=first_event.timestamp,
                 text=text,
@@ -546,6 +625,7 @@ def get_action_events(events: list[Event]) -> list[ActionEvent]:
         MouseDoubleClickEvent,
         MouseDragEvent,
         KeyTypeEvent,
+        KeyShortcutEvent,
     )
     return [e for e in events if isinstance(e, action_types)]
 

openadapt_capture/storage.py

@@ -19,6 +19,7 @@
     Event,
     EventType,
     KeyDownEvent,
+    KeyShortcutEvent,
     KeyTypeEvent,
     KeyUpEvent,
     MouseClickEvent,
@@ -140,6 +141,7 @@ class Capture(BaseModel):
     EventType.MOUSE_DOUBLECLICK.value: MouseDoubleClickEvent,
     EventType.MOUSE_DRAG.value: MouseDragEvent,
     EventType.KEY_TYPE.value: KeyTypeEvent,
+    EventType.KEY_SHORTCUT.value: KeyShortcutEvent,
 }
 
 

openadapt_capture/storage/sqlite.py

@@ -17,6 +17,7 @@
     Event,
     EventType,
     KeyDownEvent,
+    KeyShortcutEvent,
     KeyTypeEvent,
     KeyUpEvent,
     MouseClickEvent,
@@ -47,6 +48,7 @@
     EventType.MOUSE_DOUBLECLICK.value: MouseDoubleClickEvent,
     EventType.MOUSE_DRAG.value: MouseDragEvent,
     EventType.KEY_TYPE.value: KeyTypeEvent,
+    EventType.KEY_SHORTCUT.value: KeyShortcutEvent,
 }
 
 

openadapt_capture/storage_impl.py

@@ -19,6 +19,7 @@
     Event,
     EventType,
     KeyDownEvent,
+    KeyShortcutEvent,
     KeyTypeEvent,
     KeyUpEvent,
     MouseClickEvent,
@@ -140,6 +141,7 @@ class Capture(BaseModel):
     EventType.MOUSE_DOUBLECLICK.value: MouseDoubleClickEvent,
     EventType.MOUSE_DRAG.value: MouseDragEvent,
     EventType.KEY_TYPE.value: KeyTypeEvent,
+    EventType.KEY_SHORTCUT.value: KeyShortcutEvent,
 }
 
 

tests/test_events.py

@@ -5,6 +5,7 @@
     AudioChunkEvent,
     EventType,
     KeyDownEvent,
+    KeyShortcutEvent,
     KeyTypeEvent,
     KeyUpEvent,
     MouseButton,
@@ -168,6 +169,24 @@ def test_key_type_event(self):
         assert event.text == "hi"
         assert len(event.children) == 4
 
+    def test_key_shortcut_event(self):
+        children = [
+            KeyDownEvent(timestamp=1.0, key_name="ctrl"),
+            KeyDownEvent(timestamp=1.1, key_char="s"),
+            KeyUpEvent(timestamp=1.2, key_char="s"),
+            KeyUpEvent(timestamp=1.3, key_name="ctrl"),
+        ]
+        event = KeyShortcutEvent(
+            timestamp=1.0,
+            modifiers=["ctrl"],
+            key="s",
+            children=children,
+        )
+
+        assert event.model_dump()["type"] == "key.shortcut"
+        assert event.modifiers == ["ctrl"]
+        assert event.key == "s"
+
 
 class TestScreenEvents:
     """Tests for screen event types."""

tests/test_processing.py

@@ -3,6 +3,7 @@
 
 from openadapt_capture.events import (
     KeyDownEvent,
+    KeyShortcutEvent,
     KeyTypeEvent,
     KeyUpEvent,
     MouseButton,
@@ -111,6 +112,37 @@ def test_preserves_non_keyboard_events(self):
         result = merge_consecutive_keyboard_events(events)
         assert len(result) == 3  # KeyTypeEvent("a"), MouseMove, KeyTypeEvent("b")
 
+    def test_preserves_modifier_chord_as_typed_shortcut(self):
+        events = [
+            KeyDownEvent(timestamp=1.0, key_name="control"),
+            KeyDownEvent(timestamp=1.1, key_name="shift"),
+            KeyDownEvent(timestamp=1.2, key_char="s"),
+            KeyUpEvent(timestamp=1.3, key_char="s"),
+            KeyUpEvent(timestamp=1.4, key_name="shift"),
+            KeyUpEvent(timestamp=1.5, key_name="control"),
+        ]
+
+        result = merge_consecutive_keyboard_events(events)
+
+        assert len(result) == 1
+        assert isinstance(result[0], KeyShortcutEvent)
+        assert result[0].modifiers == ["ctrl", "shift"]
+        assert result[0].key == "s"
+
+    def test_shift_printable_character_remains_typed_text(self):
+        events = [
+            KeyDownEvent(timestamp=1.0, key_name="shift"),
+            KeyDownEvent(timestamp=1.1, key_char="A"),
+            KeyUpEvent(timestamp=1.2, key_char="A"),
+            KeyUpEvent(timestamp=1.3, key_name="shift"),
+        ]
+
+        result = merge_consecutive_keyboard_events(events)
+
+        assert len(result) == 1
+        assert isinstance(result[0], KeyTypeEvent)
+        assert result[0].text == "A"
+
 
 class TestMergeConsecutiveMouseMoveEvents:
     """Tests for merge_consecutive_mouse_move_events."""

tests/test_storage.py

@@ -8,6 +8,8 @@
 from openadapt_capture.events import (
     EventType,
     KeyDownEvent,
+    KeyShortcutEvent,
+    KeyUpEvent,
     MouseButton,
     MouseDownEvent,
     MouseMoveEvent,
@@ -116,6 +118,39 @@ def test_write_and_get_event(self, temp_db):
             assert events[0].x == 100.0
             assert events[0].y == 200.0
 
+    def test_shortcut_round_trips_with_raw_children(self, temp_db):
+        children = [
+            KeyDownEvent(timestamp=1.0, key_name="ctrl"),
+            KeyDownEvent(timestamp=1.1, key_char="s"),
+            KeyUpEvent(timestamp=1.2, key_char="s"),
+            KeyUpEvent(timestamp=1.3, key_name="ctrl"),
+        ]
+        shortcut = KeyShortcutEvent(
+            timestamp=1.0,
+            modifiers=["ctrl"],
+            key="s",
+            children=children,
+        )
+
+        with CaptureStorage(temp_db) as storage:
+            storage.write_event(shortcut)
+            loaded = storage.get_events()
+            stored_rows = storage.get_events(include_children=True)
+
+        assert len(loaded) == 1
+        restored = loaded[0]
+        assert isinstance(restored, KeyShortcutEvent)
+        assert restored.modifiers == ["ctrl"]
+        assert restored.key == "s"
+        assert restored.children == []
+        assert [event.type for event in stored_rows] == [
+            EventType.KEY_SHORTCUT,
+            EventType.KEY_DOWN,
+            EventType.KEY_DOWN,
+            EventType.KEY_UP,
+            EventType.KEY_UP,
+        ]
+
     def test_write_multiple_events(self, temp_db):
         """Test writing multiple events."""
         with CaptureStorage(temp_db) as storage: