diff --git a/README.md b/README.md index b355467..014ab2b 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,7 @@ Documentation for the whole stack lives at | `Episode` / `Step` | Complete task trajectory (observation → action → result) | | `FailureRecord` | Classified failure for dataset pipelines | | `ControlOverlayFrameV1` / `ControlOverlayTimelineV1` | PHI-safe execution overlay state bound to exact evidence media | +| `ControlOverlayFrameV2` / `ControlOverlayTimelineV2` | Exact, privacy-safe target geometry for sibling overlays and media composition | ## Quick start @@ -144,19 +145,43 @@ print(json.dumps(schema, indent=2)) The same API exports the versioned cross-surface overlay contracts: ```python -from openadapt_types import ControlOverlayFrameV1, ControlOverlayTimelineV1 +from openadapt_types import ( + ControlOverlayFrameV1, + ControlOverlayFrameV2, + ControlOverlayTimelineV1, + ControlOverlayTimelineV2, +) frame_schema = ControlOverlayFrameV1.model_json_schema() timeline_schema = ControlOverlayTimelineV1.model_json_schema() +tracking_frame_schema = ControlOverlayFrameV2.model_json_schema() +tracking_timeline_schema = ControlOverlayTimelineV2.model_json_schema() ``` -The same schemas ship as `openadapt_types/schemas/control-overlay-frame-v1.json` -and `control-overlay-timeline-v1.json` for TypeScript, Rust, and other consumers. +The same schemas ship under `openadapt_types/schemas/` for TypeScript, Rust, +and other consumers. Version 1 remains the control-state contract. Version 2 +adds an optional normalized top-level viewport rectangle, the exact source +viewport and DPR, and an exact observation or decoded-media-frame binding +without changing V1. Overlay schemas reject unknown fields and contain only closed presentation -labels and canonical statuses. Screenshots, action targets, typed values, -identities, URLs, logs, report bodies, and user-authored workflow names remain -outside this public presentation contract. +labels and canonical statuses. Screenshot payloads, action-target selectors, +accessible names, text and values, typed input, identities, URLs, logs, report +bodies, and user-authored workflow names remain outside this public contract. +V2 may carry only normalized target geometry from a browser top-level CSS +viewport. Native and RDP device-pixel geometry is not part of this V2 schema. + +Target geometry never carries locators, accessible names, values, URLs, or +screenshots. A private live observation uses a run/export-scoped HMAC reference +instead of a linkable raw frame hash. Published media uses the exact media +SHA-256 and decoded frame index. A renderer draws tracking only when that +binding matches; it omits the rectangle rather than replaying selectors, +interpolating movement, or inferring a missing target from adjacent events. +The runtime does not guess a future viewer transform. Desktop, Cloud, and media +renderers map the normalized rectangle through their actual content box. +If multiple runtime states land in one decoded media frame, the producer must +coalesce them deterministically; it must not invent extra media frames or +approximate their timing. ## Design principles diff --git a/openadapt_types/__init__.py b/openadapt_types/__init__.py index ecdd12d..1557d12 100644 --- a/openadapt_types/__init__.py +++ b/openadapt_types/__init__.py @@ -61,6 +61,23 @@ control_overlay_state_id, is_terminal_control_overlay_phase, ) +from openadapt_types.control_overlay_tracking import ( + CONTROL_OVERLAY_FRAME_V2_SCHEMA, + CONTROL_OVERLAY_TIMELINE_V2_SCHEMA, + ControlOverlayFrameV2, + ControlOverlayMediaFrameBindingV2, + ControlOverlayNormalizedRectV2, + ControlOverlayObservationBindingV2, + ControlOverlaySourceViewportV2, + ControlOverlayTargetActionKind, + ControlOverlayTargetBindingV2, + ControlOverlayTargetTrackingV2, + ControlOverlayTimelineBindingV2, + ControlOverlayTimelineEventV2, + ControlOverlayTimelineV2, + build_control_overlay_timeline_v2, + control_overlay_state_id_v2, +) from openadapt_types.episode import Episode, Step from openadapt_types.failure import FailureCategory, FailureRecord from openadapt_types.parsing import ( @@ -101,6 +118,22 @@ "build_control_overlay_timeline", "control_overlay_state_id", "is_terminal_control_overlay_phase", + # exact control-overlay target tracking + "CONTROL_OVERLAY_FRAME_V2_SCHEMA", + "CONTROL_OVERLAY_TIMELINE_V2_SCHEMA", + "ControlOverlayFrameV2", + "ControlOverlayMediaFrameBindingV2", + "ControlOverlayNormalizedRectV2", + "ControlOverlayObservationBindingV2", + "ControlOverlaySourceViewportV2", + "ControlOverlayTargetActionKind", + "ControlOverlayTargetBindingV2", + "ControlOverlayTargetTrackingV2", + "ControlOverlayTimelineBindingV2", + "ControlOverlayTimelineEventV2", + "ControlOverlayTimelineV2", + "build_control_overlay_timeline_v2", + "control_overlay_state_id_v2", # action "Action", "ActionResult", diff --git a/openadapt_types/control_overlay_tracking.py b/openadapt_types/control_overlay_tracking.py new file mode 100644 index 0000000..222b9ba --- /dev/null +++ b/openadapt_types/control_overlay_tracking.py @@ -0,0 +1,462 @@ +"""Exact, privacy-safe target tracking for control-overlay presentation. + +Version 2 is additive: the version 1 control-state and timeline contracts stay +unchanged. It carries browser top-level CSS-viewport geometry and metadata; +native and RDP device-pixel coordinate spaces are intentionally out of scope. +It deliberately excludes selectors, accessibility content, typed values, URLs, +screenshots, identities, report text, and other customer data. + +A target rectangle is usable only when its binding matches the current private +observation or the exact decoded frame of the bound media. Renderers must omit +the rectangle when that match cannot be established. They must never replay a +selector or infer a target from nearby timeline states. +""" + +from __future__ import annotations + +import hashlib +import json +from enum import Enum +from hmac import compare_digest +from math import isfinite +from typing import Annotated, Literal, Sequence + +from pydantic import ( + BaseModel, + ConfigDict, + Field, + StrictBool, + StrictFloat, + StrictInt, + StrictStr, + model_validator, +) + +from openadapt_types.control_overlay import ( + CONTROL_OVERLAY_STATUS_BY_PHASE, + CONTROL_OVERLAY_WORKFLOW_LABEL_BY_MODE, + ControlOverlayControlsV1, + ControlOverlayDataClassification, + ControlOverlayMode, + ControlOverlayPhase, + ControlOverlayProfile, + ControlOverlayStepV1, + ControlOverlayWorkflowLabel, + control_overlay_state_id, +) + +CONTROL_OVERLAY_FRAME_V2_SCHEMA = "openadapt.control-overlay-frame/v2" +CONTROL_OVERLAY_TIMELINE_V2_SCHEMA = "openadapt.control-overlay-timeline/v2" + +_TARGET_TRACKING_JSON_SCHEMA_EXTRA = { + "x-openadapt-render-only-on-exact-binding": True, + "x-openadapt-missing-binding-behavior": "omit_target", + "x-openadapt-runtime-resolution-replay": False, + "x-openadapt-renderer-mapping": "actual_content_box", + "x-openadapt-source-scope": "browser_top_level_css_viewport", +} + +_TARGET_TIMELINE_JSON_SCHEMA_EXTRA = { + "x-openadapt-target-validity": "exact_decoded_media_frame_only", + "x-openadapt-missing-frame-behavior": "omit_target", +} + + +class _StrictContract(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + +class ControlOverlayTargetActionKind(str, Enum): + """Closed, presentation-only kinds for actions with a visible target.""" + + CLICK = "click" + DOUBLE_CLICK = "double_click" + RIGHT_CLICK = "right_click" + DRAG = "drag" + TYPE = "type" + SELECT = "select" + TOGGLE = "toggle" + INVOKE = "invoke" + EXPAND_COLLAPSE = "expand_collapse" + SCROLL = "scroll" + HOVER = "hover" + + +class ControlOverlayNormalizedRectV2(_StrictContract): + """Axis-aligned rectangle normalized to the source top-level viewport.""" + + x: StrictFloat = Field(ge=0, le=1) + y: StrictFloat = Field(ge=0, le=1) + width: StrictFloat = Field(gt=0, le=1) + height: StrictFloat = Field(gt=0, le=1) + + @model_validator(mode="after") + def _validate_extent(self) -> "ControlOverlayNormalizedRectV2": + values = (self.x, self.y, self.width, self.height) + if not all(isfinite(value) for value in values): + raise ValueError("normalized rectangle values must be finite") + if self.x + self.width > 1 + 1e-9: + raise ValueError("normalized rectangle exceeds viewport width") + if self.y + self.height > 1 + 1e-9: + raise ValueError("normalized rectangle exceeds viewport height") + return self + + +class ControlOverlaySourceViewportV2(_StrictContract): + """Browser top-level CSS viewport used to resolve the target. + + Native and RDP device-pixel dimensions must not be placed in these fields. + Those coordinate spaces require a separate future contract. + """ + + width_css_px: StrictInt = Field(gt=0, le=32768) + height_css_px: StrictInt = Field(gt=0, le=32768) + device_pixel_ratio: StrictFloat = Field(gt=0, le=16) + + @model_validator(mode="after") + def _validate_finite_dpr(self) -> "ControlOverlaySourceViewportV2": + if not isfinite(self.device_pixel_ratio): + raise ValueError("device_pixel_ratio must be finite") + return self + + +class ControlOverlayObservationBindingV2(_StrictContract): + """Opaque binding to one exact private runtime observation. + + The digest must be HMAC-SHA256 over a canonical private observation ID, + domain-separated for this contract and keyed with a run/export-scoped + secret. A raw screenshot, frame, or observation SHA-256 is not valid here: + it can be linkable across evidence boundaries. + """ + + kind: Literal["observation_hmac_sha256"] = "observation_hmac_sha256" + observation_hmac_sha256: StrictStr = Field(pattern=r"^[a-f0-9]{64}$") + + +class ControlOverlayMediaFrameBindingV2(_StrictContract): + """Binding to one exact decoded frame of one exact immutable media file.""" + + kind: Literal["media_frame"] = "media_frame" + media_sha256: StrictStr = Field(pattern=r"^[a-f0-9]{64}$") + frame_index: StrictInt = Field(ge=0) + + +ControlOverlayTargetBindingV2 = Annotated[ + ControlOverlayObservationBindingV2 | ControlOverlayMediaFrameBindingV2, + Field(discriminator="kind"), +] + + +class ControlOverlayTargetTrackingV2(_StrictContract): + """Exact browser CSS-viewport geometry, never resolution evidence.""" + + model_config = ConfigDict(json_schema_extra=_TARGET_TRACKING_JSON_SCHEMA_EXTRA) + + coordinate_space: Literal["top_level_viewport_normalized"] = Field( + default="top_level_viewport_normalized", + description=( + "Normalized browser top-level CSS viewport; not native or RDP " + "device-pixel coordinates" + ), + ) + rect: ControlOverlayNormalizedRectV2 + source_viewport: ControlOverlaySourceViewportV2 + binding: ControlOverlayTargetBindingV2 + action_kind: ControlOverlayTargetActionKind | None = None + + +def _target_digest(target: ControlOverlayTargetTrackingV2 | None) -> str: + if target is None: + return "no-target" + canonical = json.dumps( + target.model_dump(mode="json"), + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + return f"target-{hashlib.sha256(canonical).hexdigest()[:16]}" + + +def control_overlay_state_id_v2( + *, + visible: bool, + phase: ControlOverlayPhase, + mode: ControlOverlayMode, + profile: ControlOverlayProfile | None, + step: ControlOverlayStepV1, + controls: ControlOverlayControlsV1, + target_tracking: ControlOverlayTargetTrackingV2 | None, +) -> str: + """Build V2 frame identity from control state and exact tracking state.""" + + return ":".join( + ( + control_overlay_state_id( + visible=visible, + phase=phase, + mode=mode, + profile=profile, + step=step, + controls=controls, + ), + _target_digest(target_tracking), + ) + ) + + +class ControlOverlayFrameV2(_StrictContract): + """Control state plus optional exactly bound target presentation geometry.""" + + schema_version: Literal["openadapt.control-overlay-frame/v2"] = ( + CONTROL_OVERLAY_FRAME_V2_SCHEMA + ) + state_id: StrictStr = Field(min_length=1, max_length=280) + event_sequence: StrictInt = Field(ge=0) + observed_at_unix_ms: StrictInt = Field(ge=0) + observed_at_monotonic_ms: StrictFloat = Field(ge=0) + visible: StrictBool + phase: ControlOverlayPhase + workflow_label: ControlOverlayWorkflowLabel + mode: ControlOverlayMode + profile: ControlOverlayProfile | None = None + step: ControlOverlayStepV1 + controls: ControlOverlayControlsV1 + status: StrictStr = Field(min_length=1, max_length=64) + target_tracking: ControlOverlayTargetTrackingV2 | None = None + presentation: Literal[True] = True + + @model_validator(mode="after") + def _validate_derived_fields(self) -> "ControlOverlayFrameV2": + if not isfinite(self.observed_at_monotonic_ms): + raise ValueError("observed_at_monotonic_ms must be finite") + if self.workflow_label != CONTROL_OVERLAY_WORKFLOW_LABEL_BY_MODE[self.mode]: + raise ValueError("workflow_label does not match mode") + if self.status != CONTROL_OVERLAY_STATUS_BY_PHASE[self.phase]: + raise ValueError("status does not match phase") + expected_state_id = control_overlay_state_id_v2( + visible=self.visible, + phase=self.phase, + mode=self.mode, + profile=self.profile, + step=self.step, + controls=self.controls, + target_tracking=self.target_tracking, + ) + if self.state_id != expected_state_id: + raise ValueError("state_id does not match the semantic V2 frame state") + return self + + @classmethod + def build( + cls, + *, + event_sequence: int, + observed_at_unix_ms: int, + observed_at_monotonic_ms: float, + visible: bool, + phase: ControlOverlayPhase, + mode: ControlOverlayMode, + profile: ControlOverlayProfile | None = None, + current_step: int | None = None, + total_steps: int | None = None, + pause: bool = False, + resume: bool = False, + stop: bool = False, + target_tracking: ControlOverlayTargetTrackingV2 | None = None, + ) -> "ControlOverlayFrameV2": + """Build a V2 frame without deriving or reconstructing target geometry.""" + + step = ControlOverlayStepV1(current=current_step, total=total_steps) + controls = ControlOverlayControlsV1( + pause=pause, + resume=resume, + stop=stop, + ) + return cls( + state_id=control_overlay_state_id_v2( + visible=visible, + phase=phase, + mode=mode, + profile=profile, + step=step, + controls=controls, + target_tracking=target_tracking, + ), + event_sequence=event_sequence, + observed_at_unix_ms=observed_at_unix_ms, + observed_at_monotonic_ms=observed_at_monotonic_ms, + visible=visible, + phase=phase, + workflow_label=CONTROL_OVERLAY_WORKFLOW_LABEL_BY_MODE[mode], + mode=mode, + profile=profile, + step=step, + controls=controls, + status=CONTROL_OVERLAY_STATUS_BY_PHASE[phase], + target_tracking=target_tracking, + ) + + def tracking_for_observation( + self, observation_hmac_sha256: str + ) -> ControlOverlayTargetTrackingV2 | None: + """Return tracking only for an exact private-observation HMAC match.""" + + target = self.target_tracking + if target is None or not isinstance( + target.binding, ControlOverlayObservationBindingV2 + ): + return None + if not compare_digest( + target.binding.observation_hmac_sha256, + observation_hmac_sha256, + ): + return None + return target + + +class ControlOverlayTimelineEventV2(_StrictContract): + at_ms: StrictInt = Field(ge=0) + media_frame_index: StrictInt = Field(ge=0) + frame: ControlOverlayFrameV2 + + +class ControlOverlayTimelineBindingV2(_StrictContract): + evidence_pack_id: StrictStr = Field( + min_length=1, + max_length=96, + pattern=r"^[a-z0-9][a-z0-9._-]{0,95}$", + ) + media_sha256: StrictStr = Field(pattern=r"^[a-f0-9]{64}$") + media_frame_count: StrictInt = Field(gt=0) + + +class ControlOverlayTimelineV2(_StrictContract): + """Exact runtime frames aligned to decoded frames of immutable media.""" + + model_config = ConfigDict(json_schema_extra=_TARGET_TIMELINE_JSON_SCHEMA_EXTRA) + + schema_version: Literal["openadapt.control-overlay-timeline/v2"] = ( + CONTROL_OVERLAY_TIMELINE_V2_SCHEMA + ) + data_classification: ControlOverlayDataClassification + evidence_pack_id: StrictStr = Field( + min_length=1, + max_length=96, + pattern=r"^[a-z0-9][a-z0-9._-]{0,95}$", + ) + media_sha256: StrictStr = Field(pattern=r"^[a-f0-9]{64}$") + media_frame_count: StrictInt = Field(gt=0) + duration_ms: StrictInt = Field(gt=0) + events: tuple[ControlOverlayTimelineEventV2, ...] = Field(min_length=1) + + @model_validator(mode="after") + def _validate_event_order_and_bindings(self) -> "ControlOverlayTimelineV2": + if self.events[0].at_ms != 0 or self.events[0].media_frame_index != 0: + raise ValueError("timeline must begin at 0 ms and media frame 0") + previous_at = -1 + previous_frame_index = -1 + previous_sequence = -1 + previous_monotonic = -1.0 + for event in self.events: + if event.at_ms <= previous_at or event.at_ms > self.duration_ms: + raise ValueError("timeline media offsets must increase within duration") + if ( + event.media_frame_index <= previous_frame_index + or event.media_frame_index >= self.media_frame_count + ): + raise ValueError( + "timeline media frame indexes must increase within frame count" + ) + if event.frame.event_sequence <= previous_sequence: + raise ValueError( + "timeline event_sequence values must strictly increase" + ) + if event.frame.observed_at_monotonic_ms < previous_monotonic: + raise ValueError("timeline monotonic timestamps cannot go backwards") + target = event.frame.target_tracking + if target is not None: + if not isinstance( + target.binding, + ControlOverlayMediaFrameBindingV2, + ): + raise ValueError( + "timeline target tracking requires a media-frame binding" + ) + binding = target.binding + if binding.media_sha256 != self.media_sha256: + raise ValueError("target tracking does not match timeline media") + if binding.frame_index != event.media_frame_index: + raise ValueError("target tracking does not match timeline frame") + previous_at = event.at_ms + previous_frame_index = event.media_frame_index + previous_sequence = event.frame.event_sequence + previous_monotonic = event.frame.observed_at_monotonic_ms + return self + + def assert_binding(self, binding: ControlOverlayTimelineBindingV2) -> None: + """Refuse composition with different evidence or decoded media.""" + + if self.evidence_pack_id != binding.evidence_pack_id: + raise ValueError("timeline belongs to a different evidence pack") + if self.media_sha256 != binding.media_sha256: + raise ValueError("timeline does not match the exact media digest") + if self.media_frame_count != binding.media_frame_count: + raise ValueError("timeline does not match the decoded media frame count") + + def event_for_media_frame( + self, media_frame_index: int + ) -> ControlOverlayTimelineEventV2 | None: + """Return only an event bound to the exact decoded media frame.""" + + if isinstance(media_frame_index, bool) or not isinstance( + media_frame_index, int + ): + raise TypeError("media_frame_index must be an integer") + low = 0 + high = len(self.events) - 1 + while low <= high: + midpoint = (low + high) // 2 + event = self.events[midpoint] + if event.media_frame_index == media_frame_index: + return event + if event.media_frame_index < media_frame_index: + low = midpoint + 1 + else: + high = midpoint - 1 + return None + + def tracking_for_media_frame( + self, media_frame_index: int + ) -> ControlOverlayTargetTrackingV2 | None: + """Return tracking only at its exact retained decoded frame.""" + + event = self.event_for_media_frame(media_frame_index) + if event is None: + return None + target = event.frame.target_tracking + if target is None or not isinstance( + target.binding, + ControlOverlayMediaFrameBindingV2, + ): + return None + return target + + +def build_control_overlay_timeline_v2( + *, + data_classification: ControlOverlayDataClassification, + evidence_pack_id: str, + media_sha256: str, + media_frame_count: int, + duration_ms: int, + events: Sequence[ControlOverlayTimelineEventV2], +) -> ControlOverlayTimelineV2: + """Build and validate an immutable exactly aligned V2 timeline.""" + + return ControlOverlayTimelineV2( + data_classification=data_classification, + evidence_pack_id=evidence_pack_id, + media_sha256=media_sha256, + media_frame_count=media_frame_count, + duration_ms=duration_ms, + events=tuple(events), + ) diff --git a/openadapt_types/schemas/control-overlay-frame-v2.json b/openadapt_types/schemas/control-overlay-frame-v2.json new file mode 100644 index 0000000..ed8efb8 --- /dev/null +++ b/openadapt_types/schemas/control-overlay-frame-v2.json @@ -0,0 +1,406 @@ +{ + "$defs": { + "ControlOverlayControlsV1": { + "additionalProperties": false, + "properties": { + "pause": { + "default": false, + "title": "Pause", + "type": "boolean" + }, + "resume": { + "default": false, + "title": "Resume", + "type": "boolean" + }, + "stop": { + "default": false, + "title": "Stop", + "type": "boolean" + } + }, + "title": "ControlOverlayControlsV1", + "type": "object" + }, + "ControlOverlayMediaFrameBindingV2": { + "additionalProperties": false, + "description": "Binding to one exact decoded frame of one exact immutable media file.", + "properties": { + "frame_index": { + "minimum": 0, + "title": "Frame Index", + "type": "integer" + }, + "kind": { + "const": "media_frame", + "default": "media_frame", + "title": "Kind", + "type": "string" + }, + "media_sha256": { + "pattern": "^[a-f0-9]{64}$", + "title": "Media Sha256", + "type": "string" + } + }, + "required": [ + "media_sha256", + "frame_index" + ], + "title": "ControlOverlayMediaFrameBindingV2", + "type": "object" + }, + "ControlOverlayMode": { + "enum": [ + "demonstration", + "replay", + "governed", + "managed" + ], + "title": "ControlOverlayMode", + "type": "string" + }, + "ControlOverlayNormalizedRectV2": { + "additionalProperties": false, + "description": "Axis-aligned rectangle normalized to the source top-level viewport.", + "properties": { + "height": { + "exclusiveMinimum": 0, + "maximum": 1, + "title": "Height", + "type": "number" + }, + "width": { + "exclusiveMinimum": 0, + "maximum": 1, + "title": "Width", + "type": "number" + }, + "x": { + "maximum": 1, + "minimum": 0, + "title": "X", + "type": "number" + }, + "y": { + "maximum": 1, + "minimum": 0, + "title": "Y", + "type": "number" + } + }, + "required": [ + "x", + "y", + "width", + "height" + ], + "title": "ControlOverlayNormalizedRectV2", + "type": "object" + }, + "ControlOverlayObservationBindingV2": { + "additionalProperties": false, + "description": "Opaque binding to one exact private runtime observation.\n\nThe digest must be HMAC-SHA256 over a canonical private observation ID,\ndomain-separated for this contract and keyed with a run/export-scoped\nsecret. A raw screenshot, frame, or observation SHA-256 is not valid here:\nit can be linkable across evidence boundaries.", + "properties": { + "kind": { + "const": "observation_hmac_sha256", + "default": "observation_hmac_sha256", + "title": "Kind", + "type": "string" + }, + "observation_hmac_sha256": { + "pattern": "^[a-f0-9]{64}$", + "title": "Observation Hmac Sha256", + "type": "string" + } + }, + "required": [ + "observation_hmac_sha256" + ], + "title": "ControlOverlayObservationBindingV2", + "type": "object" + }, + "ControlOverlayPhase": { + "description": "Canonical cross-surface execution and terminal states.", + "enum": [ + "idle", + "observing", + "recording", + "executing", + "pausing", + "paused", + "resuming", + "stopping", + "verifying", + "verified", + "completed_unverified", + "halted", + "failed", + "rolled_back" + ], + "title": "ControlOverlayPhase", + "type": "string" + }, + "ControlOverlayProfile": { + "enum": [ + "demo", + "standard", + "regulated" + ], + "title": "ControlOverlayProfile", + "type": "string" + }, + "ControlOverlaySourceViewportV2": { + "additionalProperties": false, + "description": "Browser top-level CSS viewport used to resolve the target.\n\nNative and RDP device-pixel dimensions must not be placed in these fields.\nThose coordinate spaces require a separate future contract.", + "properties": { + "device_pixel_ratio": { + "exclusiveMinimum": 0, + "maximum": 16, + "title": "Device Pixel Ratio", + "type": "number" + }, + "height_css_px": { + "exclusiveMinimum": 0, + "maximum": 32768, + "title": "Height Css Px", + "type": "integer" + }, + "width_css_px": { + "exclusiveMinimum": 0, + "maximum": 32768, + "title": "Width Css Px", + "type": "integer" + } + }, + "required": [ + "width_css_px", + "height_css_px", + "device_pixel_ratio" + ], + "title": "ControlOverlaySourceViewportV2", + "type": "object" + }, + "ControlOverlayStepV1": { + "additionalProperties": false, + "properties": { + "current": { + "anyOf": [ + { + "minimum": 1, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Current" + }, + "total": { + "anyOf": [ + { + "minimum": 1, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Total" + } + }, + "title": "ControlOverlayStepV1", + "type": "object" + }, + "ControlOverlayTargetActionKind": { + "description": "Closed, presentation-only kinds for actions with a visible target.", + "enum": [ + "click", + "double_click", + "right_click", + "drag", + "type", + "select", + "toggle", + "invoke", + "expand_collapse", + "scroll", + "hover" + ], + "title": "ControlOverlayTargetActionKind", + "type": "string" + }, + "ControlOverlayTargetTrackingV2": { + "additionalProperties": false, + "description": "Exact browser CSS-viewport geometry, never resolution evidence.", + "properties": { + "action_kind": { + "anyOf": [ + { + "$ref": "#/$defs/ControlOverlayTargetActionKind" + }, + { + "type": "null" + } + ], + "default": null + }, + "binding": { + "discriminator": { + "mapping": { + "media_frame": "#/$defs/ControlOverlayMediaFrameBindingV2", + "observation_hmac_sha256": "#/$defs/ControlOverlayObservationBindingV2" + }, + "propertyName": "kind" + }, + "oneOf": [ + { + "$ref": "#/$defs/ControlOverlayObservationBindingV2" + }, + { + "$ref": "#/$defs/ControlOverlayMediaFrameBindingV2" + } + ], + "title": "Binding" + }, + "coordinate_space": { + "const": "top_level_viewport_normalized", + "default": "top_level_viewport_normalized", + "description": "Normalized browser top-level CSS viewport; not native or RDP device-pixel coordinates", + "title": "Coordinate Space", + "type": "string" + }, + "rect": { + "$ref": "#/$defs/ControlOverlayNormalizedRectV2" + }, + "source_viewport": { + "$ref": "#/$defs/ControlOverlaySourceViewportV2" + } + }, + "required": [ + "rect", + "source_viewport", + "binding" + ], + "title": "ControlOverlayTargetTrackingV2", + "type": "object", + "x-openadapt-missing-binding-behavior": "omit_target", + "x-openadapt-render-only-on-exact-binding": true, + "x-openadapt-renderer-mapping": "actual_content_box", + "x-openadapt-runtime-resolution-replay": false, + "x-openadapt-source-scope": "browser_top_level_css_viewport" + }, + "ControlOverlayWorkflowLabel": { + "description": "Closed presentation labels; user-authored workflow names are excluded.", + "enum": [ + "Workflow demonstration", + "Workflow replay", + "Governed workflow", + "Managed workflow" + ], + "title": "ControlOverlayWorkflowLabel", + "type": "string" + } + }, + "additionalProperties": false, + "description": "Control state plus optional exactly bound target presentation geometry.", + "properties": { + "controls": { + "$ref": "#/$defs/ControlOverlayControlsV1" + }, + "event_sequence": { + "minimum": 0, + "title": "Event Sequence", + "type": "integer" + }, + "mode": { + "$ref": "#/$defs/ControlOverlayMode" + }, + "observed_at_monotonic_ms": { + "minimum": 0, + "title": "Observed At Monotonic Ms", + "type": "number" + }, + "observed_at_unix_ms": { + "minimum": 0, + "title": "Observed At Unix Ms", + "type": "integer" + }, + "phase": { + "$ref": "#/$defs/ControlOverlayPhase" + }, + "presentation": { + "const": true, + "default": true, + "title": "Presentation", + "type": "boolean" + }, + "profile": { + "anyOf": [ + { + "$ref": "#/$defs/ControlOverlayProfile" + }, + { + "type": "null" + } + ], + "default": null + }, + "schema_version": { + "const": "openadapt.control-overlay-frame/v2", + "default": "openadapt.control-overlay-frame/v2", + "title": "Schema Version", + "type": "string" + }, + "state_id": { + "maxLength": 280, + "minLength": 1, + "title": "State Id", + "type": "string" + }, + "status": { + "maxLength": 64, + "minLength": 1, + "title": "Status", + "type": "string" + }, + "step": { + "$ref": "#/$defs/ControlOverlayStepV1" + }, + "target_tracking": { + "anyOf": [ + { + "$ref": "#/$defs/ControlOverlayTargetTrackingV2" + }, + { + "type": "null" + } + ], + "default": null + }, + "visible": { + "title": "Visible", + "type": "boolean" + }, + "workflow_label": { + "$ref": "#/$defs/ControlOverlayWorkflowLabel" + } + }, + "required": [ + "state_id", + "event_sequence", + "observed_at_unix_ms", + "observed_at_monotonic_ms", + "visible", + "phase", + "workflow_label", + "mode", + "step", + "controls", + "status" + ], + "title": "ControlOverlayFrameV2", + "type": "object" +} diff --git a/openadapt_types/schemas/control-overlay-timeline-v2.json b/openadapt_types/schemas/control-overlay-timeline-v2.json new file mode 100644 index 0000000..71b3bb6 --- /dev/null +++ b/openadapt_types/schemas/control-overlay-timeline-v2.json @@ -0,0 +1,497 @@ +{ + "$defs": { + "ControlOverlayControlsV1": { + "additionalProperties": false, + "properties": { + "pause": { + "default": false, + "title": "Pause", + "type": "boolean" + }, + "resume": { + "default": false, + "title": "Resume", + "type": "boolean" + }, + "stop": { + "default": false, + "title": "Stop", + "type": "boolean" + } + }, + "title": "ControlOverlayControlsV1", + "type": "object" + }, + "ControlOverlayDataClassification": { + "description": "The only classifications allowed to leave a private evidence boundary.", + "enum": [ + "synthetic", + "sanitized_public" + ], + "title": "ControlOverlayDataClassification", + "type": "string" + }, + "ControlOverlayFrameV2": { + "additionalProperties": false, + "description": "Control state plus optional exactly bound target presentation geometry.", + "properties": { + "controls": { + "$ref": "#/$defs/ControlOverlayControlsV1" + }, + "event_sequence": { + "minimum": 0, + "title": "Event Sequence", + "type": "integer" + }, + "mode": { + "$ref": "#/$defs/ControlOverlayMode" + }, + "observed_at_monotonic_ms": { + "minimum": 0, + "title": "Observed At Monotonic Ms", + "type": "number" + }, + "observed_at_unix_ms": { + "minimum": 0, + "title": "Observed At Unix Ms", + "type": "integer" + }, + "phase": { + "$ref": "#/$defs/ControlOverlayPhase" + }, + "presentation": { + "const": true, + "default": true, + "title": "Presentation", + "type": "boolean" + }, + "profile": { + "anyOf": [ + { + "$ref": "#/$defs/ControlOverlayProfile" + }, + { + "type": "null" + } + ], + "default": null + }, + "schema_version": { + "const": "openadapt.control-overlay-frame/v2", + "default": "openadapt.control-overlay-frame/v2", + "title": "Schema Version", + "type": "string" + }, + "state_id": { + "maxLength": 280, + "minLength": 1, + "title": "State Id", + "type": "string" + }, + "status": { + "maxLength": 64, + "minLength": 1, + "title": "Status", + "type": "string" + }, + "step": { + "$ref": "#/$defs/ControlOverlayStepV1" + }, + "target_tracking": { + "anyOf": [ + { + "$ref": "#/$defs/ControlOverlayTargetTrackingV2" + }, + { + "type": "null" + } + ], + "default": null + }, + "visible": { + "title": "Visible", + "type": "boolean" + }, + "workflow_label": { + "$ref": "#/$defs/ControlOverlayWorkflowLabel" + } + }, + "required": [ + "state_id", + "event_sequence", + "observed_at_unix_ms", + "observed_at_monotonic_ms", + "visible", + "phase", + "workflow_label", + "mode", + "step", + "controls", + "status" + ], + "title": "ControlOverlayFrameV2", + "type": "object" + }, + "ControlOverlayMediaFrameBindingV2": { + "additionalProperties": false, + "description": "Binding to one exact decoded frame of one exact immutable media file.", + "properties": { + "frame_index": { + "minimum": 0, + "title": "Frame Index", + "type": "integer" + }, + "kind": { + "const": "media_frame", + "default": "media_frame", + "title": "Kind", + "type": "string" + }, + "media_sha256": { + "pattern": "^[a-f0-9]{64}$", + "title": "Media Sha256", + "type": "string" + } + }, + "required": [ + "media_sha256", + "frame_index" + ], + "title": "ControlOverlayMediaFrameBindingV2", + "type": "object" + }, + "ControlOverlayMode": { + "enum": [ + "demonstration", + "replay", + "governed", + "managed" + ], + "title": "ControlOverlayMode", + "type": "string" + }, + "ControlOverlayNormalizedRectV2": { + "additionalProperties": false, + "description": "Axis-aligned rectangle normalized to the source top-level viewport.", + "properties": { + "height": { + "exclusiveMinimum": 0, + "maximum": 1, + "title": "Height", + "type": "number" + }, + "width": { + "exclusiveMinimum": 0, + "maximum": 1, + "title": "Width", + "type": "number" + }, + "x": { + "maximum": 1, + "minimum": 0, + "title": "X", + "type": "number" + }, + "y": { + "maximum": 1, + "minimum": 0, + "title": "Y", + "type": "number" + } + }, + "required": [ + "x", + "y", + "width", + "height" + ], + "title": "ControlOverlayNormalizedRectV2", + "type": "object" + }, + "ControlOverlayObservationBindingV2": { + "additionalProperties": false, + "description": "Opaque binding to one exact private runtime observation.\n\nThe digest must be HMAC-SHA256 over a canonical private observation ID,\ndomain-separated for this contract and keyed with a run/export-scoped\nsecret. A raw screenshot, frame, or observation SHA-256 is not valid here:\nit can be linkable across evidence boundaries.", + "properties": { + "kind": { + "const": "observation_hmac_sha256", + "default": "observation_hmac_sha256", + "title": "Kind", + "type": "string" + }, + "observation_hmac_sha256": { + "pattern": "^[a-f0-9]{64}$", + "title": "Observation Hmac Sha256", + "type": "string" + } + }, + "required": [ + "observation_hmac_sha256" + ], + "title": "ControlOverlayObservationBindingV2", + "type": "object" + }, + "ControlOverlayPhase": { + "description": "Canonical cross-surface execution and terminal states.", + "enum": [ + "idle", + "observing", + "recording", + "executing", + "pausing", + "paused", + "resuming", + "stopping", + "verifying", + "verified", + "completed_unverified", + "halted", + "failed", + "rolled_back" + ], + "title": "ControlOverlayPhase", + "type": "string" + }, + "ControlOverlayProfile": { + "enum": [ + "demo", + "standard", + "regulated" + ], + "title": "ControlOverlayProfile", + "type": "string" + }, + "ControlOverlaySourceViewportV2": { + "additionalProperties": false, + "description": "Browser top-level CSS viewport used to resolve the target.\n\nNative and RDP device-pixel dimensions must not be placed in these fields.\nThose coordinate spaces require a separate future contract.", + "properties": { + "device_pixel_ratio": { + "exclusiveMinimum": 0, + "maximum": 16, + "title": "Device Pixel Ratio", + "type": "number" + }, + "height_css_px": { + "exclusiveMinimum": 0, + "maximum": 32768, + "title": "Height Css Px", + "type": "integer" + }, + "width_css_px": { + "exclusiveMinimum": 0, + "maximum": 32768, + "title": "Width Css Px", + "type": "integer" + } + }, + "required": [ + "width_css_px", + "height_css_px", + "device_pixel_ratio" + ], + "title": "ControlOverlaySourceViewportV2", + "type": "object" + }, + "ControlOverlayStepV1": { + "additionalProperties": false, + "properties": { + "current": { + "anyOf": [ + { + "minimum": 1, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Current" + }, + "total": { + "anyOf": [ + { + "minimum": 1, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Total" + } + }, + "title": "ControlOverlayStepV1", + "type": "object" + }, + "ControlOverlayTargetActionKind": { + "description": "Closed, presentation-only kinds for actions with a visible target.", + "enum": [ + "click", + "double_click", + "right_click", + "drag", + "type", + "select", + "toggle", + "invoke", + "expand_collapse", + "scroll", + "hover" + ], + "title": "ControlOverlayTargetActionKind", + "type": "string" + }, + "ControlOverlayTargetTrackingV2": { + "additionalProperties": false, + "description": "Exact browser CSS-viewport geometry, never resolution evidence.", + "properties": { + "action_kind": { + "anyOf": [ + { + "$ref": "#/$defs/ControlOverlayTargetActionKind" + }, + { + "type": "null" + } + ], + "default": null + }, + "binding": { + "discriminator": { + "mapping": { + "media_frame": "#/$defs/ControlOverlayMediaFrameBindingV2", + "observation_hmac_sha256": "#/$defs/ControlOverlayObservationBindingV2" + }, + "propertyName": "kind" + }, + "oneOf": [ + { + "$ref": "#/$defs/ControlOverlayObservationBindingV2" + }, + { + "$ref": "#/$defs/ControlOverlayMediaFrameBindingV2" + } + ], + "title": "Binding" + }, + "coordinate_space": { + "const": "top_level_viewport_normalized", + "default": "top_level_viewport_normalized", + "description": "Normalized browser top-level CSS viewport; not native or RDP device-pixel coordinates", + "title": "Coordinate Space", + "type": "string" + }, + "rect": { + "$ref": "#/$defs/ControlOverlayNormalizedRectV2" + }, + "source_viewport": { + "$ref": "#/$defs/ControlOverlaySourceViewportV2" + } + }, + "required": [ + "rect", + "source_viewport", + "binding" + ], + "title": "ControlOverlayTargetTrackingV2", + "type": "object", + "x-openadapt-missing-binding-behavior": "omit_target", + "x-openadapt-render-only-on-exact-binding": true, + "x-openadapt-renderer-mapping": "actual_content_box", + "x-openadapt-runtime-resolution-replay": false, + "x-openadapt-source-scope": "browser_top_level_css_viewport" + }, + "ControlOverlayTimelineEventV2": { + "additionalProperties": false, + "properties": { + "at_ms": { + "minimum": 0, + "title": "At Ms", + "type": "integer" + }, + "frame": { + "$ref": "#/$defs/ControlOverlayFrameV2" + }, + "media_frame_index": { + "minimum": 0, + "title": "Media Frame Index", + "type": "integer" + } + }, + "required": [ + "at_ms", + "media_frame_index", + "frame" + ], + "title": "ControlOverlayTimelineEventV2", + "type": "object" + }, + "ControlOverlayWorkflowLabel": { + "description": "Closed presentation labels; user-authored workflow names are excluded.", + "enum": [ + "Workflow demonstration", + "Workflow replay", + "Governed workflow", + "Managed workflow" + ], + "title": "ControlOverlayWorkflowLabel", + "type": "string" + } + }, + "additionalProperties": false, + "description": "Exact runtime frames aligned to decoded frames of immutable media.", + "properties": { + "data_classification": { + "$ref": "#/$defs/ControlOverlayDataClassification" + }, + "duration_ms": { + "exclusiveMinimum": 0, + "title": "Duration Ms", + "type": "integer" + }, + "events": { + "items": { + "$ref": "#/$defs/ControlOverlayTimelineEventV2" + }, + "minItems": 1, + "title": "Events", + "type": "array" + }, + "evidence_pack_id": { + "maxLength": 96, + "minLength": 1, + "pattern": "^[a-z0-9][a-z0-9._-]{0,95}$", + "title": "Evidence Pack Id", + "type": "string" + }, + "media_frame_count": { + "exclusiveMinimum": 0, + "title": "Media Frame Count", + "type": "integer" + }, + "media_sha256": { + "pattern": "^[a-f0-9]{64}$", + "title": "Media Sha256", + "type": "string" + }, + "schema_version": { + "const": "openadapt.control-overlay-timeline/v2", + "default": "openadapt.control-overlay-timeline/v2", + "title": "Schema Version", + "type": "string" + } + }, + "required": [ + "data_classification", + "evidence_pack_id", + "media_sha256", + "media_frame_count", + "duration_ms", + "events" + ], + "title": "ControlOverlayTimelineV2", + "type": "object", + "x-openadapt-missing-frame-behavior": "omit_target", + "x-openadapt-target-validity": "exact_decoded_media_frame_only" +} diff --git a/scripts/export_control_overlay_schemas.py b/scripts/export_control_overlay_schemas.py index 1788239..0f6cafe 100644 --- a/scripts/export_control_overlay_schemas.py +++ b/scripts/export_control_overlay_schemas.py @@ -5,13 +5,20 @@ import json from pathlib import Path -from openadapt_types import ControlOverlayFrameV1, ControlOverlayTimelineV1 +from openadapt_types import ( + ControlOverlayFrameV1, + ControlOverlayFrameV2, + ControlOverlayTimelineV1, + ControlOverlayTimelineV2, +) ROOT = Path(__file__).resolve().parents[1] SCHEMA_DIR = ROOT / "openadapt_types" / "schemas" SCHEMAS = { "control-overlay-frame-v1.json": ControlOverlayFrameV1, "control-overlay-timeline-v1.json": ControlOverlayTimelineV1, + "control-overlay-frame-v2.json": ControlOverlayFrameV2, + "control-overlay-timeline-v2.json": ControlOverlayTimelineV2, } diff --git a/tests/test_control_overlay_tracking.py b/tests/test_control_overlay_tracking.py new file mode 100644 index 0000000..9fd084f --- /dev/null +++ b/tests/test_control_overlay_tracking.py @@ -0,0 +1,194 @@ +"""Semantic and adversarial tests for exact overlay target tracking.""" + +import json +from importlib.resources import files + +import pytest +from pydantic import ValidationError + +from openadapt_types import ( + CONTROL_OVERLAY_FRAME_V2_SCHEMA, + CONTROL_OVERLAY_TIMELINE_V2_SCHEMA, + ControlOverlayDataClassification, + ControlOverlayFrameV2, + ControlOverlayMediaFrameBindingV2, + ControlOverlayMode, + ControlOverlayNormalizedRectV2, + ControlOverlayObservationBindingV2, + ControlOverlayPhase, + ControlOverlayProfile, + ControlOverlaySourceViewportV2, + ControlOverlayTargetActionKind, + ControlOverlayTargetTrackingV2, + ControlOverlayTimelineEventV2, + ControlOverlayTimelineV2, +) + +DIGEST = "a" * 64 +OBSERVATION_HMAC = "b" * 64 + + +def _target(*, frame_index: int | None = None) -> ControlOverlayTargetTrackingV2: + binding = ( + ControlOverlayObservationBindingV2(observation_hmac_sha256=OBSERVATION_HMAC) + if frame_index is None + else ControlOverlayMediaFrameBindingV2( + media_sha256=DIGEST, + frame_index=frame_index, + ) + ) + return ControlOverlayTargetTrackingV2( + rect=ControlOverlayNormalizedRectV2( + x=0.1, + y=0.2, + width=0.3, + height=0.1, + ), + source_viewport=ControlOverlaySourceViewportV2( + width_css_px=1280, + height_css_px=720, + device_pixel_ratio=2.0, + ), + binding=binding, + action_kind=ControlOverlayTargetActionKind.CLICK, + ) + + +def _frame( + sequence: int, target: ControlOverlayTargetTrackingV2 | None +) -> ControlOverlayFrameV2: + return ControlOverlayFrameV2.build( + event_sequence=sequence, + observed_at_unix_ms=1_785_000_000_000 + sequence, + observed_at_monotonic_ms=1000.0 + sequence, + visible=True, + phase=ControlOverlayPhase.EXECUTING, + mode=ControlOverlayMode.GOVERNED, + profile=ControlOverlayProfile.STANDARD, + current_step=sequence, + total_steps=3, + target_tracking=target, + ) + + +def test_v2_is_additive_and_exact_observation_matching_is_fail_closed() -> None: + frame = _frame(1, _target()) + + assert frame.schema_version == CONTROL_OVERLAY_FRAME_V2_SCHEMA + assert frame.tracking_for_observation(OBSERVATION_HMAC) == frame.target_tracking + assert frame.tracking_for_observation("c" * 64) is None + assert ":target-" in frame.state_id + assert frame.state_id != _frame(1, None).state_id + + +def test_geometry_and_viewport_refuse_non_finite_or_out_of_bounds_values() -> None: + with pytest.raises(ValidationError, match="exceeds viewport width"): + ControlOverlayNormalizedRectV2(x=0.8, y=0.0, width=0.3, height=0.1) + with pytest.raises(ValidationError): + ControlOverlayNormalizedRectV2(x=float("nan"), y=0.0, width=0.3, height=0.1) + with pytest.raises(ValidationError, match="less than or equal to 32768"): + ControlOverlaySourceViewportV2( + width_css_px=32769, + height_css_px=720, + device_pixel_ratio=2.0, + ) + + +def test_contract_rejects_selectors_content_and_raw_observation_hash_fields() -> None: + payload = _target().model_dump(mode="json") + for forbidden in ( + "selector", + "accessible_name", + "typed_value", + "url", + "screenshot", + "viewport_to_presentation", + ): + candidate = dict(payload) + candidate[forbidden] = "private" + with pytest.raises(ValidationError, match="Extra inputs are not permitted"): + ControlOverlayTargetTrackingV2.model_validate(candidate) + + binding = payload["binding"] + binding["observation_sha256"] = binding.pop("observation_hmac_sha256") + with pytest.raises(ValidationError): + ControlOverlayTargetTrackingV2.model_validate(payload) + + +def test_timeline_exposes_tracking_only_on_the_exact_decoded_frame() -> None: + timeline = ControlOverlayTimelineV2( + data_classification=ControlOverlayDataClassification.SYNTHETIC, + evidence_pack_id="reference-v2", + media_sha256=DIGEST, + media_frame_count=120, + duration_ms=4000, + events=( + ControlOverlayTimelineEventV2( + at_ms=0, + media_frame_index=0, + frame=_frame(1, _target(frame_index=0)), + ), + ControlOverlayTimelineEventV2( + at_ms=1000, + media_frame_index=30, + frame=_frame(2, _target(frame_index=30)), + ), + ), + ) + + assert timeline.schema_version == CONTROL_OVERLAY_TIMELINE_V2_SCHEMA + assert timeline.tracking_for_media_frame(30) is not None + assert timeline.tracking_for_media_frame(29) is None + assert timeline.event_for_media_frame(30) == timeline.events[1] + + payload = timeline.model_dump(mode="json") + payload["events"][1]["frame"] = _frame(2, _target(frame_index=29)).model_dump( + mode="json" + ) + with pytest.raises(ValidationError, match="does not match timeline frame"): + ControlOverlayTimelineV2.model_validate(payload) + + +def test_timeline_refuses_live_observation_hmac_target_tracking() -> None: + with pytest.raises(ValidationError, match="requires a media-frame binding"): + ControlOverlayTimelineV2( + data_classification=ControlOverlayDataClassification.SYNTHETIC, + evidence_pack_id="reference-v2", + media_sha256=DIGEST, + media_frame_count=120, + duration_ms=4000, + events=( + ControlOverlayTimelineEventV2( + at_ms=0, + media_frame_index=0, + frame=_frame(1, None), + ), + ControlOverlayTimelineEventV2( + at_ms=1000, + media_frame_index=30, + frame=_frame(2, _target()), + ), + ), + ) + + +def test_v2_json_schemas_are_deterministic_packaged_contracts() -> None: + packaged = files("openadapt_types.schemas") + for filename, model in ( + ("control-overlay-frame-v2.json", ControlOverlayFrameV2), + ("control-overlay-timeline-v2.json", ControlOverlayTimelineV2), + ): + schema = model.model_json_schema() + assert schema["additionalProperties"] is False + assert "selector" not in json.dumps(schema).lower() + assert json.loads(packaged.joinpath(filename).read_text()) == schema + assert ( + ControlOverlayTimelineV2.model_json_schema()["x-openadapt-target-validity"] + == "exact_decoded_media_frame_only" + ) + assert ( + ControlOverlayFrameV2.model_json_schema()["$defs"][ + "ControlOverlayTargetTrackingV2" + ]["x-openadapt-source-scope"] + == "browser_top_level_css_viewport" + )