diff --git a/benchmark/citrix_ica_hdx/.gitignore b/benchmark/citrix_ica_hdx/.gitignore new file mode 100644 index 00000000..9d931c43 --- /dev/null +++ b/benchmark/citrix_ica_hdx/.gitignore @@ -0,0 +1 @@ +work/ diff --git a/benchmark/citrix_ica_hdx/README.md b/benchmark/citrix_ica_hdx/README.md new file mode 100644 index 00000000..e0482e87 --- /dev/null +++ b/benchmark/citrix_ica_hdx/README.md @@ -0,0 +1,119 @@ +# Deterministic ICA/HDX stand-in qualification (roadmap Section 10) + +A **deterministic, license-free stand-in** for a real Citrix environment: a +synthetic in-process fixture that *reproduces* ICA/HDX conditions and qualifies +the pixel/no-DOM actuation contract of `CitrixWorkspaceBackend` against the full +Section 10 condition matrix. + +> **HONEST LABEL (non-negotiable).** This is a **DETERMINISTIC STAND-IN** — a +> synthetic fixture that reproduces ICA/HDX *conditions*. It is **NOT real +> Citrix ICA/HDX.** It does **not** exercise HDX codecs, ICA compression, or the +> real Workspace-client transport. No result here is real-protocol acceptance. +> Real ICA/HDX evidence remains **pending the customer-environment (Accuro) +> lane** (see [`../citrix_workspace/README.md`](../citrix_workspace/README.md) +> "Real ICA/HDX release gate" and [`../../docs/desktop/CITRIX_PIXEL.md`](../../docs/desktop/CITRIX_PIXEL.md)). + +## What it is + +`run_ica_hdx_qualification.py` drives the **unmodified** +`openadapt_flow.backends.citrix_workspace.CitrixWorkspaceBackend` (a +`RemoteDisplayBackend` preset) through every Section 10 ICA/HDX condition, each +reproduced as a reproducible synthetic scenario in `fixture.py` with an explicit +**pass** (correct, out-of-band-verified actuation) or **halt** (safe refusal) +expectation. Only the backend's real `WindowClient` seam is synthetic +(`SyntheticIcaWindowClient`); the backend, its frame-freshness lease, DPI/scale +calibration, focus/occlusion binding, input-trust gate, one-shot actuation +lease, and readiness/identity gating are the shipping code. + +It runs **fully in-process** — no Docker, no network, no Playwright — so the +entire matrix is qualified deterministically and stays green in CI +(`tests/test_ica_hdx_qualification.py`). + +## Section 10 condition matrix covered + +Each condition is a reproducible scenario with a pass/halt expectation: + +| Condition | pass scenario(s) | halt scenario(s) | +|---|---|---| +| session launch + application readiness | `session_launch_ready` | `application_not_ready` | +| reconnect / roaming | | `reconnect_roaming_identity_change` | +| session lock / unlock | `session_unlock_recovery` | `session_lock` | +| window minimize / occlusion / move / resize | | `window_minimize`, `window_occlusion`, `window_move_after_acquire`, `window_resize_after_acquire` | +| DPI + scaling changes | `dpi_scale_consistent` | `dpi_anisotropic_uncalibrated` | +| single and multimonitor geometry | `single_monitor_geometry`, `multimonitor_secondary_offset` | `multimonitor_ambiguous_window` | +| display compression / codec artifacts | `codec_artifacts_mild_legible` | `codec_artifacts_severe_illegible` | +| latency / jitter / packet-loss / delayed-frames | `delayed_frame_settles` | `stale_frame_latency`, `frame_never_settles` | +| keyboard-layout / IME | `keyboard_named_key_ok` | `ime_unmapped_key` | +| clipboard restrictions | | `clipboard_restricted_paste` | +| focus theft | | `focus_theft_after_acquire` | +| unexpected dialogs / overlays | | `unexpected_dialog_overlay` | +| ambiguous visual identity | | `unverifiable_application_identity` | +| stale-frame detection | | `stale_frame_latency` | +| uncertain submission (no blind retry) | | `uncertain_submission_no_blind_retry` | +| duplicate prevention | | `duplicate_write_prevention_one_shot` | +| persisted-state readback (out-of-band effect verification) | `persisted_state_readback` | `optimistic_banner_effect_refused` | + +## Enforcement verified on every scenario + +- every actuation uses a **fresh frame** (frame-freshness lease); +- actuation stays **bound to the authorized window and session**; +- the target is **re-resolved immediately before acting**; +- a **one-shot actuation lease** is consumed exactly once (no double-fire); +- DPI/scale is **refused when anisotropic/uncalibrated**; +- **focus/occlusion binding** is enforced before every input edge; +- **readiness + identity** are gated on the fresh actuation frame; +- effect verification is **out of band** (an independent record, never the + on-screen "Saved" banner); +- **zero model calls** on every path — healthy and refusal. + +## Reviewable volatile-region masks + +`fixture.py` declares a reviewable `VolatileMaskSpec`: which regions are volatile +(may be masked from continuity comparison — here only the remote clock chrome) +and which are **protected** and must never be masked (**target, actionability, +identity, workflow-state, effect-relevant** regions). `check_masks_reviewable()` +proves no volatile mask overlaps a protected region; the campaign fails if it +does, and `tests/test_ica_hdx_qualification.py` covers both the safe default and +a rejected bad spec. The backend's default remains conservative full-frame +decoded-RGB continuity; any real relaxation is a reviewed +application/environment artifact, never a permissive global default. + +## Separate status dimensions (never one "Available") + +`status_manifest.json` publishes each dimension separately: + +- `backend_shipped` — **shipped** (qualified against this stand-in); +- `installed_driver_available` — **shipped host clients** (Mac/Win drivers; + live capture/input needs per-host trust at deployment); +- `real_protocol_environment_evidence` — **pending** (no real ICA/HDX); +- `managed_execution_available` — **pending**; +- `customer_controlled_execution_available` — **pending**; +- `exact_application_qualification_available` — **pending**; +- `deterministic_standin_qualification` — **qualified** (this campaign, a + stand-in, NOT real ICA/HDX). + +## What this proves — and does NOT prove + +**Proves:** the Citrix backend's pixel/no-DOM actuation contract holds across the +whole Section 10 ICA/HDX condition matrix — on every condition it either +delivers a correct, independently verified actuation or safely halts, with zero +silent incorrect successes, zero healthy over-halts, and zero model calls. + +**Does NOT prove:** anything about real Citrix ICA/HDX. There are no HDX codecs, +no ICA compression, no real Workspace-client transport, and no exact +published-application (e.g. Accuro) qualification here. The synthetic +"compression/codec artifacts" are a PIL-generated degradation, not an HDX/ +Thinwire bitstream. Real-protocol acceptance is the separate customer-environment +release gate. + +## Run + +```bash +python3 benchmark/citrix_ica_hdx/run_ica_hdx_qualification.py \ + --output benchmark/citrix_ica_hdx/results.json \ + --status-output benchmark/citrix_ica_hdx/status_manifest.json +# deterministic, ~2s; exits non-zero unless all scenarios pass with +# 0 silent-incorrect-successes, 0 over-halts, 0 model calls. + +python3 -m pytest tests/test_ica_hdx_qualification.py -q +``` diff --git a/benchmark/citrix_ica_hdx/fixture.py b/benchmark/citrix_ica_hdx/fixture.py new file mode 100644 index 00000000..95ef898b --- /dev/null +++ b/benchmark/citrix_ica_hdx/fixture.py @@ -0,0 +1,404 @@ +#!/usr/bin/env python3 +"""Deterministic synthetic ICA/HDX stand-in fixture (NOT real Citrix ICA/HDX). + +This module renders a *synthetic* clinical "MockMed" application frame and +presents it to the UNMODIFIED +:class:`~openadapt_flow.backends.citrix_workspace.CitrixWorkspaceBackend` +through its real :class:`~openadapt_flow.backends.remote_display.WindowClient` +seam. Every ICA/HDX degradation in the Section 10 condition matrix is reproduced +here *deterministically and in-process* -- no Docker, no network, no Playwright, +no live protocol -- so the backend's fail-loud safety gates (frame-freshness +lease, focus/occlusion binding, DPI/scale consistency, input-trust, one-shot +actuation lease, readiness/identity re-resolution) can be qualified against the +whole matrix while CI stays green. + +HONEST LABEL (non-negotiable): this is a **DETERMINISTIC STAND-IN** -- a +synthetic fixture that *reproduces* ICA/HDX conditions. It is **NOT** real Citrix +ICA/HDX. It does not exercise HDX codecs, ICA compression, or the real +Workspace-client transport. No result here is real-protocol acceptance; the +real-ICA lane remains the customer-environment (Accuro) release gate documented +in ``benchmark/citrix_workspace/README.md``. + +The frame the ``SyntheticIcaWindowClient`` returns is a real PNG that the real +backend decodes, hashes, DPI-maps, occlusion-checks, and readiness/identity +probes; the synthetic app also *interprets delivered input* (roster select, note +focus, typing, Save) and commits the write to an out-of-band :class:`FaultDB` +oracle. Effect verification reads that oracle, never the on-screen banner. +""" + +from __future__ import annotations + +import hashlib +import io +from dataclasses import dataclass, field +from typing import Optional + +from PIL import Image, ImageDraw, ImageFilter, ImageOps + +from openadapt_flow.backends.remote_display import RemoteDisplayError, WindowInfo + +# -- fixed synthetic geometry (pixel space of the captured frame) -------------- +# Window bounds are screen POINTS; the captured frame is pixels. scale 2.0 (a +# common Citrix/Retina HiDPI factor) exercises the backend's pixel<->point map. +FRAME_PX: tuple[int, int] = (600, 400) +WINDOW_BOUNDS: tuple[float, float, float, float] = (40.0, 24.0, 300.0, 200.0) +SCALE: float = 2.0 +WINDOW_ID: int = 4101 +WINDOW_PID: int = 9200 + +# Target rungs (pixel coordinates on the captured frame). +ROSTER_ROW: tuple[int, int] = (150, 96) # "Ada Lovelace MRN A1001" +NOTE_FIELD: tuple[int, int] = (170, 300) # clinical-note entry box +SAVE_BUTTON: tuple[int, int] = (470, 300) # "Save Note" (irreversible write) + +EXPECTED_MRN: str = "A1001" +NOTE_VALUE: str = "followup in two weeks" + +# On-frame marker swatches (deterministic pixel probes stand in for OCR markers; +# the real backend accepts an injected pixel predicate exactly this way). +_READY_SWATCH = (8, 8, 40, 24) # green when in-app, red when locked +_APP_SWATCH = (48, 8, 80, 24) # application-identity swatch +_STATE_SWATCH = (88, 8, 120, 24) # workflow-state swatch +_CLOCK_REGION = (520, 4, 596, 26) # VOLATILE remote chrome (a clock) + +_READY_GREEN = (0, 170, 0) +_LOCK_RED = (185, 0, 0) +_APP_BLUE = (0, 70, 190) +_STATE_TEAL = (0, 150, 150) +_TOL = 42 # per-channel mean tolerance for probes + + +# -- out-of-band effect oracle ------------------------------------------------- +@dataclass +class FaultDB: + """Synthetic system-of-record. The app commits here on Save; the harness + reads it OUT OF BAND (never from the screen) to verify the real effect.""" + + records: list[tuple[str, str]] = field(default_factory=list) + rejected_writes: int = 0 + + def commit(self, mrn: str, note: str) -> None: + self.records.append((mrn, note)) + + def last(self) -> Optional[tuple[str, str]]: + return self.records[-1] if self.records else None + + def write_count(self) -> int: + return len(self.records) + + +def _mean(img: Image.Image, box: tuple[int, int, int, int]) -> tuple[float, ...]: + region = img.crop(box) + hist = region.resize((1, 1), Image.BILINEAR) + return hist.getpixel((0, 0)) + + +def _near(observed: tuple[float, ...], target: tuple[int, int, int]) -> bool: + return all(abs(float(o) - t) <= _TOL for o, t in zip(observed, target)) + + +def readiness_probe(png: bytes) -> bool: + """True iff the fresh frame is the in-app (unlocked, launched) surface.""" + with Image.open(io.BytesIO(png)) as im: + return _near(_mean(im.convert("RGB"), _READY_SWATCH), _READY_GREEN) + + +def application_marker_probe(png: bytes) -> bool: + with Image.open(io.BytesIO(png)) as im: + return _near(_mean(im.convert("RGB"), _APP_SWATCH), _APP_BLUE) + + +def workflow_state_marker_probe(png: bytes) -> bool: + with Image.open(io.BytesIO(png)) as im: + return _near(_mean(im.convert("RGB"), _STATE_SWATCH), _STATE_TEAL) + + +@dataclass +class AppState: + """Mutable synthetic-app + session state the renderer and client read.""" + + ready: bool = True # in-app (True) vs lock/login/spinner (False) + selected_mrn: Optional[str] = None + note_buffer: str = "" + note_focused: bool = False + committed: bool = False + overlay: bool = False # an unexpected dialog/overlay is painted + clock: int = 0 # volatile chrome value (only ticks on demand) + degrade: float = 0.0 # 0.0 none .. 1.0 severe codec/compression + theme_invert: bool = False + identity_broken: bool = False # application-identity marker unverifiable + write_rejected: bool = False # system of record rejects the Save (banner lies) + + +def render_frame(state: AppState) -> bytes: + """Render the synthetic MockMed frame deterministically to PNG bytes.""" + w, h = FRAME_PX + img = Image.new("RGB", (w, h), (245, 245, 248)) + d = ImageDraw.Draw(img) + + # top status bar + marker swatches + d.rectangle((0, 0, w, 30), fill=(225, 228, 235)) + d.rectangle(_READY_SWATCH, fill=_READY_GREEN if state.ready else _LOCK_RED) + d.rectangle( + _APP_SWATCH, + fill=_APP_BLUE + if (state.ready and not state.identity_broken) + else (120, 120, 120), + ) + d.rectangle( + _STATE_SWATCH, + fill=_STATE_TEAL if (state.ready and state.note_focused) else (120, 120, 120), + ) + # volatile clock (remote chrome) -- legitimately masked, never a target + d.rectangle(_CLOCK_REGION, fill=(235, 235, 235)) + d.text((524, 8), f"{state.clock:04d}", fill=(90, 90, 90)) + + if state.ready: + # roster row + row_fill = ( + (210, 232, 255) if state.selected_mrn == EXPECTED_MRN else (255, 255, 255) + ) + d.rectangle((40, 78, 560, 116), fill=row_fill, outline=(80, 80, 80)) + d.text((52, 90), f"Ada Lovelace MRN {EXPECTED_MRN}", fill=(0, 0, 0)) + # note field + d.rectangle((40, 280, 300, 320), fill=(255, 255, 255), outline=(80, 80, 80)) + d.text((48, 294), state.note_buffer[:34], fill=(0, 0, 0)) + # save button + d.rectangle((420, 280, 520, 320), fill=(60, 130, 60), outline=(20, 60, 20)) + d.text((432, 294), "Save Note", fill=(255, 255, 255)) + if state.committed: + # an optimistic on-screen banner -- deliberately NOT the oracle + d.text((330, 340), "Saved", fill=(0, 120, 0)) + else: + d.rectangle((150, 150, 450, 250), fill=(60, 60, 60)) + d.text((190, 195), "Session Locked", fill=(240, 240, 240)) + + if state.overlay: + d.rectangle((120, 120, 480, 260), fill=(250, 244, 190), outline=(120, 90, 0)) + d.text((150, 180), "Unexpected dialog", fill=(60, 40, 0)) + + if state.theme_invert: + img = ImageOps.invert(img) + + if state.degrade > 0.0: + # reproduce compression/codec artifacts: downscale round-trip + blur + + # JPEG. Severity scales the illegibility. This is a *synthetic* artifact + # generator, not a real HDX/Thinwire bitstream. + factor = max(0.06, 1.0 - state.degrade) + small = img.resize( + (max(1, int(w * factor)), max(1, int(h * factor))), Image.BILINEAR + ).resize((w, h), Image.BILINEAR) + if state.degrade >= 0.6: + small = small.filter(ImageFilter.GaussianBlur(2.0 * state.degrade)) + buf = io.BytesIO() + small.save(buf, format="JPEG", quality=max(4, int(90 * factor))) + img = Image.open(io.BytesIO(buf.getvalue())).convert("RGB") + + out = io.BytesIO() + img.save(out, format="PNG") + return out.getvalue() + + +def _session_digest(token: str) -> str: + return hashlib.sha256(("oa-ica-standin\x00" + token).encode()).hexdigest() + + +class SyntheticIcaWindowClient: + """A real :class:`WindowClient` backed by the deterministic synthetic frame. + + It is NOT a mock of the backend: the backend under test is unmodified. Only + this host-window seam is synthetic. It renders a real PNG the backend + decodes/hashes/DPI-maps, reports window geometry/focus/occlusion the backend + binds against, and -- crucially -- *interprets delivered OS input* the way + the remote app would (roster select, note focus, typing, Save->oracle + commit), so a delivered click has a real, independently observable effect. + """ + + def __init__(self, oracle: FaultDB, *, state: Optional[AppState] = None) -> None: + self.oracle = oracle + self.state = state or AppState() + self.trusted = True + self.frontmost = True + self.session_token = "sess-A" + self.window = WindowInfo( + window_id=WINDOW_ID, + owner="Citrix Viewer", + title="MockMed", + pid=WINDOW_PID, + bounds=WINDOW_BOUNDS, + on_screen=True, + ) + self.windows = [self.window] + self.key_window_override: Optional[int] = None + self.hit_window_override: Optional[int] = None + self.paste_blocked = False # Citrix clipboard restriction + self.unmapped_keys: set[str] = set() + self.aniso = False # uncalibrated anisotropic DPI + self.hover_unsettle_frames = 0 # delayed remote hover-paint frames + self._settle_countdown = 0 + self.calls: list[tuple] = [] + + # -- session identity (reconnect/roaming) -------------------------------- + def session_observer(self) -> Optional[str]: + return _session_digest(self.session_token) + + # -- WindowClient protocol ---------------------------------------------- + def input_trusted(self) -> bool: + return self.trusted + + def frontmost_pid(self) -> Optional[int]: + return self.window.pid if self.frontmost else 7 + + def find_windows(self, owner: str, title: Optional[str]) -> list[WindowInfo]: + return [ + w + for w in self.windows + if w.owner.casefold() == owner.casefold() + and (title is None or w.title.casefold() == title.casefold()) + ] + + def key_window_id(self, pid: int) -> Optional[int]: + if not self.frontmost: + return None + if self.key_window_override is not None: + return self.key_window_override + return self.window.window_id if pid == self.window.pid else None + + def window_at_point(self, x: float, y: float) -> Optional[int]: + if self.hit_window_override is not None: + return self.hit_window_override + return self.window.window_id + + def capture(self, window_id: int) -> tuple[bytes, int, int]: + # Delayed remote hover paint: while unsettled, advance the volatile + # chrome so consecutive frames differ (the backend's settle gate must + # wait for byte-stable frames before arming a lease). + if self._settle_countdown > 0: + self.state.clock += 1 + self._settle_countdown -= 1 + png = render_frame(self.state) + if self.aniso: + # Report captured pixels whose x/y scale disagrees with the window + # bounds -> the backend must refuse uncalibrated input. + with Image.open(io.BytesIO(png)) as im: + skewed = im.convert("RGB").resize((560, 400), Image.BILINEAR) + buf = io.BytesIO() + skewed.save(buf, format="PNG") + return buf.getvalue(), 560, 400 + return png, FRAME_PX[0], FRAME_PX[1] + + def activate(self, pid: int) -> None: + self.calls.append(("activate", pid)) + + def _pixel_from_screen(self, sx: float, sy: float) -> tuple[float, float]: + ox, oy = self.window.bounds[0], self.window.bounds[1] + return (sx - ox) * SCALE, (sy - oy) * SCALE + + def _in(self, px: float, py: float, box: tuple[int, int, int, int]) -> bool: + x0, y0, x1, y1 = box + return x0 <= px <= x1 and y0 <= py <= y1 + + def mouse( + self, x: float, y: float, *, button: str, down: bool, click_count: int = 1 + ) -> None: + self.calls.append(("mouse", button, down)) + if down or not self.state.ready: + return + px, py = self._pixel_from_screen(x, y) + if self._in(px, py, (40, 78, 560, 116)): + self.state.selected_mrn = EXPECTED_MRN + elif self._in(px, py, (40, 280, 300, 320)): + self.state.note_focused = True + elif self._in(px, py, (420, 280, 520, 320)): + # the irreversible write. The on-screen banner is optimistic; only a + # non-rejected write reaches the out-of-band oracle. + if self.state.selected_mrn is not None: + self.state.committed = True + if self.state.write_rejected: + self.oracle.rejected_writes += 1 + else: + self.oracle.commit(self.state.selected_mrn, self.state.note_buffer) + + def mouse_move(self, x: float, y: float) -> None: + self.calls.append(("move",)) + # Each new pointer positioning restarts the settle countdown, so the + # backend must re-observe stable frames after the move it just made. + self._settle_countdown = self.hover_unsettle_frames + + def type_chars(self, text: str) -> None: + self.calls.append(("type", text)) + if self.state.ready and self.state.note_focused: + self.state.note_buffer += text + + def key(self, keycode: int, *, down: bool, flags: list[str]) -> None: + self.calls.append(("key", keycode, down, tuple(flags))) + if self.paste_blocked and "control" in flags: + # Citrix policy disables the clipboard channel: fail LOUD rather + # than let a no-op paste look like a completed action. + raise RemoteDisplayError( + "clipboard redirection is disabled by session policy; paste " + "was not delivered (refusing to let a no-op look like success)" + ) + + def scroll(self, dx: int, dy: int) -> None: + self.calls.append(("scroll", dx, dy)) + + def resolve_key(self, token: str) -> Optional[tuple[int, bool]]: + if token in self.unmapped_keys: + return None + table = {"enter": (0x24, False), "tab": (0x30, False), "v": (0x09, False)} + return table.get(token.lower()) + + +# -- reviewable volatile-mask contract ---------------------------------------- +Rect = tuple[int, int, int, int] + + +@dataclass(frozen=True) +class VolatileMaskSpec: + """A reviewable declaration of which frame regions are volatile (may be + masked from continuity comparison) and which are PROTECTED and must never be + masked: target, actionability, identity, workflow-state, effect-relevant. + + The backend's default is deliberately conservative (full-frame decoded-RGB + continuity), so any real relaxation must be reviewed. This spec makes the + relaxation auditable and lets a check prove no protected region is covered. + """ + + volatile: dict[str, Rect] + protected: dict[str, Rect] + + +def default_mask_spec() -> VolatileMaskSpec: + return VolatileMaskSpec( + volatile={"remote_clock_chrome": _CLOCK_REGION}, + protected={ + "target_roster_row": (40, 78, 560, 116), + "target_note_field": (40, 280, 300, 320), + "target_save_button": (420, 280, 520, 320), + "actionability_save_label": (432, 294, 500, 306), + "identity_readiness": _READY_SWATCH, + "identity_application": _APP_SWATCH, + "workflow_state": _STATE_SWATCH, + "effect_saved_banner": (330, 336, 400, 352), + }, + ) + + +def _overlaps(a: Rect, b: Rect) -> bool: + ax0, ay0, ax1, ay1 = a + bx0, by0, bx1, by1 = b + return not (ax1 <= bx0 or bx1 <= ax0 or ay1 <= by0 or by1 <= ay0) + + +def check_masks_reviewable(spec: VolatileMaskSpec) -> list[str]: + """Return a list of violations: any volatile mask overlapping a protected + region. Empty list == the mask spec is reviewable and safe. Volatile masks + must NEVER cover a target, actionability, identity, workflow-state, or + effect-relevant region.""" + violations: list[str] = [] + for vname, vrect in spec.volatile.items(): + for pname, prect in spec.protected.items(): + if _overlaps(vrect, prect): + violations.append(f"volatile mask {vname!r} covers protected {pname!r}") + return violations diff --git a/benchmark/citrix_ica_hdx/results.json b/benchmark/citrix_ica_hdx/results.json new file mode 100644 index 00000000..252cc5e6 --- /dev/null +++ b/benchmark/citrix_ica_hdx/results.json @@ -0,0 +1,760 @@ +{ + "backend_under_test": "CitrixWorkspaceBackend (RemoteDisplayBackend preset)", + "condition_count": 16, + "deterministic_standin_accepted": true, + "enforcement_verified": [ + "every actuation uses a fresh frame (frame-freshness lease)", + "actuation stays bound to the authorized window and session", + "target re-resolved immediately before acting", + "one-shot actuation lease consumed exactly once (no double-fire)", + "DPI/scale calibration refused when anisotropic/uncalibrated", + "focus/occlusion binding enforced before every input edge", + "readiness + identity gating on the fresh actuation frame", + "out-of-band effect verification (never the on-screen banner)", + "zero model calls on every path (healthy and refusal)" + ], + "evidence_scope": "deterministic_synthetic_ica_hdx_standin", + "failure_taxonomy": [ + "healthy_over_halt", + "silent_incorrect_success", + "silent_write", + "unsafe_completion", + "volatile_mask_covers_protected_region" + ], + "generated_at": "2026-07-26T18:13:53.666190+00:00", + "halt_expectations": 20, + "healthy_over_halts": 0, + "ica_hdx_accepted": false, + "ica_hdx_status": "pending_real_environment_customer_lane", + "is_real_ica_hdx": false, + "label": "DETERMINISTIC STAND-IN reproducing ICA/HDX conditions -- NOT real Citrix ICA/HDX. No HDX codecs, no ICA compression, no real Workspace-client transport. Real-protocol evidence pending the customer-environment (Accuro) lane.", + "model_calls": 0, + "oracle": "in-process FaultDB read out of band (never the screen banner)", + "pass_expectations": 9, + "scenarios_passed": 29, + "schema_version": "openadapt.citrix-ica-hdx-standin-qualification.v1", + "section10_conditions_covered": [ + "DPI + scaling changes", + "ambiguous visual identity", + "clipboard restrictions", + "display compression / codec artifacts", + "duplicate prevention", + "focus theft", + "keyboard-layout / IME", + "latency / jitter / packet-loss / delayed-frames", + "persisted-state readback (out-of-band effect verification)", + "reconnect / roaming", + "session launch + application readiness", + "session lock / unlock", + "single and multimonitor geometry", + "uncertain submission (no blind retry)", + "unexpected dialogs / overlays", + "window minimize / occlusion / move / resize" + ], + "silent_incorrect_successes": 0, + "silent_writes": 0, + "status_dimensions": { + "backend_shipped": { + "detail": "CitrixWorkspaceBackend (RemoteDisplayBackend preset) is in the package and qualified against the deterministic stand-in.", + "status": "shipped" + }, + "customer_controlled_execution_available": { + "detail": "Customer-controlled on-prem execution is the target of the real-ICA release gate; not asserted by this stand-in.", + "status": "pending" + }, + "deterministic_standin_qualification": { + "detail": "The Section 10 ICA/HDX condition matrix is qualified against a DETERMINISTIC SYNTHETIC STAND-IN (this campaign) -- NOT real ICA/HDX.", + "status": "qualified" + }, + "exact_application_qualification_available": { + "detail": "Exact published-application qualification (e.g. Accuro) is performed in the customer's exact environment; not asserted here.", + "status": "pending" + }, + "installed_driver_available": { + "detail": "MacWindowClient (Quartz) and Win32WindowClient host drivers ship; live capture/input requires per-host Screen-Recording / Accessibility / integrity trust granted at deployment.", + "status": "shipped_host_clients" + }, + "managed_execution_available": { + "detail": "Hosted/managed execution of this workflow is not asserted by this stand-in.", + "status": "pending" + }, + "real_protocol_environment_evidence": { + "detail": "No real Citrix ICA/HDX acceptance. Pending the customer-environment (Accuro) lane: HDX codecs, ICA compression, and the real Workspace-client transport are NOT exercised here.", + "status": "pending" + } + }, + "substrate": "citrix-workspace-backend-over-deterministic-ica-hdx-standin", + "trial_count": 29, + "trials": [ + { + "effect_expected": [ + "A1001", + "followup in two weeks" + ], + "effect_observed": [ + "A1001", + "followup in two weeks" + ], + "effect_verified": true, + "expectation": "pass", + "halt_reason": null, + "healthy_over_halt": false, + "id": "session_launch_ready", + "kind": "write", + "model_calls": 0, + "oracle_write_count": 1, + "outcome": "completed", + "passed": true, + "section10_condition": "session launch + application readiness", + "silent_incorrect_success": false, + "silent_write": false + }, + { + "effect_expected": [ + "A1001", + "followup in two weeks" + ], + "effect_observed": null, + "effect_verified": false, + "expectation": "halt", + "halt_reason": "remote-display readiness probe rejected the current frame (locked, disconnected, or unexpected session); refusing input", + "healthy_over_halt": false, + "id": "application_not_ready", + "kind": "write", + "model_calls": 0, + "oracle_write_count": 0, + "outcome": "halted", + "passed": true, + "section10_condition": "session launch + application readiness", + "silent_incorrect_success": false, + "silent_write": false + }, + { + "effect_expected": [ + "A1001", + "followup in two weeks" + ], + "effect_observed": null, + "effect_verified": false, + "expectation": "halt", + "halt_reason": "remote session identity changed or became unverifiable after capture; refusing input", + "healthy_over_halt": false, + "id": "reconnect_roaming_identity_change", + "kind": "write", + "model_calls": 0, + "oracle_write_count": 0, + "outcome": "halted", + "passed": true, + "section10_condition": "reconnect / roaming", + "silent_incorrect_success": false, + "silent_write": false + }, + { + "effect_expected": [ + "A1001", + "followup in two weeks" + ], + "effect_observed": null, + "effect_verified": false, + "expectation": "halt", + "halt_reason": "remote-display readiness probe rejected the current frame (locked, disconnected, or unexpected session); refusing input", + "healthy_over_halt": false, + "id": "session_lock", + "kind": "write", + "model_calls": 0, + "oracle_write_count": 0, + "outcome": "halted", + "passed": true, + "section10_condition": "session lock / unlock", + "silent_incorrect_success": false, + "silent_write": false + }, + { + "effect_expected": [ + "A1001", + "followup in two weeks" + ], + "effect_observed": [ + "A1001", + "followup in two weeks" + ], + "effect_verified": true, + "expectation": "pass", + "halt_reason": null, + "healthy_over_halt": false, + "id": "session_unlock_recovery", + "kind": "write", + "locked_attempt_refused": true, + "model_calls": 0, + "oracle_write_count": 1, + "outcome": "completed", + "passed": true, + "section10_condition": "session lock / unlock", + "silent_incorrect_success": false, + "silent_write": false + }, + { + "effect_expected": [ + "A1001", + "followup in two weeks" + ], + "effect_observed": null, + "effect_verified": false, + "expectation": "halt", + "halt_reason": "the exact remote-display window is not visible, app-frontmost, and keyboard-frontmost after activation; refusing input", + "healthy_over_halt": false, + "id": "window_minimize", + "kind": "write", + "model_calls": 0, + "oracle_write_count": 0, + "outcome": "halted", + "passed": true, + "section10_condition": "window minimize / occlusion / move / resize", + "silent_incorrect_success": false, + "silent_write": false + }, + { + "effect_expected": [ + "A1001", + "followup in two weeks" + ], + "effect_observed": null, + "effect_verified": false, + "expectation": "halt", + "halt_reason": "screen point (115.0, 72.0) is covered by window 9999, not the leased remote-display window 4101; refusing a click that could hit an occluder", + "healthy_over_halt": false, + "id": "window_occlusion", + "kind": "write", + "model_calls": 0, + "oracle_write_count": 0, + "outcome": "halted", + "passed": true, + "section10_condition": "window minimize / occlusion / move / resize", + "silent_incorrect_success": false, + "silent_write": false + }, + { + "effect_expected": [ + "A1001", + "followup in two weeks" + ], + "effect_observed": null, + "effect_verified": false, + "expectation": "halt", + "halt_reason": "remote-display window identity or geometry changed since capture; capture and re-resolve before input", + "healthy_over_halt": false, + "id": "window_move_after_acquire", + "kind": "write", + "model_calls": 0, + "oracle_write_count": 0, + "outcome": "halted", + "passed": true, + "section10_condition": "window minimize / occlusion / move / resize", + "silent_incorrect_success": false, + "silent_write": false + }, + { + "effect_expected": [ + "A1001", + "followup in two weeks" + ], + "effect_observed": null, + "effect_verified": false, + "expectation": "halt", + "halt_reason": "remote-display window identity or geometry changed since capture; capture and re-resolve before input", + "healthy_over_halt": false, + "id": "window_resize_after_acquire", + "kind": "write", + "model_calls": 0, + "oracle_write_count": 0, + "outcome": "halted", + "passed": true, + "section10_condition": "window minimize / occlusion / move / resize", + "silent_incorrect_success": false, + "silent_write": false + }, + { + "effect_expected": [ + "A1001", + "followup in two weeks" + ], + "effect_observed": [ + "A1001", + "followup in two weeks" + ], + "effect_verified": true, + "expectation": "pass", + "halt_reason": null, + "healthy_over_halt": false, + "id": "dpi_scale_consistent", + "kind": "write", + "model_calls": 0, + "oracle_write_count": 1, + "outcome": "completed", + "passed": true, + "scale": 2.0, + "section10_condition": "DPI + scaling changes", + "silent_incorrect_success": false, + "silent_write": false + }, + { + "effect_expected": [ + "A1001", + "followup in two weeks" + ], + "effect_observed": null, + "effect_verified": false, + "expectation": "halt", + "halt_reason": "captured frame and client bounds have inconsistent DPI scale (1.8667x vs 2.0000y); refusing uncalibrated input", + "healthy_over_halt": false, + "id": "dpi_anisotropic_uncalibrated", + "kind": "write", + "model_calls": 0, + "oracle_write_count": 0, + "outcome": "halted", + "passed": true, + "section10_condition": "DPI + scaling changes", + "silent_incorrect_success": false, + "silent_write": false + }, + { + "effect_expected": [ + "A1001", + "followup in two weeks" + ], + "effect_observed": [ + "A1001", + "followup in two weeks" + ], + "effect_verified": true, + "expectation": "pass", + "halt_reason": null, + "healthy_over_halt": false, + "id": "single_monitor_geometry", + "kind": "write", + "model_calls": 0, + "oracle_write_count": 1, + "outcome": "completed", + "passed": true, + "section10_condition": "single and multimonitor geometry", + "silent_incorrect_success": false, + "silent_write": false + }, + { + "effect_expected": [ + "A1001", + "followup in two weeks" + ], + "effect_observed": [ + "A1001", + "followup in two weeks" + ], + "effect_verified": true, + "expectation": "pass", + "halt_reason": null, + "healthy_over_halt": false, + "id": "multimonitor_secondary_offset", + "kind": "write", + "model_calls": 0, + "oracle_write_count": 1, + "outcome": "completed", + "passed": true, + "section10_condition": "single and multimonitor geometry", + "silent_incorrect_success": false, + "silent_write": false + }, + { + "effect_expected": [ + "A1001", + "followup in two weeks" + ], + "effect_observed": null, + "effect_verified": false, + "expectation": "halt", + "halt_reason": "ambiguous remote-display target: expected one exact owner/title match, found 2; target identities are withheld because window titles can contain sensitive record data", + "healthy_over_halt": false, + "id": "multimonitor_ambiguous_window", + "kind": "write", + "model_calls": 0, + "oracle_write_count": 0, + "outcome": "halted", + "passed": true, + "section10_condition": "single and multimonitor geometry", + "silent_incorrect_success": false, + "silent_write": false + }, + { + "effect_expected": [ + "A1001", + "followup in two weeks" + ], + "effect_observed": [ + "A1001", + "followup in two weeks" + ], + "effect_verified": true, + "expectation": "pass", + "halt_reason": null, + "healthy_over_halt": false, + "id": "codec_artifacts_mild_legible", + "kind": "write", + "model_calls": 0, + "oracle_write_count": 1, + "outcome": "completed", + "passed": true, + "section10_condition": "display compression / codec artifacts", + "silent_incorrect_success": false, + "silent_write": false + }, + { + "effect_expected": [ + "A1001", + "followup in two weeks" + ], + "effect_observed": null, + "effect_verified": false, + "expectation": "halt", + "halt_reason": "remote-display readiness probe rejected the current frame (locked, disconnected, or unexpected session); refusing input", + "healthy_over_halt": false, + "id": "codec_artifacts_severe_illegible", + "kind": "write", + "model_calls": 0, + "oracle_write_count": 0, + "outcome": "halted", + "passed": true, + "section10_condition": "display compression / codec artifacts", + "silent_incorrect_success": false, + "silent_write": false + }, + { + "effect_expected": [ + "A1001", + "followup in two weeks" + ], + "effect_observed": null, + "effect_verified": false, + "expectation": "halt", + "halt_reason": "remote-display frame is stale (0.001s > 0.000s); halting intentionally so the runtime can capture, re-resolve, and re-check identity", + "healthy_over_halt": false, + "id": "stale_frame_latency", + "kind": "write", + "model_calls": 0, + "oracle_write_count": 0, + "outcome": "halted", + "passed": true, + "section10_condition": "latency / jitter / packet-loss / delayed-frames", + "silent_incorrect_success": false, + "silent_write": false + }, + { + "effect_expected": [ + "A1001", + "followup in two weeks" + ], + "effect_observed": [ + "A1001", + "followup in two weeks" + ], + "effect_verified": true, + "expectation": "pass", + "halt_reason": null, + "healthy_over_halt": false, + "id": "delayed_frame_settles", + "kind": "write", + "model_calls": 0, + "oracle_write_count": 1, + "outcome": "completed", + "passed": true, + "section10_condition": "latency / jitter / packet-loss / delayed-frames", + "silent_incorrect_success": false, + "silent_write": false + }, + { + "effect_expected": [ + "A1001", + "followup in two weeks" + ], + "effect_observed": null, + "effect_verified": false, + "expectation": "halt", + "halt_reason": "remote-display pixels did not settle after pointer positioning; refusing to acquire a consequential actuation frame", + "healthy_over_halt": false, + "id": "frame_never_settles", + "kind": "write", + "model_calls": 0, + "oracle_write_count": 0, + "outcome": "halted", + "passed": true, + "section10_condition": "latency / jitter / packet-loss / delayed-frames", + "silent_incorrect_success": false, + "silent_write": false + }, + { + "effect_expected": null, + "effect_observed": null, + "effect_verified": null, + "expectation": "pass", + "halt_reason": null, + "healthy_over_halt": false, + "id": "keyboard_named_key_ok", + "kind": "input", + "model_calls": 0, + "oracle_write_count": 0, + "outcome": "completed", + "passed": true, + "section10_condition": "keyboard-layout / IME", + "silent_incorrect_success": false, + "silent_write": false + }, + { + "effect_expected": null, + "effect_observed": null, + "effect_verified": null, + "expectation": "halt", + "halt_reason": "no key mapping for 'compose' in 'compose'", + "healthy_over_halt": false, + "id": "ime_unmapped_key", + "kind": "input", + "model_calls": 0, + "oracle_write_count": 0, + "outcome": "halted", + "passed": true, + "section10_condition": "keyboard-layout / IME", + "silent_incorrect_success": false, + "silent_write": false + }, + { + "effect_expected": null, + "effect_observed": null, + "effect_verified": null, + "expectation": "halt", + "halt_reason": "clipboard redirection is disabled by session policy; paste was not delivered (refusing to let a no-op look like success)", + "healthy_over_halt": false, + "id": "clipboard_restricted_paste", + "kind": "input", + "model_calls": 0, + "oracle_write_count": 0, + "outcome": "halted", + "passed": true, + "section10_condition": "clipboard restrictions", + "silent_incorrect_success": false, + "silent_write": false + }, + { + "effect_expected": [ + "A1001", + "followup in two weeks" + ], + "effect_observed": null, + "effect_verified": false, + "expectation": "halt", + "halt_reason": "the exact remote-display window is not visible, app-frontmost, and keyboard-frontmost after activation; refusing input", + "healthy_over_halt": false, + "id": "focus_theft_after_acquire", + "kind": "write", + "model_calls": 0, + "oracle_write_count": 0, + "outcome": "halted", + "passed": true, + "section10_condition": "focus theft", + "silent_incorrect_success": false, + "silent_write": false + }, + { + "effect_expected": [ + "A1001", + "followup in two weeks" + ], + "effect_observed": null, + "effect_verified": false, + "expectation": "halt", + "halt_reason": "remote-display frame content changed after target and identity resolution; refusing input and requiring a fresh actuation lease", + "healthy_over_halt": false, + "id": "unexpected_dialog_overlay", + "kind": "write", + "model_calls": 0, + "oracle_write_count": 0, + "outcome": "halted", + "passed": true, + "section10_condition": "unexpected dialogs / overlays", + "silent_incorrect_success": false, + "silent_write": false + }, + { + "effect_expected": [ + "A1001", + "followup in two weeks" + ], + "effect_observed": null, + "effect_verified": false, + "expectation": "halt", + "halt_reason": "remote-display actuation lease was invalidated by another observation; refusing input and requiring a fresh lease", + "healthy_over_halt": false, + "id": "unverifiable_application_identity", + "kind": "write", + "model_calls": 0, + "oracle_write_count": 0, + "outcome": "halted", + "passed": true, + "section10_condition": "ambiguous visual identity", + "silent_incorrect_success": false, + "silent_write": false + }, + { + "effect_expected": [ + "A1001", + "followup in two weeks" + ], + "effect_observed": null, + "effect_verified": false, + "expectation": "halt", + "halt_reason": "remote-display actuation lease was invalidated by another observation; refusing input and requiring a fresh lease", + "healthy_over_halt": false, + "id": "uncertain_submission_no_blind_retry", + "kind": "write", + "model_calls": 0, + "oracle_write_count": 0, + "outcome": "halted", + "passed": true, + "section10_condition": "uncertain submission (no blind retry)", + "silent_incorrect_success": false, + "silent_write": false + }, + { + "effect_expected": [ + "A1001", + "followup in two weeks" + ], + "effect_observed": [ + "A1001", + "followup in two weeks" + ], + "effect_verified": true, + "expectation": "halt", + "halt_reason": "consequential remote click was not pre-positioned at the freshly resolved target; refusing pointer delivery", + "healthy_over_halt": false, + "id": "duplicate_write_prevention_one_shot", + "kind": "write", + "model_calls": 0, + "oracle_write_count": 1, + "outcome": "halted", + "passed": true, + "section10_condition": "duplicate prevention", + "silent_incorrect_success": false, + "silent_write": false + }, + { + "banner_shown": true, + "effect_expected": [ + "A1001", + "followup in two weeks" + ], + "effect_observed": [ + "A1001", + "followup in two weeks" + ], + "effect_verified": true, + "expectation": "pass", + "halt_reason": null, + "healthy_over_halt": false, + "id": "persisted_state_readback", + "kind": "write", + "model_calls": 0, + "oracle_write_count": 1, + "outcome": "completed", + "passed": true, + "section10_condition": "persisted-state readback (out-of-band effect verification)", + "silent_incorrect_success": false, + "silent_write": false, + "verified_via": "out_of_band_oracle" + }, + { + "banner_shown": true, + "effect_expected": [ + "A1001", + "followup in two weeks" + ], + "effect_observed": null, + "effect_verified": false, + "expectation": "effect_refused", + "halt_reason": null, + "healthy_over_halt": false, + "id": "optimistic_banner_effect_refused", + "independent_record_present": false, + "kind": "write", + "model_calls": 0, + "oracle_write_count": 0, + "outcome": "completed", + "passed": true, + "section10_condition": "persisted-state readback (out-of-band effect verification)", + "silent_incorrect_success": false, + "silent_write": false + } + ], + "volatile_mask_review": { + "note": "Volatile-region masks must never cover target, actionability, identity, workflow-state, or effect-relevant regions.", + "reviewable_and_safe": true, + "spec": { + "protected": { + "actionability_save_label": [ + 432, + 294, + 500, + 306 + ], + "effect_saved_banner": [ + 330, + 336, + 400, + 352 + ], + "identity_application": [ + 48, + 8, + 80, + 24 + ], + "identity_readiness": [ + 8, + 8, + 40, + 24 + ], + "target_note_field": [ + 40, + 280, + 300, + 320 + ], + "target_roster_row": [ + 40, + 78, + 560, + 116 + ], + "target_save_button": [ + 420, + 280, + 520, + 320 + ], + "workflow_state": [ + 88, + 8, + 120, + 24 + ] + }, + "volatile": { + "remote_clock_chrome": [ + 520, + 4, + 596, + 26 + ] + } + }, + "violations": [] + }, + "window_client": "SyntheticIcaWindowClient (in-process deterministic frames)" +} diff --git a/benchmark/citrix_ica_hdx/run_ica_hdx_qualification.py b/benchmark/citrix_ica_hdx/run_ica_hdx_qualification.py new file mode 100644 index 00000000..bc07c52f --- /dev/null +++ b/benchmark/citrix_ica_hdx/run_ica_hdx_qualification.py @@ -0,0 +1,961 @@ +#!/usr/bin/env python3 +"""Comprehensive DETERMINISTIC ICA/HDX stand-in qualification (roadmap Section 10). + +This campaign drives the UNMODIFIED +:class:`~openadapt_flow.backends.citrix_workspace.CitrixWorkspaceBackend` through +the full Section 10 ICA/HDX condition matrix, each reproduced as a reproducible, +in-process synthetic fixture scenario (see ``fixture.py``) with an explicit +pass/halt expectation. It verifies that on every condition the backend either +delivers a correct, out-of-band-verified actuation OR safely halts -- and NEVER +produces a silent incorrect success. + +HONEST LABEL (non-negotiable, brand moat): this is a **DETERMINISTIC STAND-IN** +that reproduces ICA/HDX conditions synthetically. It is **NOT real Citrix +ICA/HDX**. It does not exercise HDX codecs, ICA compression, or the real +Workspace-client transport. Real-protocol acceptance remains pending the +customer-environment (Accuro) lane; see ``benchmark/citrix_workspace/README.md`` +and ``docs/desktop/CITRIX_PIXEL.md``. + +What it qualifies: the pixel/no-DOM actuation contract of the Citrix backend -- +fresh-frame lease, authorized-window/session binding, immediate re-resolution +before acting, one-shot actuation lease, DPI/scale calibration, focus/occlusion +binding, readiness/identity gating, and out-of-band effect verification -- +across every Section 10 condition. Zero model calls on every path. + +Run:: + + python3 benchmark/citrix_ica_hdx/run_ica_hdx_qualification.py \ + --output benchmark/citrix_ica_hdx/results.json + +No Docker, no network, no Playwright: fully deterministic and CI-green. +""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib.util +import json +import sys +from datetime import datetime, timezone +from pathlib import Path +from typing import Callable, Optional + +from openadapt_flow.backends.citrix_workspace import CitrixWorkspaceBackend +from openadapt_flow.backends.remote_display import RemoteDisplayError, WindowInfo + +_FIXTURE_PATH = Path(__file__).resolve().parent / "fixture.py" + + +def _load_fixture(): + spec = importlib.util.spec_from_file_location("ica_hdx_fixture", _FIXTURE_PATH) + assert spec is not None and spec.loader is not None + mod = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = mod + spec.loader.exec_module(mod) + return mod + + +fx = _load_fixture() + +EXPECTED = (fx.EXPECTED_MRN, fx.NOTE_VALUE) + + +def _backend(client, **overrides) -> CitrixWorkspaceBackend: + kwargs = dict( + window_title="MockMed", + require_input_trust=True, + activate_before_input=True, + settle_s=0.0, + pointer_settle_stable_frames=2, + pointer_settle_timeout_s=0.25, + max_frame_age_s=30.0, + readiness_probe=fx.readiness_probe, + application_marker="MockMed", + application_marker_probe=fx.application_marker_probe, + workflow_state_marker="NoteEntry", + workflow_state_marker_probe=fx.workflow_state_marker_probe, + session_identity_observer=client.session_observer, + ) + kwargs.update(overrides) + return CitrixWorkspaceBackend(client, **kwargs) + + +def _new_env(**state_kwargs): + oracle = fx.FaultDB() + state = fx.AppState(**state_kwargs) + client = fx.SyntheticIcaWindowClient(oracle, state=state) + return oracle, state, client + + +def _drive_workflow( + client, + backend, + *, + after_acquire: Optional[Callable] = None, + invalidate_before_click: bool = False, +) -> dict: + """Record-shaped workflow: reversible roster/note/type steps, then the ONE + irreversible Save through the two-phase one-shot actuation lease (mirroring + the governed runtime's consequential-actuation handshake).""" + backend.screenshot() + backend.click(*fx.ROSTER_ROW) # select patient (reversible) + backend.click(*fx.NOTE_FIELD) # focus note (reversible) + backend.type_text(fx.NOTE_VALUE) # type note + backend.screenshot() + backend.prepare_pointer_actuation(*fx.SAVE_BUTTON) + backend.acquire_actuation_frame() # arm one-shot lease + identity = { + "application": backend.application_identity(), + "workflow_state": backend.workflow_state_identity(), + "session": backend.session_identity(), + } + if after_acquire is not None: + after_acquire(client, backend) + if invalidate_before_click: + backend.screenshot() # diagnostic observation invalidates lease + backend.click(*fx.SAVE_BUTTON) # consequential write; lease consumed + return identity + + +def _guard(fn) -> tuple[bool, Optional[str]]: + try: + fn() + return False, None + except RemoteDisplayError as exc: + return True, str(exc)[:180] + + +def _record( + scenario_id: str, + condition: str, + expectation: str, + *, + halted: bool, + reason: Optional[str], + oracle, + kind: str = "write", + effect_verified: Optional[bool] = None, +) -> dict: + """Classify one scenario into the honest metrics the campaign publishes.""" + observed = oracle.last() if oracle is not None else None + writes = oracle.write_count() if oracle is not None else 0 + verified = (observed == EXPECTED) if effect_verified is None else effect_verified + completed = not halted + + silent_incorrect_success = False + silent_write = False + healthy_over_halt = False + + if expectation == "pass": + passed = completed and (verified if kind == "write" else True) + healthy_over_halt = halted + silent_incorrect_success = completed and kind == "write" and not verified + elif expectation == "halt": + passed = halted and writes == 0 + silent_write = writes > 0 + silent_incorrect_success = completed and ( + writes > 0 if kind == "write" else True + ) + elif expectation == "effect_refused": + # Backend delivered input, but the independent oracle shows no matching + # persisted record -> out-of-band verification correctly withholds + # success (an optimistic on-screen "Saved" banner must never confirm). + passed = completed and observed != EXPECTED + silent_incorrect_success = completed and observed == EXPECTED and not passed + else: + raise ValueError(f"unknown expectation {expectation!r}") + + return { + "id": scenario_id, + "section10_condition": condition, + "kind": kind, + "expectation": expectation, + "outcome": "halted" if halted else "completed", + "halt_reason": reason, + "effect_expected": list(EXPECTED) if kind == "write" else None, + "effect_observed": list(observed) if observed is not None else None, + "effect_verified": bool(verified) if kind == "write" else None, + "oracle_write_count": writes, + "model_calls": 0, + "silent_incorrect_success": bool(silent_incorrect_success), + "silent_write": bool(silent_write), + "healthy_over_halt": bool(healthy_over_halt), + "passed": bool(passed), + } + + +# ============================================================================= +# Section 10 condition-matrix scenarios +# ============================================================================= + + +def s_session_launch_ready() -> dict: + oracle, state, client = _new_env() + backend = _backend(client) + halted, reason = _guard(lambda: _drive_workflow(client, backend)) + return _record( + "session_launch_ready", + "session launch + application readiness", + "pass", + halted=halted, + reason=reason, + oracle=oracle, + ) + + +def s_application_not_ready() -> dict: + oracle, state, client = _new_env(ready=False) + backend = _backend(client) + halted, reason = _guard(lambda: _drive_workflow(client, backend)) + return _record( + "application_not_ready", + "session launch + application readiness", + "halt", + halted=halted, + reason=reason, + oracle=oracle, + ) + + +def s_reconnect_roaming() -> dict: + oracle, state, client = _new_env() + backend = _backend(client) + + def roam(cl, be): + cl.session_token = "sess-B" # reconnect/roam to a different session + + halted, reason = _guard( + lambda: _drive_workflow(client, backend, after_acquire=roam) + ) + return _record( + "reconnect_roaming_identity_change", + "reconnect / roaming", + "halt", + halted=halted, + reason=reason, + oracle=oracle, + ) + + +def s_session_lock() -> dict: + oracle, state, client = _new_env() + backend = _backend(client) + + def lock(cl, be): + cl.state.ready = False # session locked mid-flow + + halted, reason = _guard( + lambda: _drive_workflow(client, backend, after_acquire=lock) + ) + return _record( + "session_lock", + "session lock / unlock", + "halt", + halted=halted, + reason=reason, + oracle=oracle, + ) + + +def s_session_unlock_recovery() -> dict: + # Start locked; prove the first attempt refuses, then unlock and succeed. + oracle, state, client = _new_env(ready=False) + backend = _backend(client) + locked_halted, _ = _guard(lambda: _drive_workflow(client, backend)) + client.state.ready = True # operator unlocks the session + backend2 = _backend(client) + halted, reason = _guard(lambda: _drive_workflow(client, backend2)) + rec = _record( + "session_unlock_recovery", + "session lock / unlock", + "pass", + halted=halted, + reason=reason, + oracle=oracle, + ) + rec["locked_attempt_refused"] = bool(locked_halted) + return rec + + +def s_window_minimize() -> dict: + oracle, state, client = _new_env() + client.window = WindowInfo( + window_id=fx.WINDOW_ID, + owner="Citrix Viewer", + title="MockMed", + pid=fx.WINDOW_PID, + bounds=fx.WINDOW_BOUNDS, + on_screen=False, # minimized / hidden + ) + client.windows = [client.window] + backend = _backend(client) + halted, reason = _guard(lambda: _drive_workflow(client, backend)) + return _record( + "window_minimize", + "window minimize / occlusion / move / resize", + "halt", + halted=halted, + reason=reason, + oracle=oracle, + ) + + +def s_window_occlusion() -> dict: + oracle, state, client = _new_env() + client.hit_window_override = 9999 # another window covers the target point + backend = _backend(client) + halted, reason = _guard(lambda: _drive_workflow(client, backend)) + return _record( + "window_occlusion", + "window minimize / occlusion / move / resize", + "halt", + halted=halted, + reason=reason, + oracle=oracle, + ) + + +def s_window_move() -> dict: + oracle, state, client = _new_env() + backend = _backend(client) + + def move(cl, be): + old = cl.window + cl.window = WindowInfo( + window_id=old.window_id, + owner=old.owner, + title=old.title, + pid=old.pid, + bounds=(old.bounds[0] + 120.0, old.bounds[1], old.bounds[2], old.bounds[3]), + on_screen=True, + ) + cl.windows = [cl.window] + + halted, reason = _guard( + lambda: _drive_workflow(client, backend, after_acquire=move) + ) + return _record( + "window_move_after_acquire", + "window minimize / occlusion / move / resize", + "halt", + halted=halted, + reason=reason, + oracle=oracle, + ) + + +def s_window_resize() -> dict: + oracle, state, client = _new_env() + backend = _backend(client) + + def resize(cl, be): + old = cl.window + cl.window = WindowInfo( + window_id=old.window_id, + owner=old.owner, + title=old.title, + pid=old.pid, + bounds=(old.bounds[0], old.bounds[1], old.bounds[2] - 40.0, old.bounds[3]), + on_screen=True, + ) + cl.windows = [cl.window] + + halted, reason = _guard( + lambda: _drive_workflow(client, backend, after_acquire=resize) + ) + return _record( + "window_resize_after_acquire", + "window minimize / occlusion / move / resize", + "halt", + halted=halted, + reason=reason, + oracle=oracle, + ) + + +def s_dpi_scale_consistent() -> dict: + oracle, state, client = _new_env() + backend = _backend(client) + halted, reason = _guard(lambda: _drive_workflow(client, backend)) + rec = _record( + "dpi_scale_consistent", + "DPI + scaling changes", + "pass", + halted=halted, + reason=reason, + oracle=oracle, + ) + rec["scale"] = fx.SCALE + return rec + + +def s_dpi_anisotropic() -> dict: + oracle, state, client = _new_env() + client.aniso = True # captured pixels imply x/y scale mismatch + backend = _backend(client) + halted, reason = _guard(lambda: _drive_workflow(client, backend)) + return _record( + "dpi_anisotropic_uncalibrated", + "DPI + scaling changes", + "halt", + halted=halted, + reason=reason, + oracle=oracle, + ) + + +def s_single_monitor_geometry() -> dict: + oracle, state, client = _new_env() + backend = _backend(client) + halted, reason = _guard(lambda: _drive_workflow(client, backend)) + return _record( + "single_monitor_geometry", + "single and multimonitor geometry", + "pass", + halted=halted, + reason=reason, + oracle=oracle, + ) + + +def s_multimonitor_secondary_offset() -> dict: + oracle, state, client = _new_env() + client.window = WindowInfo( + window_id=fx.WINDOW_ID, + owner="Citrix Viewer", + title="MockMed", + pid=fx.WINDOW_PID, + bounds=(1920.0, 0.0, fx.WINDOW_BOUNDS[2], fx.WINDOW_BOUNDS[3]), # 2nd monitor + on_screen=True, + ) + client.windows = [client.window] + backend = _backend(client) + halted, reason = _guard(lambda: _drive_workflow(client, backend)) + return _record( + "multimonitor_secondary_offset", + "single and multimonitor geometry", + "pass", + halted=halted, + reason=reason, + oracle=oracle, + ) + + +def s_multimonitor_ambiguous() -> dict: + oracle, state, client = _new_env() + twin = WindowInfo( + window_id=fx.WINDOW_ID + 1, + owner="Citrix Viewer", + title="MockMed", + pid=fx.WINDOW_PID, + bounds=(1920.0, 0.0, fx.WINDOW_BOUNDS[2], fx.WINDOW_BOUNDS[3]), + on_screen=True, + ) + client.windows = [client.window, twin] # duplicate session windows + backend = _backend(client) + halted, reason = _guard(lambda: _drive_workflow(client, backend)) + return _record( + "multimonitor_ambiguous_window", + "single and multimonitor geometry", + "halt", + halted=halted, + reason=reason, + oracle=oracle, + ) + + +def s_codec_mild() -> dict: + oracle, state, client = _new_env(degrade=0.12) + backend = _backend(client) + halted, reason = _guard(lambda: _drive_workflow(client, backend)) + return _record( + "codec_artifacts_mild_legible", + "display compression / codec artifacts", + "pass", + halted=halted, + reason=reason, + oracle=oracle, + ) + + +def s_codec_severe() -> dict: + oracle, state, client = _new_env(degrade=0.9) + backend = _backend(client) + halted, reason = _guard(lambda: _drive_workflow(client, backend)) + return _record( + "codec_artifacts_severe_illegible", + "display compression / codec artifacts", + "halt", + halted=halted, + reason=reason, + oracle=oracle, + ) + + +def s_stale_frame() -> dict: + oracle, state, client = _new_env() + backend = _backend(client, max_frame_age_s=1e-9) # any elapsed time is stale + halted, reason = _guard(lambda: _drive_workflow(client, backend)) + return _record( + "stale_frame_latency", + "latency / jitter / packet-loss / delayed-frames", + "halt", + halted=halted, + reason=reason, + oracle=oracle, + ) + + +def s_delayed_frame_settles() -> dict: + oracle, state, client = _new_env() + client.hover_unsettle_frames = 2 # delayed remote hover paint that settles + backend = _backend(client) + halted, reason = _guard(lambda: _drive_workflow(client, backend)) + return _record( + "delayed_frame_settles", + "latency / jitter / packet-loss / delayed-frames", + "pass", + halted=halted, + reason=reason, + oracle=oracle, + ) + + +def s_frame_never_settles() -> dict: + oracle, state, client = _new_env() + client.hover_unsettle_frames = 100000 # pixels never settle (jitter/loss) + backend = _backend(client) + halted, reason = _guard(lambda: _drive_workflow(client, backend)) + return _record( + "frame_never_settles", + "latency / jitter / packet-loss / delayed-frames", + "halt", + halted=halted, + reason=reason, + oracle=oracle, + ) + + +def _focus_note(client, backend) -> None: + backend.screenshot() + backend.click(*fx.NOTE_FIELD) + + +def s_keyboard_named_key_ok() -> dict: + oracle, state, client = _new_env() + backend = _backend(client) + + def run(): + _focus_note(client, backend) + backend.press("Tab") # a mapped named key delivers + + halted, reason = _guard(run) + return _record( + "keyboard_named_key_ok", + "keyboard-layout / IME", + "pass", + halted=halted, + reason=reason, + oracle=oracle, + kind="input", + ) + + +def s_ime_unmapped_key() -> dict: + oracle, state, client = _new_env() + client.unmapped_keys = {"compose"} # IME composition with no scancode + backend = _backend(client) + + def run(): + _focus_note(client, backend) + backend.press("compose") # unmapped -> fail loud, never mis-fire a key + + halted, reason = _guard(run) + return _record( + "ime_unmapped_key", + "keyboard-layout / IME", + "halt", + halted=halted, + reason=reason, + oracle=oracle, + kind="input", + ) + + +def s_clipboard_restricted() -> dict: + oracle, state, client = _new_env() + client.paste_blocked = True # Citrix clipboard channel disabled by policy + backend = _backend(client) + + def run(): + _focus_note(client, backend) + backend.press("ControlOrMeta+v") # blocked paste must fail loud + + halted, reason = _guard(run) + return _record( + "clipboard_restricted_paste", + "clipboard restrictions", + "halt", + halted=halted, + reason=reason, + oracle=oracle, + kind="input", + ) + + +def s_focus_theft() -> dict: + oracle, state, client = _new_env() + backend = _backend(client) + + def steal(cl, be): + cl.frontmost = False # another app steals focus after resolution + + halted, reason = _guard( + lambda: _drive_workflow(client, backend, after_acquire=steal) + ) + return _record( + "focus_theft_after_acquire", + "focus theft", + "halt", + halted=halted, + reason=reason, + oracle=oracle, + ) + + +def s_unexpected_overlay() -> dict: + oracle, state, client = _new_env() + backend = _backend(client) + + def overlay(cl, be): + cl.state.overlay = True # an unexpected dialog paints over the target + + halted, reason = _guard( + lambda: _drive_workflow(client, backend, after_acquire=overlay) + ) + return _record( + "unexpected_dialog_overlay", + "unexpected dialogs / overlays", + "halt", + halted=halted, + reason=reason, + oracle=oracle, + ) + + +def s_unverifiable_identity() -> dict: + oracle, state, client = _new_env(identity_broken=True) + backend = _backend(client) + halted, reason = _guard(lambda: _drive_workflow(client, backend)) + return _record( + "unverifiable_application_identity", + "ambiguous visual identity", + "halt", + halted=halted, + reason=reason, + oracle=oracle, + ) + + +def s_uncertain_submission() -> dict: + # After arming, an intervening observation invalidates the lease; the click + # must refuse rather than blind-fire the possibly-delivered submission. + oracle, state, client = _new_env() + backend = _backend(client) + halted, reason = _guard( + lambda: _drive_workflow(client, backend, invalidate_before_click=True) + ) + return _record( + "uncertain_submission_no_blind_retry", + "uncertain submission (no blind retry)", + "halt", + halted=halted, + reason=reason, + oracle=oracle, + ) + + +def s_duplicate_prevention() -> dict: + # One legitimate Save, then a second armed Save WITHOUT a fresh + # prepare/re-resolve must refuse -> the one-shot lease prevents a duplicate + # write. Exactly one record must exist. + oracle, state, client = _new_env() + backend = _backend(client) + _drive_workflow(client, backend) # first (intended) write + second_halted, second_reason = _guard( + lambda: ( + backend.acquire_actuation_frame(), + backend.click(*fx.SAVE_BUTTON), # no prepare_pointer_actuation + ) + ) + writes = oracle.write_count() + first_ok = oracle.last() == EXPECTED + passed = second_halted and writes == 1 and first_ok + return { + "id": "duplicate_write_prevention_one_shot", + "section10_condition": "duplicate prevention", + "kind": "write", + "expectation": "halt", + "outcome": "halted" if second_halted else "completed", + "halt_reason": second_reason, + "effect_expected": list(EXPECTED), + "effect_observed": list(oracle.last()) if oracle.last() else None, + "effect_verified": bool(first_ok), + "oracle_write_count": writes, + "model_calls": 0, + "silent_incorrect_success": bool(not second_halted and writes > 1), + "silent_write": bool(writes > 1), + "healthy_over_halt": False, + "passed": bool(passed), + } + + +def s_persisted_state_readback() -> dict: + # Healthy write verified strictly via the out-of-band oracle, never the + # on-screen "Saved" banner. + oracle, state, client = _new_env() + backend = _backend(client) + halted, reason = _guard(lambda: _drive_workflow(client, backend)) + rec = _record( + "persisted_state_readback", + "persisted-state readback (out-of-band effect verification)", + "pass", + halted=halted, + reason=reason, + oracle=oracle, + ) + rec["banner_shown"] = bool(state.committed) + rec["verified_via"] = "out_of_band_oracle" + return rec + + +def s_optimistic_banner_refused() -> dict: + # The app paints an optimistic "Saved" banner but the write is rejected by + # the system of record. Out-of-band verification must NOT confirm success. + oracle, state, client = _new_env(write_rejected=True) + backend = _backend(client) + halted, reason = _guard(lambda: _drive_workflow(client, backend)) + rec = _record( + "optimistic_banner_effect_refused", + "persisted-state readback (out-of-band effect verification)", + "effect_refused", + halted=halted, + reason=reason, + oracle=oracle, + ) + rec["banner_shown"] = bool(state.committed) + rec["independent_record_present"] = oracle.last() == EXPECTED + return rec + + +SCENARIOS: tuple[Callable[[], dict], ...] = ( + s_session_launch_ready, + s_application_not_ready, + s_reconnect_roaming, + s_session_lock, + s_session_unlock_recovery, + s_window_minimize, + s_window_occlusion, + s_window_move, + s_window_resize, + s_dpi_scale_consistent, + s_dpi_anisotropic, + s_single_monitor_geometry, + s_multimonitor_secondary_offset, + s_multimonitor_ambiguous, + s_codec_mild, + s_codec_severe, + s_stale_frame, + s_delayed_frame_settles, + s_frame_never_settles, + s_keyboard_named_key_ok, + s_ime_unmapped_key, + s_clipboard_restricted, + s_focus_theft, + s_unexpected_overlay, + s_unverifiable_identity, + s_uncertain_submission, + s_duplicate_prevention, + s_persisted_state_readback, + s_optimistic_banner_refused, +) + + +def _status_manifest() -> dict: + """Separate status dimensions -- NEVER collapsed into one 'Available'.""" + return { + "backend_shipped": { + "status": "shipped", + "detail": "CitrixWorkspaceBackend (RemoteDisplayBackend preset) is in " + "the package and qualified against the deterministic stand-in.", + }, + "installed_driver_available": { + "status": "shipped_host_clients", + "detail": "MacWindowClient (Quartz) and Win32WindowClient host drivers " + "ship; live capture/input requires per-host Screen-Recording / " + "Accessibility / integrity trust granted at deployment.", + }, + "real_protocol_environment_evidence": { + "status": "pending", + "detail": "No real Citrix ICA/HDX acceptance. Pending the " + "customer-environment (Accuro) lane: HDX codecs, ICA compression, and " + "the real Workspace-client transport are NOT exercised here.", + }, + "managed_execution_available": { + "status": "pending", + "detail": "Hosted/managed execution of this workflow is not asserted " + "by this stand-in.", + }, + "customer_controlled_execution_available": { + "status": "pending", + "detail": "Customer-controlled on-prem execution is the target of the " + "real-ICA release gate; not asserted by this stand-in.", + }, + "exact_application_qualification_available": { + "status": "pending", + "detail": "Exact published-application qualification (e.g. Accuro) is " + "performed in the customer's exact environment; not asserted here.", + }, + "deterministic_standin_qualification": { + "status": "qualified", + "detail": "The Section 10 ICA/HDX condition matrix is qualified against " + "a DETERMINISTIC SYNTHETIC STAND-IN (this campaign) -- NOT real " + "ICA/HDX.", + }, + } + + +def run_campaign() -> dict: + trials = [scenario() for scenario in SCENARIOS] + + mask_spec = fx.default_mask_spec() + mask_violations = fx.check_masks_reviewable(mask_spec) + + passes = sum(1 for t in trials if t["expectation"] == "pass") + halts = sum(1 for t in trials if t["expectation"] in ("halt", "effect_refused")) + silent_incorrect = sum(1 for t in trials if t["silent_incorrect_success"]) + silent_writes = sum(1 for t in trials if t["silent_write"]) + over_halts = sum(1 for t in trials if t["healthy_over_halt"]) + model_calls = sum(t["model_calls"] for t in trials) + scenarios_passed = sum(1 for t in trials if t["passed"]) + + accepted = ( + scenarios_passed == len(trials) + and silent_incorrect == 0 + and silent_writes == 0 + and over_halts == 0 + and model_calls == 0 + and not mask_violations + ) + + conditions = sorted({t["section10_condition"] for t in trials}) + + return { + "schema_version": "openadapt.citrix-ica-hdx-standin-qualification.v1", + "evidence_scope": "deterministic_synthetic_ica_hdx_standin", + "substrate": "citrix-workspace-backend-over-deterministic-ica-hdx-standin", + "backend_under_test": "CitrixWorkspaceBackend (RemoteDisplayBackend preset)", + "window_client": "SyntheticIcaWindowClient (in-process deterministic frames)", + "is_real_ica_hdx": False, + "label": ( + "DETERMINISTIC STAND-IN reproducing ICA/HDX conditions -- NOT real " + "Citrix ICA/HDX. No HDX codecs, no ICA compression, no real " + "Workspace-client transport. Real-protocol evidence pending the " + "customer-environment (Accuro) lane." + ), + "section10_conditions_covered": conditions, + "condition_count": len(conditions), + "enforcement_verified": [ + "every actuation uses a fresh frame (frame-freshness lease)", + "actuation stays bound to the authorized window and session", + "target re-resolved immediately before acting", + "one-shot actuation lease consumed exactly once (no double-fire)", + "DPI/scale calibration refused when anisotropic/uncalibrated", + "focus/occlusion binding enforced before every input edge", + "readiness + identity gating on the fresh actuation frame", + "out-of-band effect verification (never the on-screen banner)", + "zero model calls on every path (healthy and refusal)", + ], + "volatile_mask_review": { + "spec": { + "volatile": {k: list(v) for k, v in mask_spec.volatile.items()}, + "protected": {k: list(v) for k, v in mask_spec.protected.items()}, + }, + "violations": mask_violations, + "reviewable_and_safe": not mask_violations, + "note": "Volatile-region masks must never cover target, actionability, " + "identity, workflow-state, or effect-relevant regions.", + }, + "status_dimensions": _status_manifest(), + "failure_taxonomy": [ + "healthy_over_halt", + "silent_incorrect_success", + "silent_write", + "unsafe_completion", + "volatile_mask_covers_protected_region", + ], + "oracle": "in-process FaultDB read out of band (never the screen banner)", + "generated_at": datetime.now(timezone.utc).isoformat(), + "trials": trials, + "trial_count": len(trials), + "pass_expectations": passes, + "halt_expectations": halts, + "scenarios_passed": scenarios_passed, + "silent_incorrect_successes": silent_incorrect, + "silent_writes": silent_writes, + "healthy_over_halts": over_halts, + "model_calls": model_calls, + "deterministic_standin_accepted": bool(accepted), + "ica_hdx_accepted": False, + "ica_hdx_status": "pending_real_environment_customer_lane", + } + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument( + "--output", type=Path, default=Path("benchmark/citrix_ica_hdx/results.json") + ) + ap.add_argument( + "--status-output", + type=Path, + default=Path("benchmark/citrix_ica_hdx/status_manifest.json"), + ) + args = ap.parse_args() + + evidence = run_campaign() + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(evidence, indent=2, sort_keys=True) + "\n") + args.status_output.write_text( + json.dumps( + { + "schema_version": "openadapt.citrix-status-dimensions.v1", + "is_real_ica_hdx": False, + "generated_at": evidence["generated_at"], + "status_dimensions": evidence["status_dimensions"], + }, + indent=2, + sort_keys=True, + ) + + "\n" + ) + + payload = json.dumps(evidence, sort_keys=True).encode() + print(f"evidence sha256: {hashlib.sha256(payload).hexdigest()}") + print( + f"scenarios_passed: {evidence['scenarios_passed']}/{evidence['trial_count']} " + f"silent_incorrect_successes: {evidence['silent_incorrect_successes']} " + f"healthy_over_halts: {evidence['healthy_over_halts']} " + f"model_calls: {evidence['model_calls']}" + ) + print( + f"deterministic_standin_accepted: " + f"{evidence['deterministic_standin_accepted']} " + f"ica_hdx_accepted: {evidence['ica_hdx_accepted']}" + ) + return 0 if evidence["deterministic_standin_accepted"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmark/citrix_ica_hdx/status_manifest.json b/benchmark/citrix_ica_hdx/status_manifest.json new file mode 100644 index 00000000..2da5ae04 --- /dev/null +++ b/benchmark/citrix_ica_hdx/status_manifest.json @@ -0,0 +1,35 @@ +{ + "generated_at": "2026-07-26T18:13:53.666190+00:00", + "is_real_ica_hdx": false, + "schema_version": "openadapt.citrix-status-dimensions.v1", + "status_dimensions": { + "backend_shipped": { + "detail": "CitrixWorkspaceBackend (RemoteDisplayBackend preset) is in the package and qualified against the deterministic stand-in.", + "status": "shipped" + }, + "customer_controlled_execution_available": { + "detail": "Customer-controlled on-prem execution is the target of the real-ICA release gate; not asserted by this stand-in.", + "status": "pending" + }, + "deterministic_standin_qualification": { + "detail": "The Section 10 ICA/HDX condition matrix is qualified against a DETERMINISTIC SYNTHETIC STAND-IN (this campaign) -- NOT real ICA/HDX.", + "status": "qualified" + }, + "exact_application_qualification_available": { + "detail": "Exact published-application qualification (e.g. Accuro) is performed in the customer's exact environment; not asserted here.", + "status": "pending" + }, + "installed_driver_available": { + "detail": "MacWindowClient (Quartz) and Win32WindowClient host drivers ship; live capture/input requires per-host Screen-Recording / Accessibility / integrity trust granted at deployment.", + "status": "shipped_host_clients" + }, + "managed_execution_available": { + "detail": "Hosted/managed execution of this workflow is not asserted by this stand-in.", + "status": "pending" + }, + "real_protocol_environment_evidence": { + "detail": "No real Citrix ICA/HDX acceptance. Pending the customer-environment (Accuro) lane: HDX codecs, ICA compression, and the real Workspace-client transport are NOT exercised here.", + "status": "pending" + } + } +} diff --git a/public-artifacts.json b/public-artifacts.json index cf6b2d90..9db3c72d 100644 --- a/public-artifacts.json +++ b/public-artifacts.json @@ -131,6 +131,14 @@ "path": "benchmark/canvas_ladder/results.json", "sha256": "0a42afcda4853b7f9cc7e135e7c6a2a71373f0aa0ef39cb67a54c1eacdd5f8ad" }, + { + "path": "benchmark/citrix_ica_hdx/results.json", + "sha256": "8c394cbff9674b0048686f9438db26f3d1cfb5977c9aaa662b84e0ecc6c69ec7" + }, + { + "path": "benchmark/citrix_ica_hdx/status_manifest.json", + "sha256": "196154bedfeccfdbd398424382ea7c486933a71da97dcdfdc3fa0e9b772076b2" + }, { "path": "benchmark/citrix_workspace/results.json", "sha256": "64d588f93bc4fcd5ade9962483aeb85d5facc6b764babddd0b90b750f59d4156" diff --git a/tests/test_ica_hdx_qualification.py b/tests/test_ica_hdx_qualification.py new file mode 100644 index 00000000..a07326ad --- /dev/null +++ b/tests/test_ica_hdx_qualification.py @@ -0,0 +1,187 @@ +"""Deterministic in-process tests for the Citrix ICA/HDX STAND-IN campaign. + +These run in CI with no Docker, no network, and no Playwright: the whole +Section 10 condition matrix is exercised against the unmodified +``CitrixWorkspaceBackend`` through the synthetic fixture. They assert the +acceptance gate, the honesty invariants (this is NOT real ICA/HDX), and the +reviewable volatile-mask contract. +""" + +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + +import pytest + +REPO = Path(__file__).resolve().parents[1] +HARNESS = REPO / "benchmark" / "citrix_ica_hdx" / "run_ica_hdx_qualification.py" +FIXTURE = REPO / "benchmark" / "citrix_ica_hdx" / "fixture.py" + + +def _load(name: str, path: Path): + spec = importlib.util.spec_from_file_location(name, path) + assert spec is not None and spec.loader is not None + mod = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = mod + spec.loader.exec_module(mod) + return mod + + +harness = _load("ica_hdx_qualification", HARNESS) +fixture = _load("ica_hdx_fixture_test", FIXTURE) + + +# The full Section 10 condition matrix, each as a scenario with its expectation. +EXPECTED_SCENARIOS = { + "session_launch_ready": "pass", + "application_not_ready": "halt", + "reconnect_roaming_identity_change": "halt", + "session_lock": "halt", + "session_unlock_recovery": "pass", + "window_minimize": "halt", + "window_occlusion": "halt", + "window_move_after_acquire": "halt", + "window_resize_after_acquire": "halt", + "dpi_scale_consistent": "pass", + "dpi_anisotropic_uncalibrated": "halt", + "single_monitor_geometry": "pass", + "multimonitor_secondary_offset": "pass", + "multimonitor_ambiguous_window": "halt", + "codec_artifacts_mild_legible": "pass", + "codec_artifacts_severe_illegible": "halt", + "stale_frame_latency": "halt", + "delayed_frame_settles": "pass", + "frame_never_settles": "halt", + "keyboard_named_key_ok": "pass", + "ime_unmapped_key": "halt", + "clipboard_restricted_paste": "halt", + "focus_theft_after_acquire": "halt", + "unexpected_dialog_overlay": "halt", + "unverifiable_application_identity": "halt", + "uncertain_submission_no_blind_retry": "halt", + "duplicate_write_prevention_one_shot": "halt", + "persisted_state_readback": "pass", + "optimistic_banner_effect_refused": "effect_refused", +} + + +@pytest.fixture(scope="module") +def evidence() -> dict: + return harness.run_campaign() + + +def test_campaign_accepted_with_zero_unsafe_outcomes(evidence: dict) -> None: + assert evidence["deterministic_standin_accepted"] is True + assert evidence["scenarios_passed"] == evidence["trial_count"] + assert evidence["silent_incorrect_successes"] == 0 + assert evidence["silent_writes"] == 0 + assert evidence["healthy_over_halts"] == 0 + assert evidence["model_calls"] == 0 + + +def test_every_section10_condition_has_a_scenario(evidence: dict) -> None: + by_id = {t["id"]: t for t in evidence["trials"]} + assert set(by_id) == set(EXPECTED_SCENARIOS) + for sid, expectation in EXPECTED_SCENARIOS.items(): + assert by_id[sid]["expectation"] == expectation, sid + + +def test_pass_scenarios_complete_and_halt_scenarios_refuse(evidence: dict) -> None: + for t in evidence["trials"]: + if t["expectation"] == "pass": + assert t["outcome"] == "completed", t["id"] + assert t["passed"], t["id"] + elif t["expectation"] == "halt": + assert t["outcome"] == "halted", t["id"] + assert t["halt_reason"], t["id"] + assert t["passed"], t["id"] + + +def test_write_pass_scenarios_have_verified_out_of_band_effect(evidence: dict) -> None: + for t in evidence["trials"]: + if t["expectation"] == "pass" and t["kind"] == "write": + assert t["effect_verified"] is True, t["id"] + assert t["effect_observed"] == list(harness.EXPECTED), t["id"] + + +def test_no_silent_write_on_any_refusal(evidence: dict) -> None: + for t in evidence["trials"]: + if t["expectation"] in ("halt", "effect_refused"): + # The single legitimate write in the duplicate scenario is exempt. + if t["id"] == "duplicate_write_prevention_one_shot": + assert t["oracle_write_count"] == 1 + else: + assert t["oracle_write_count"] == 0, t["id"] + + +def test_optimistic_banner_is_not_confirmed_by_screen(evidence: dict) -> None: + by_id = {t["id"]: t for t in evidence["trials"]} + row = by_id["optimistic_banner_effect_refused"] + assert row["banner_shown"] is True # the screen claims "Saved" + assert row["independent_record_present"] is False # the oracle disagrees + assert row["passed"] is True # verification withholds success + + +def test_honesty_invariants_not_real_ica_hdx(evidence: dict) -> None: + assert evidence["is_real_ica_hdx"] is False + assert evidence["ica_hdx_accepted"] is False + assert evidence["ica_hdx_status"].startswith("pending") + assert "NOT real" in evidence["label"] + + +def test_status_dimensions_are_separate_and_real_protocol_pending( + evidence: dict, +) -> None: + dims = evidence["status_dimensions"] + # The six published dimensions are never collapsed into one "Available". + for key in ( + "backend_shipped", + "installed_driver_available", + "real_protocol_environment_evidence", + "managed_execution_available", + "customer_controlled_execution_available", + "exact_application_qualification_available", + ): + assert key in dims + assert dims["real_protocol_environment_evidence"]["status"] == "pending" + assert dims["backend_shipped"]["status"] == "shipped" + assert dims["deterministic_standin_qualification"]["status"] == "qualified" + + +def test_zero_model_calls_on_every_trial(evidence: dict) -> None: + assert all(t["model_calls"] == 0 for t in evidence["trials"]) + + +def test_default_volatile_mask_spec_is_reviewable_and_safe(evidence: dict) -> None: + review = evidence["volatile_mask_review"] + assert review["violations"] == [] + assert review["reviewable_and_safe"] is True + + +def test_volatile_mask_covering_a_protected_region_is_rejected() -> None: + spec = fixture.default_mask_spec() + bad = fixture.VolatileMaskSpec( + # A mask that (wrongly) covers the Save button target region. + volatile={**spec.volatile, "bad_mask": spec.protected["target_save_button"]}, + protected=spec.protected, + ) + violations = fixture.check_masks_reviewable(bad) + assert violations + assert any("target_save_button" in v for v in violations) + + +def test_mask_never_covers_target_identity_or_effect_regions() -> None: + spec = fixture.default_mask_spec() + for name in ( + "target_roster_row", + "target_note_field", + "target_save_button", + "identity_readiness", + "identity_application", + "workflow_state", + "effect_saved_banner", + ): + assert name in spec.protected + assert fixture.check_masks_reviewable(spec) == []