From 867268b8316689daf33159f4b01f6d9c83d5ec54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oskar=20Kwas=CC=81niewski?= Date: Thu, 24 Sep 2026 16:32:04 +0200 Subject: [PATCH 1/5] fix(android): carry the checked state of a checkable control on the snapshot node The snapshot helper never serialized `checked` or `checkable`, and the host reads only the helper's XML, so no later layer could recover it: a switch, checkbox, or radio button looked the same on or off, `get attrs` had no `checked` field, and snapshot text showed the control as plain. The helper now writes `checked` on every node Android reports as checkable, with both answers, so an unchecked switch reads `false`, a node that cannot be checked reads nothing, and so does a helper older than the attribute. The parser, the Android hierarchy node, and the published snapshot node carry it to `get attrs`, the unchanged-snapshot comparison, the selector digest, and the `[checked]` and `[unchecked]` markers in snapshot text. Both answers render because the diff compares the state and a checkable control rendered as plain would hide that it toggles. --- .../snapshothelper/AccessibilityTreeXml.java | 11 ++- .../__tests__/snapshot-diff-checked.test.ts | 49 +++++++++++++ .../capture-kit/src/snapshot/snapshot-diff.ts | 10 +-- .../snapshot/snapshot-freshness/android.ts | 11 +-- .../src/snapshot/snapshot-lines.ts | 23 ++++-- packages/kernel/src/snapshot.ts | 2 + .../__tests__/ui-hierarchy-checked.test.ts | 70 +++++++++++++++++++ .../src/ui-hierarchy-builder.ts | 1 + .../platform-android/src/ui-hierarchy-node.ts | 1 + packages/platform-android/src/ui-hierarchy.ts | 4 ++ src/__tests__/android-ui-hierarchy.test.ts | 23 ++++++ .../capture/runtime/snapshot-unchanged.ts | 1 + src/commands/output/snapshot.test.ts | 25 +++++++ src/daemon/response-views.ts | 1 + website/docs/docs/commands.md | 4 +- website/docs/docs/snapshots.md | 13 ++-- 16 files changed, 223 insertions(+), 26 deletions(-) create mode 100644 packages/capture-kit/src/snapshot/__tests__/snapshot-diff-checked.test.ts create mode 100644 packages/platform-android/src/__tests__/ui-hierarchy-checked.test.ts diff --git a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeXml.java b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeXml.java index c5c1da5249..775d622f7b 100644 --- a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeXml.java +++ b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeXml.java @@ -69,6 +69,12 @@ static void appendNode( // unselected control reports selected=false and a helper older than this attribute reports // nothing at all. appendAttribute(xml, "selected", Boolean.toString(node.isSelected())); + // Present only on a checkable control, with both answers: an unchecked switch reports + // checked=false, while a node that cannot be checked reports nothing, like a helper older than + // this attribute. + if (node.isCheckable()) { + appendAttribute(xml, "checked", Boolean.toString(node.isChecked())); + } boolean scrollable = node.isScrollable(); if (scrollable) { appendAttribute(xml, "scrollable", "true"); @@ -164,9 +170,8 @@ private static boolean isHeading(AccessibilityNodeInfo node) { return Build.VERSION.SDK_INT >= Build.VERSION_CODES.P && node.isHeading(); } - // Declared residue (agent-device #1832): checked / checkable / long-clickable are not serialized, - // so toggle state is invisible to agents. Adding them is a helper protocol change (new attributes - // + host parser + fields on the wire node), tracked there. + // Declared residue (agent-device #1832): long-clickable is not serialized. Adding it is a helper + // protocol change (new attribute + host parser + field on the wire node). private static void appendDrawingOrderAttribute(StringBuilder xml, AccessibilityNodeInfo node) { if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) { appendAttribute(xml, "drawing-order", Integer.toString(node.getDrawingOrder())); diff --git a/packages/capture-kit/src/snapshot/__tests__/snapshot-diff-checked.test.ts b/packages/capture-kit/src/snapshot/__tests__/snapshot-diff-checked.test.ts new file mode 100644 index 0000000000..bf95506373 --- /dev/null +++ b/packages/capture-kit/src/snapshot/__tests__/snapshot-diff-checked.test.ts @@ -0,0 +1,49 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import type { SnapshotNode } from '@agent-device/kernel/snapshot'; +import { buildSnapshotDiff } from '../snapshot-diff.ts'; + +function toggle(checked?: boolean): SnapshotNode { + return { + ref: 'e12', + index: 0, + depth: 0, + type: 'android.widget.Switch', + label: 'Wi-Fi switch', + enabled: true, + hittable: true, + ...(checked === undefined ? {} : { checked }), + }; +} + +test('a checked-only flip diffs as changed lines that read differently', () => { + const diff = buildSnapshotDiff([toggle(false)], [toggle(true)]); + const changed = diff.lines.filter((line) => line.kind !== 'unchanged'); + + assert.equal(diff.summary.unchanged, 0); + assert.equal(changed.length > 0, true); + // The comparable key carries the checked state, so the rendered line has to as well, and both + // answers render: the pair reads `[unchecked]` -> `[checked]`, not two identical lines. + assert.match(changed[0]!.text, /\[unchecked\]/); + assert.match(changed.at(-1)!.text, /\[checked\]/); +}); + +test.each([ + ['both unchecked', [toggle(false)], [toggle(false)]], + ['both unreported', [toggle()], [toggle()]], +])('a still toggle with %s diffs as unchanged', (_label, previous, current) => { + const diff = buildSnapshotDiff(previous, current); + + assert.equal( + diff.lines.every((line) => line.kind === 'unchanged'), + true, + ); +}); + +test('a node that cannot be checked never reads as unchecked', () => { + const diff = buildSnapshotDiff([toggle()], [toggle()]); + assert.equal( + diff.lines.every((line) => !line.text.includes('checked')), + true, + ); +}); diff --git a/packages/capture-kit/src/snapshot/snapshot-diff.ts b/packages/capture-kit/src/snapshot/snapshot-diff.ts index 3c7bf020b8..2b28a9198b 100644 --- a/packages/capture-kit/src/snapshot/snapshot-diff.ts +++ b/packages/capture-kit/src/snapshot/snapshot-diff.ts @@ -5,6 +5,7 @@ import { displayLabel, formatRole, formatSnapshotLine, + stateMarkers, } from './snapshot-lines.ts'; export type SnapshotDiffResult = { @@ -27,8 +28,9 @@ type SnapshotComparableLine = { function snapshotNodeToComparableLine(node: SnapshotNode, depthOverride?: number): string { const role = formatRole(node.type ?? 'Element'); const textPart = displayLabel(node, role); - const enabledPart = node.enabled === false ? 'disabled' : 'enabled'; - const selectedPart = node.selected === true ? 'selected' : 'unselected'; + // The states the rendered line prints are the states the key compares, from one function, so a + // fact the diff weighs is always visible in the line it prints. + const statePart = stateMarkers(node).join(','); const hittablePart = node.hittable === true ? 'hittable' : 'not-hittable'; const depthPart = String(depthOverride ?? node.depth ?? 0); // The rendered line carries the actions list, so the comparable key has to as @@ -36,9 +38,7 @@ function snapshotNodeToComparableLine(node: SnapshotNode, depthOverride?: number // silently differs from the baseline's. JSON-encoded because the names are // app-authored and may contain the field separator. const actionsPart = node.actions ? JSON.stringify(node.actions) : ''; - return [depthPart, role, textPart, enabledPart, selectedPart, hittablePart, actionsPart].join( - '|', - ); + return [depthPart, role, textPart, statePart, hittablePart, actionsPart].join('|'); } export function buildSnapshotDiff( diff --git a/packages/capture-kit/src/snapshot/snapshot-freshness/android.ts b/packages/capture-kit/src/snapshot/snapshot-freshness/android.ts index 8421f6236c..26955e8ab5 100644 --- a/packages/capture-kit/src/snapshot/snapshot-freshness/android.ts +++ b/packages/capture-kit/src/snapshot/snapshot-freshness/android.ts @@ -28,11 +28,12 @@ export function isNavigationSensitiveAction(command: string): boolean { /** * Route signature of an Android snapshot, from the fields the Android backend actually carries. - * The helper serializes no `role`, `checked` or `long-clickable` (declared residue, - * #1832), so a signature keying on them would compare constants and claim discrimination it does - * not have. `selected` is left out as a judgement call rather than an inability: a tab-bar tap - * flips it on two nodes, which the 90%-identical threshold below absorbs at every tree size this - * check runs on, so keying on it would only add capture retries. + * The helper serializes no `role` or `long-clickable` (declared residue, #1832), so a signature + * keying on them would compare constants and claim discrimination it does not have. `selected` + * and `checked` are left out as a judgement call rather than an inability: a tab-bar tap flips + * selection on two nodes and a toggle tap flips one checked state, which the 90%-identical + * threshold below absorbs at every tree size this check runs on, so keying on them would only add + * capture retries. */ export function buildSnapshotSignatures(nodes: SnapshotState['nodes']): string[] { return nodes.map((node) => diff --git a/packages/capture-kit/src/snapshot/snapshot-lines.ts b/packages/capture-kit/src/snapshot/snapshot-lines.ts index f6004dc995..0a2c598579 100644 --- a/packages/capture-kit/src/snapshot/snapshot-lines.ts +++ b/packages/capture-kit/src/snapshot/snapshot-lines.ts @@ -195,6 +195,21 @@ export function formatRole(type: string): string { return lookupRoleLabel(normalized) || normalized || 'element'; } +/** + * The state markers every rendering path prints, and the states a snapshot diff compares: the diff + * renders its lines without text-surface summarizing, and a fact it weighs has to be visible in the + * line it prints, or a flip reads as a changed pair whose two lines look identical. Both checked + * answers render, since a checkable control shown plain would hide that it toggles; a node that + * cannot be checked carries neither. + */ +export function stateMarkers(node: SnapshotNode): string[] { + const markers: string[] = []; + if (node.enabled === false) markers.push('disabled'); + if (node.selected === true) markers.push('selected'); + if (node.checked !== undefined) markers.push(node.checked ? 'checked' : 'unchecked'); + return markers; +} + function lookupRoleLabel(normalized: string): string | undefined { return Object.prototype.hasOwnProperty.call(ROLE_LABELS, normalized) ? ROLE_LABELS[normalized] @@ -238,13 +253,7 @@ function buildLineMetadata( options: SnapshotLineFormatOptions, textSurface: { text: string; isLargeSurface: boolean; shouldSummarize: boolean }, ): string[] { - const metadata: string[] = []; - if (node.enabled === false) metadata.push('disabled'); - // Selection is a state a snapshot diff can report as changed, and the diff renders its lines - // without text-surface summarizing. A fact the diff compares has to be visible in the line it - // prints, or a selection flip reads as a changed pair whose two lines look identical. - if (node.selected === true) metadata.push('selected'); - metadata.push(...(node.presentationHints ?? [])); + const metadata = [...stateMarkers(node), ...(node.presentationHints ?? [])]; if (!options.summarizeTextSurfaces) { return uniqueMetadata(metadata); } diff --git a/packages/kernel/src/snapshot.ts b/packages/kernel/src/snapshot.ts index a6a108d9c9..e1367ff19b 100644 --- a/packages/kernel/src/snapshot.ts +++ b/packages/kernel/src/snapshot.ts @@ -251,6 +251,8 @@ export type RawSnapshotNode = { rect?: Rect; enabled?: boolean; selected?: boolean; + /** Checked state of a checkable control (switch, checkbox, radio); absent means not checkable or unavailable. */ + checked?: boolean; focused?: boolean; /** Accessibility heading flag an app set on the node; absent means not a heading or unavailable. */ heading?: boolean; diff --git a/packages/platform-android/src/__tests__/ui-hierarchy-checked.test.ts b/packages/platform-android/src/__tests__/ui-hierarchy-checked.test.ts new file mode 100644 index 0000000000..fe0d5b29d9 --- /dev/null +++ b/packages/platform-android/src/__tests__/ui-hierarchy-checked.test.ts @@ -0,0 +1,70 @@ +import { expect, test } from 'vitest'; +import { buildUiHierarchySnapshot, parseUiHierarchyTree } from '../ui-hierarchy.ts'; + +// A settings row: a switch the helper reports as checkable beside a label that is not. The helper +// writes `checked` only on the checkable node, with both answers. +function togglesXml(wifiChecked: boolean): string { + return ` + + + + + + `; +} + +// A helper older than the `checked` attribute, or a switch it did not report as checkable. +const UNREPORTED_CHECKED_XML = + ''; + +function toggleNodes(raw: boolean, interactiveOnly = false, wifiChecked = true) { + const { nodes } = buildUiHierarchySnapshot( + parseUiHierarchyTree(togglesXml(wifiChecked)), + undefined, + { raw, interactiveOnly }, + ); + const byId = (identifier: string) => nodes.find((node) => node.identifier === identifier); + return { label: byId('wifi-label'), wifi: byId('wifi-switch'), small: byId('size-small') }; +} + +test.each([ + { raw: false, interactiveOnly: false }, + { raw: false, interactiveOnly: true }, + { raw: true, interactiveOnly: false }, + { raw: true, interactiveOnly: true }, +])( + 'checked state reaches snapshot nodes in every projection (raw=$raw, -i=$interactiveOnly)', + ({ raw, interactiveOnly }) => { + const { wifi, small } = toggleNodes(raw, interactiveOnly); + expect(wifi?.checked).toBe(true); + expect(small?.checked).toBe(false); + }, +); + +test.each([true, false])('the switch answers the state the helper observed (%s)', (wifiChecked) => { + expect(toggleNodes(false, false, wifiChecked).wifi?.checked).toBe(wifiChecked); +}); + +test('attrs answer explicit false where a node that cannot be checked answers nothing', () => { + // Serialized, because that is the answer an agent reads: an unavailable fact drops the key + // while JSON encodes an observed `false`. + const { label, small } = toggleNodes(false); + expect(JSON.parse(JSON.stringify(small)).checked).toBe(false); + expect(JSON.parse(JSON.stringify(label))).not.toHaveProperty('checked'); +}); + +test('an unreported checked state stays unknown instead of becoming false', () => { + const node = buildUiHierarchySnapshot(parseUiHierarchyTree(UNREPORTED_CHECKED_XML), undefined, { + raw: false, + }).nodes.find((node) => node.identifier === 'legacy-switch'); + expect(node).toBeDefined(); + expect(node?.checked).toBeUndefined(); +}); diff --git a/packages/platform-android/src/ui-hierarchy-builder.ts b/packages/platform-android/src/ui-hierarchy-builder.ts index e4f0ffaffb..087fb44bf2 100644 --- a/packages/platform-android/src/ui-hierarchy-builder.ts +++ b/packages/platform-android/src/ui-hierarchy-builder.ts @@ -356,6 +356,7 @@ function createAndroidRawSnapshotNode( selected: node.selected, heading: node.heading, roleDescription: node.roleDescription, + checked: node.checked, editable: node.editable, password: node.password, hintShowing: node.hintShowing, diff --git a/packages/platform-android/src/ui-hierarchy-node.ts b/packages/platform-android/src/ui-hierarchy-node.ts index 205c56ef4a..9dfb81cbc5 100644 --- a/packages/platform-android/src/ui-hierarchy-node.ts +++ b/packages/platform-android/src/ui-hierarchy-node.ts @@ -19,6 +19,7 @@ export type AndroidUiHierarchy = { selected?: boolean; heading?: boolean; roleDescription?: string; + checked?: boolean; editable?: boolean; password?: boolean; hintShowing?: boolean; diff --git a/packages/platform-android/src/ui-hierarchy.ts b/packages/platform-android/src/ui-hierarchy.ts index 7317d057ba..508939aa37 100644 --- a/packages/platform-android/src/ui-hierarchy.ts +++ b/packages/platform-android/src/ui-hierarchy.ts @@ -39,6 +39,8 @@ export type AndroidUiNodeMetadata = { heading?: boolean; /** Helper-only: the localized role description an app set beside the class, verbatim. */ roleDescription?: string; + /** Helper-only, present on a checkable control: the checked state of a switch, checkbox, or radio. */ + checked?: boolean; password?: boolean; editable?: boolean; selectionStart?: number; @@ -175,6 +177,7 @@ function readNodeAttributes(node: string): Omit { ...optionalBoolAttr('selected', 'selected'), ...optionalBoolAttr('heading', 'heading'), ...optionalStringAttr('roleDescription', 'role-description'), + ...optionalBoolAttr('checked', 'checked'), ...optionalNumberAttr('drawingOrder', 'drawing-order'), ...optionalBoolAttr('scrollable', 'scrollable'), ...optionalBoolAttr('canScrollForward', 'can-scroll-forward'), @@ -331,6 +334,7 @@ function normalizeAndroidUiHierarchyNode( selected: attrs.selected, heading: attrs.heading, roleDescription: attrs.roleDescription, + checked: attrs.checked, editable: attrs.editable, password: attrs.password, hintShowing: attrs.hintShowing, diff --git a/src/__tests__/android-ui-hierarchy.test.ts b/src/__tests__/android-ui-hierarchy.test.ts index d9edc4296d..e9149fd58c 100644 --- a/src/__tests__/android-ui-hierarchy.test.ts +++ b/src/__tests__/android-ui-hierarchy.test.ts @@ -272,6 +272,29 @@ test('a published Android snapshot carries the heading flag and the role descrip ); }); +// A settings screen: the helper writes `checked` on the checkable switch and radio, with both +// answers, and nothing on the label beside them. +const ANDROID_TOGGLES_XML = ` + + + + + +`; + +test('a published Android snapshot carries the checked state of checkable controls only', () => { + const nodes = publishUiHierarchy(ANDROID_TOGGLES_XML).nodes; + const byId = (identifier: string) => nodes.find((node) => node.identifier === identifier)!; + + assert.equal(byId('com.example.app:id/wifi-switch').checked, true); + assert.equal(byId('com.example.app:id/size-small').checked, false); + assert.equal(byId('com.example.app:id/wifi-label').checked, undefined); + assert.deepEqual( + Array.from(androidUiNodes(ANDROID_TOGGLES_XML)).map((node) => node.checked), + [undefined, undefined, true, false], + ); +}); + test('parseUiHierarchy discards stale inactive Android application windows', () => { const xml = ` diff --git a/src/commands/capture/runtime/snapshot-unchanged.ts b/src/commands/capture/runtime/snapshot-unchanged.ts index da581313bc..c03287df28 100644 --- a/src/commands/capture/runtime/snapshot-unchanged.ts +++ b/src/commands/capture/runtime/snapshot-unchanged.ts @@ -111,6 +111,7 @@ const PRESENTATION_SCALAR_FIELDS = { identifier: true, enabled: true, selected: true, + checked: true, focused: true, heading: true, roleDescription: true, diff --git a/src/commands/output/snapshot.test.ts b/src/commands/output/snapshot.test.ts index 8440986243..ccaff38177 100644 --- a/src/commands/output/snapshot.test.ts +++ b/src/commands/output/snapshot.test.ts @@ -820,3 +820,28 @@ test('formatSnapshotLine marks selection anywhere, and keeps text-surface metada assert.doesNotMatch(line, /\[editable\]/); assert.doesNotMatch(line, /\[scrollable\]/); }); + +test.each([ + [true, /\[checked\]/], + [false, /\[unchecked\]/], +])( + 'formatSnapshotLine renders both checked answers on the default path (%s)', + (checked, marker) => { + const line = formatSnapshotLine( + { + ref: 'e2', + index: 0, + depth: 0, + type: 'Switch', + label: 'Wi-Fi switch', + enabled: true, + checked, + }, + 0, + false, + ); + // A checkable control rendered as plain would hide that it toggles, and the diff compares the + // state, so the line prints whichever answer the helper observed. + assert.match(line, marker); + }, +); diff --git a/src/daemon/response-views.ts b/src/daemon/response-views.ts index 2454c8c59a..ee18e4a8c7 100644 --- a/src/daemon/response-views.ts +++ b/src/daemon/response-views.ts @@ -114,6 +114,7 @@ const SELECTOR_DIGEST_NODE_FIELDS = [ 'identifier', 'enabled', 'selected', + 'checked', 'focused', 'heading', 'roleDescription', diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 8a74a11cbb..d9eb54e337 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -391,8 +391,8 @@ agent-device get attrs @e1 act on stale React Native screens. API 23 cannot report sibling `drawing-order`, so this scan fails conservative and `androidSnapshot.occlusionScanUnavailable: true` discloses the difference. Android `--raw` is the acquired tree: it also keeps nodes Android marks invisible and stale - application windows. The helper does not report `checked`/`checkable` state, and it caps - captures at 5000 nodes before any `--scope` applies (`truncated: true`). + application windows. The helper caps captures at 5000 nodes before any `--scope` applies + (`truncated: true`). - `truncated: true` means the backend cut the capture at one of its limits — the Android helper and the iOS Simulator AX bridge at 5000 nodes, the XCTest runner and the web provider at their own bounds. Every backend walks the tree in document order, so what falls off is what comes diff --git a/website/docs/docs/snapshots.md b/website/docs/docs/snapshots.md index 99b08b5bf1..f07adc2541 100644 --- a/website/docs/docs/snapshots.md +++ b/website/docs/docs/snapshots.md @@ -117,16 +117,21 @@ the strategy owns which tiers it may use. ## Android node metadata Android snapshot nodes and `get attrs` (including the digest response) carry the native -`selected`, `heading`, `roleDescription`, `editable`, `password`, `hintShowing`, `placeholder`, -`selectionStart`, and `selectionEnd` facts whenever the accessibility tree reports them. Explicit -`false` and `0` are kept; an absent field means the fact was unavailable, not false. `hintShowing` -and `placeholder` need Android API 26 or later, `heading` API 28 or later. +`selected`, `checked`, `heading`, `roleDescription`, `editable`, `password`, `hintShowing`, +`placeholder`, `selectionStart`, and `selectionEnd` facts whenever the accessibility tree reports +them. Explicit `false` and `0` are kept; an absent field means the fact was unavailable, not false. +`hintShowing` and `placeholder` need Android API 26 or later, `heading` API 28 or later. - `selected` is the accessibility selected state an app sets on a control — the active bottom-tab or segmented-control item, or the chosen row of a list. Android reports it explicitly as `true` or `false`; an older helper APK omits the field, which means the answer is unavailable rather than unselected. Snapshot text marks the node `[selected]`, and `is selected`, a `selected=true` selector, and a Maestro `selected:` qualifier all match on it. +- `checked` is the checked state of a checkable control — a switch, a checkbox, a radio button, or a + view an app marked checkable. Android reports it as `true` or `false` on those nodes only; a node + that cannot be checked, or an older helper APK, omits the field. Snapshot text marks the node + `[checked]` or `[unchecked]`, so a toggle that reads as plain text is one Android did not report as + checkable. - `heading` is the accessibility heading flag an app sets on a node, the way React Native's `accessibilityRole="header"` does on a plain `View`; it is present only as `true`. - `roleDescription` is the localized role description an app sets beside the native class, verbatim From 1ceb6ba00c867dff953f9d5c133438986243f7fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oskar=20Kwas=CC=81niewski?= Date: Thu, 24 Sep 2026 17:47:59 +0200 Subject: [PATCH 2/5] fix(daemon): the interaction outcome weighs the states stateMarkers prints, and a scroll's container check matches content by identity Review follow-up. The interaction surface key folded enabled and selected but not checked, so a tap whose only effect was a toggle read as a no-op on the outcome lane while the diff called it a change, and a no-change retry would tap the switch straight back. The key now spreads stateMarkers, so the outcome lane, the unchanged-snapshot comparison, and the diff weigh one list. With checked in the key, a flip inside a scroll container beside an unrelated change elsewhere (the status clock) read as content moving within the container. discriminatingSurfaceChangedWithinRect now matches entries on the flip-tolerant identity where one exists, told apart by document order when repeated, so a state flip at the same rect is not movement. Tests: a checked-only flip is a change on the outcome lane; a flip at the same rect is not movement and a moved row is; the scroll claim is withheld for a flipped toggle inside the container; checked in both answers invalidates snapshot reuse and survives the selector digest. --- .../runtime/snapshot-unchanged.test.ts | 2 + .../interaction-outcome-policy.test.ts | 45 ++++++++++++++++++ src/daemon/__tests__/response-views.test.ts | 12 +++++ src/daemon/__tests__/scroll-movement.test.ts | 28 +++++++++++ src/daemon/interaction-outcome-policy.ts | 47 ++++++++++++++----- 5 files changed, 121 insertions(+), 13 deletions(-) diff --git a/src/commands/capture/runtime/snapshot-unchanged.test.ts b/src/commands/capture/runtime/snapshot-unchanged.test.ts index 1b3cf7ce6f..32f5adfd61 100644 --- a/src/commands/capture/runtime/snapshot-unchanged.test.ts +++ b/src/commands/capture/runtime/snapshot-unchanged.test.ts @@ -73,6 +73,8 @@ test.each>([ { contentDescription: 'Create a draft' }, { enabled: false }, { selected: true }, + { checked: true }, + { checked: false }, { focused: true }, { placeholder: 'Key echo' }, { hittable: false }, diff --git a/src/daemon/__tests__/interaction-outcome-policy.test.ts b/src/daemon/__tests__/interaction-outcome-policy.test.ts index 9cd0d34c5a..bea482ea32 100644 --- a/src/daemon/__tests__/interaction-outcome-policy.test.ts +++ b/src/daemon/__tests__/interaction-outcome-policy.test.ts @@ -467,3 +467,48 @@ test('discriminatingSurfaceChangedWithinRect counts content appearing inside the assert.equal(discriminatingSurfaceChangedWithinRect(before, after, LIST_RECT), true); }); + +// Android is the only producer of `checked`, and a tap whose only effect is a toggle changes nothing +// else on a screen without a mirrored label. The outcome lane has to read the flip as a change, or a +// no-change retry taps the switch straight back. +test('classifyInteractionSurfaceChange reads a checked-only flip as a change', () => { + const before = buildInteractionSurfaceSignature(makeToggleSnapshot(false).nodes); + const after = buildInteractionSurfaceSignature(makeToggleSnapshot(true).nodes); + + assert.equal(classifyInteractionSurfaceChange(before, after), 'changed'); +}); + +test('discriminatingSurfaceChangedWithinRect reads a flip at the same rect as no movement, and a moved row as movement', () => { + const rect = { x: 0, y: 0, width: 390, height: 844 }; + const signature = (checked: boolean, y?: number) => + buildInteractionSurfaceSignature(makeToggleSnapshot(checked, y).nodes); + + assert.equal( + discriminatingSurfaceChangedWithinRect(signature(false), signature(true), rect), + false, + ); + assert.equal( + discriminatingSurfaceChangedWithinRect(signature(false, 300), signature(false, 200), rect), + true, + ); +}); + +function makeToggleSnapshot(checked: boolean, y = 300): SnapshotState { + const base = makeSnapshot('Inbox'); + return { + ...base, + nodes: [ + ...base.nodes, + { + ref: 'e3', + index: 2, + parentIndex: 0, + type: 'android.widget.Switch', + identifier: 'wifi-switch', + label: 'Wi-Fi switch', + checked, + rect: { x: 300, y, width: 60, height: 40 }, + }, + ], + }; +} diff --git a/src/daemon/__tests__/response-views.test.ts b/src/daemon/__tests__/response-views.test.ts index 4d7cea294d..4b3a103026 100644 --- a/src/daemon/__tests__/response-views.test.ts +++ b/src/daemon/__tests__/response-views.test.ts @@ -338,6 +338,18 @@ test('attrs digest keeps explicit false/zero/empty field facts; unavailable ones } }); +test('attrs digest carries both checked answers, and none for a node that cannot be checked', () => { + for (const checked of [true, false]) { + const digest = getView!({ ref: 'e7', node: { ...MATCHED_NODE, checked } }, 'digest'); + expect(digest.node).toEqual({ ...COMPACT_NODE, checked }); + } + const plain = getView!({ ref: 'e7', node: MATCHED_NODE }, 'digest').node as Record< + string, + unknown + >; + expect('checked' in plain).toBe(false); +}); + test('find/get default and full return today’s shape unchanged (same reference)', () => { const data: DaemonResponseData = { ref: '@e7', text: 'Sign in', node: MATCHED_NODE }; expect(findView!(data, 'default')).toBe(data); diff --git a/src/daemon/__tests__/scroll-movement.test.ts b/src/daemon/__tests__/scroll-movement.test.ts index bf46f3cbdc..1097fa2ad3 100644 --- a/src/daemon/__tests__/scroll-movement.test.ts +++ b/src/daemon/__tests__/scroll-movement.test.ts @@ -388,6 +388,34 @@ test('a difference outside the scrolled container does not buy the movement clai assert.equal(spy.calls(), 1); }); +// Android reports `checked`, and a swipe that starts on a switch can flip it. On its own the flip is +// identity-invariant and reads as an unchanged surface. Beside a change elsewhere, such as the status +// clock ticking, the pair reads as changed, and the within-container check then sees a switch whose +// key differs at the same rect: a state flip, not the list moving, so it buys no movement claim. +test('a toggle the swipe flipped inside the container does not buy the movement claim', async () => { + const chrome = { + type: 'Image', + identifier: 'status-clock', + label: '2:40', + rect: { x: 20, y: 20, width: 60, height: 20 }, + } as SnapshotNode; + const toggle = (checked: boolean) => + ({ + type: 'android.widget.Switch', + identifier: 'wifi-switch', + label: 'Wi-Fi switch', + checked, + rect: { x: 300, y: 320, width: 60, height: 40 }, + }) as SnapshotNode; + const { observation } = observe({ + baseline: baselineOf([...screen(0), chrome, toggle(false)]), + screens: [[...screen(0), { ...chrome, label: '2:41' } as SnapshotNode, toggle(true)]], + }); + + assert.equal(await observation, 'unobserved'); + assertWithheld('change-outside-container'); +}); + test('a surface with no container to confine the claim to keeps the whole-surface answer', async () => { const rowsOnly = (offset: number) => screen(offset).filter((node) => node.type !== 'ScrollView'); const { observation } = observe({ diff --git a/src/daemon/interaction-outcome-policy.ts b/src/daemon/interaction-outcome-policy.ts index 3519a01e83..661f00a5d8 100644 --- a/src/daemon/interaction-outcome-policy.ts +++ b/src/daemon/interaction-outcome-policy.ts @@ -2,6 +2,7 @@ import type { CommandFlags } from '@agent-device/contracts/command'; import { isMobilePlatform } from '@agent-device/kernel/device'; import type { Rect, SnapshotNode, SnapshotState } from '@agent-device/kernel/snapshot'; import { collectKeyboardChromeRefs } from '@agent-device/capture-kit/snapshot-chrome'; +import { stateMarkers } from '@agent-device/capture-kit/snapshot-lines'; import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; import { isViewportRootNode } from '@agent-device/contracts/snapshot'; import { contextFromFlags, type DaemonCommandContext } from './context.ts'; @@ -465,8 +466,11 @@ export function summarizeDiscriminatingSurfaceDivergence( } /** - * Whether the DISCRIMINATING entries inside `rect` differ across a gesture, on the same key-matched, - * rect-tolerant view `haveIdenticalDiscriminatingSurfaces` uses — restricted to one region. + * Whether the DISCRIMINATING entries inside `rect` moved across a gesture: one left or entered the + * region, or its rect moved beyond tolerance. Entries match on the flip-tolerant `identity` where + * they have one, told apart by document order when repeated, and on `key` otherwise. A scroll moves + * content, while a state flip inside the container (a switch the swipe brushed, a row it selected) + * changes the key at the same rect and is not movement. * * A whole-surface difference is not automatically the gesture's doing. A captured tree carries system * chrome with it, and on Android the status bar clocks and icons change on their own while the app's @@ -478,17 +482,30 @@ export function discriminatingSurfaceChangedWithinRect( after: InteractionSurfaceSignature, rect: Rect, ): boolean { - const beforeInRect = discriminatingEntriesWithinRect(before, rect); - const afterByKey = new Map( - discriminatingEntriesWithinRect(after, rect).map((entry) => [entry.key, entry]), - ); - for (const entry of beforeInRect) { - const other = afterByKey.get(entry.key); + const beforeInRect = contentKeyed(discriminatingEntriesWithinRect(before, rect)); + const afterInRect = contentKeyed(discriminatingEntriesWithinRect(after, rect)); + for (const [content, entry] of beforeInRect) { + const other = afterInRect.get(content); if (!other) return true; if (!rectsWithinTolerance(entry, other)) return true; - afterByKey.delete(entry.key); + afterInRect.delete(content); } - return afterByKey.size > 0; + return afterInRect.size > 0; +} + +/** Entries by what they are rather than the state they are in; repeated content is told apart by document order. */ +function contentKeyed( + entries: InteractionSurfaceSignature, +): Map { + const occurrences = new Map(); + const keyed = new Map(); + for (const entry of entries) { + const content = entry.identity ?? entry.key; + const occurrence = occurrences.get(content) ?? 0; + occurrences.set(content, occurrence + 1); + keyed.set(`${content}|#${occurrence}`, entry); + } + return keyed; } function discriminatingEntriesWithinRect( @@ -552,7 +569,7 @@ function buildInteractionSurfaceEntry( /** * What the element IS — never where it sits, and never volatile state a gesture * is expected to change. `interactionSurfaceSemanticKey` deliberately folds in - * `hittable`/`enabled`/`selected` and an occurrence index, which is right for + * the states `stateMarkers` prints, `hittable`, and an occurrence index, which is right for * "did these two back-to-back captures agree" and wrong for "is this the same * element as before the gesture": scrolling flips `hittable` the moment a * node's centre leaves the viewport, so keying on it evicts precisely the @@ -584,6 +601,11 @@ function isNonDiscriminatingSurfaceNode( return isViewportRootNode(node) || (node.ref !== undefined && keyboardChromeRefs.has(node.ref)); } +/** + * What the element is and the state it is in. The states are the ones `stateMarkers` prints, so the + * outcome lane, the unchanged-snapshot comparison, and the diff weigh one list: a tap whose only + * effect is a toggle is a change here, not a no-op to retry. + */ function interactionSurfaceSemanticKey(node: SnapshotNode): string | undefined { const semanticKey = [ node.identifier, @@ -591,8 +613,7 @@ function interactionSurfaceSemanticKey(node: SnapshotNode): string | undefined { node.value, node.type, node.role, - node.enabled === false ? 'disabled' : 'enabled', - node.selected === true ? 'selected' : 'unselected', + ...stateMarkers(node), node.hittable === true ? 'hittable' : 'not-hittable', ] .map((value) => (typeof value === 'string' ? value.trim() : '')) From ad8d7a35dd372f6d287b09370fbf312e3d1cc6d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oskar=20Kwas=CC=81niewski?= Date: Thu, 24 Sep 2026 17:51:35 +0200 Subject: [PATCH 3/5] test(daemon): the capture-retry fixtures spell the interaction key with state markers The two hand-written pre-signatures carried the literal enabled and unselected segments the key no longer writes for a plain node, so the post-tap capture read as changed and the retry never fired. --- .../handlers/__tests__/snapshot-handler-capture-retry.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/daemon/handlers/__tests__/snapshot-handler-capture-retry.test.ts b/src/daemon/handlers/__tests__/snapshot-handler-capture-retry.test.ts index 59350cb063..7a6d2fa80d 100644 --- a/src/daemon/handlers/__tests__/snapshot-handler-capture-retry.test.ts +++ b/src/daemon/handlers/__tests__/snapshot-handler-capture-retry.test.ts @@ -96,7 +96,7 @@ test('captureSnapshot lazily retries pending no-change touch before returning fr attemptsRemaining: 2, preSignature: [ { - key: 'open-feed|Open feed||Button||enabled|unselected|hittable|#0', + key: 'open-feed|Open feed||Button||hittable|#0', x: 20, y: 120, width: 160, @@ -239,7 +239,7 @@ test('captureSnapshot retries pending tap outcome before post-gesture stabilizat attemptsRemaining: 2, preSignature: [ { - key: '|Navigate to Third||android.widget.Button||enabled|unselected|hittable|#0', + key: '|Navigate to Third||android.widget.Button||hittable|#0', x: 302, y: 1301, width: 476, From 0358c80ec5d81676ab06962439b08b8eb7a3d7e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oskar=20Kwas=CC=81niewski?= Date: Thu, 24 Sep 2026 18:00:16 +0200 Subject: [PATCH 4/5] fix(daemon): a surface entry carries a state-free content key, so an anonymous toggle is not movement Review follow-up. The within-container movement check fell back to `key` for an entry without an identity, and `key` carries the checked state, so an unlabelled switch a swipe brushed still read as content moving. Every entry now carries `content`: the identity where the node has one, else its type and role; the check compares on that alone. --- .../interaction-outcome-policy.test.ts | 43 +++++++++++-------- .../snapshot-handler-capture-retry.test.ts | 4 ++ src/daemon/interaction-outcome-policy.ts | 21 +++++---- src/daemon/session-state.ts | 7 +++ 4 files changed, 49 insertions(+), 26 deletions(-) diff --git a/src/daemon/__tests__/interaction-outcome-policy.test.ts b/src/daemon/__tests__/interaction-outcome-policy.test.ts index bea482ea32..20ef6154b2 100644 --- a/src/daemon/__tests__/interaction-outcome-policy.test.ts +++ b/src/daemon/__tests__/interaction-outcome-policy.test.ts @@ -478,22 +478,30 @@ test('classifyInteractionSurfaceChange reads a checked-only flip as a change', ( assert.equal(classifyInteractionSurfaceChange(before, after), 'changed'); }); -test('discriminatingSurfaceChangedWithinRect reads a flip at the same rect as no movement, and a moved row as movement', () => { - const rect = { x: 0, y: 0, width: 390, height: 844 }; - const signature = (checked: boolean, y?: number) => - buildInteractionSurfaceSignature(makeToggleSnapshot(checked, y).nodes); - - assert.equal( - discriminatingSurfaceChangedWithinRect(signature(false), signature(true), rect), - false, - ); - assert.equal( - discriminatingSurfaceChangedWithinRect(signature(false, 300), signature(false, 200), rect), - true, - ); -}); - -function makeToggleSnapshot(checked: boolean, y = 300): SnapshotState { +test.each([ + { anonymous: false, label: 'a labelled switch' }, + { anonymous: true, label: 'an anonymous switch' }, +])( + 'discriminatingSurfaceChangedWithinRect reads a flip of $label at the same rect as no movement, and a moved one as movement', + ({ anonymous }) => { + const rect = { x: 0, y: 0, width: 390, height: 844 }; + const signature = (checked: boolean, y?: number) => + buildInteractionSurfaceSignature(makeToggleSnapshot(checked, y, anonymous).nodes); + + assert.equal( + discriminatingSurfaceChangedWithinRect(signature(false), signature(true), rect), + false, + ); + assert.equal( + discriminatingSurfaceChangedWithinRect(signature(false, 300), signature(false, 200), rect), + true, + ); + }, +); + +// An anonymous switch has no identity, so its content is its type: the flip still changes only the +// key, and a swipe that brushed it must not read as the list moving. +function makeToggleSnapshot(checked: boolean, y = 300, anonymous = false): SnapshotState { const base = makeSnapshot('Inbox'); return { ...base, @@ -504,8 +512,7 @@ function makeToggleSnapshot(checked: boolean, y = 300): SnapshotState { index: 2, parentIndex: 0, type: 'android.widget.Switch', - identifier: 'wifi-switch', - label: 'Wi-Fi switch', + ...(anonymous ? {} : { identifier: 'wifi-switch', label: 'Wi-Fi switch' }), checked, rect: { x: 300, y, width: 60, height: 40 }, }, diff --git a/src/daemon/handlers/__tests__/snapshot-handler-capture-retry.test.ts b/src/daemon/handlers/__tests__/snapshot-handler-capture-retry.test.ts index 7a6d2fa80d..0d00967083 100644 --- a/src/daemon/handlers/__tests__/snapshot-handler-capture-retry.test.ts +++ b/src/daemon/handlers/__tests__/snapshot-handler-capture-retry.test.ts @@ -97,6 +97,8 @@ test('captureSnapshot lazily retries pending no-change touch before returning fr preSignature: [ { key: 'open-feed|Open feed||Button||hittable|#0', + identity: 'open-feed|Open feed||Button', + content: 'open-feed|Open feed||Button', x: 20, y: 120, width: 160, @@ -240,6 +242,8 @@ test('captureSnapshot retries pending tap outcome before post-gesture stabilizat preSignature: [ { key: '|Navigate to Third||android.widget.Button||hittable|#0', + identity: '|Navigate to Third||android.widget.Button', + content: '|Navigate to Third||android.widget.Button', x: 302, y: 1301, width: 476, diff --git a/src/daemon/interaction-outcome-policy.ts b/src/daemon/interaction-outcome-policy.ts index 661f00a5d8..40c78cea55 100644 --- a/src/daemon/interaction-outcome-policy.ts +++ b/src/daemon/interaction-outcome-policy.ts @@ -467,10 +467,10 @@ export function summarizeDiscriminatingSurfaceDivergence( /** * Whether the DISCRIMINATING entries inside `rect` moved across a gesture: one left or entered the - * region, or its rect moved beyond tolerance. Entries match on the flip-tolerant `identity` where - * they have one, told apart by document order when repeated, and on `key` otherwise. A scroll moves - * content, while a state flip inside the container (a switch the swipe brushed, a row it selected) - * changes the key at the same rect and is not movement. + * region, or its rect moved beyond tolerance. Entries match on `content`: the flip-tolerant + * `identity` where they have one, the type and role of an anonymous node otherwise, told apart by + * document order when repeated. A scroll moves content, while a state flip inside the container (a + * switch the swipe brushed, a row it selected) changes the key at the same rect and is not movement. * * A whole-surface difference is not automatically the gesture's doing. A captured tree carries system * chrome with it, and on Android the status bar clocks and icons change on their own while the app's @@ -500,10 +500,9 @@ function contentKeyed( const occurrences = new Map(); const keyed = new Map(); for (const entry of entries) { - const content = entry.identity ?? entry.key; - const occurrence = occurrences.get(content) ?? 0; - occurrences.set(content, occurrence + 1); - keyed.set(`${content}|#${occurrence}`, entry); + const occurrence = occurrences.get(entry.content) ?? 0; + occurrences.set(entry.content, occurrence + 1); + keyed.set(`${entry.content}|#${occurrence}`, entry); } return keyed; } @@ -558,6 +557,7 @@ function buildInteractionSurfaceEntry( return { key: `${semanticKey}|#${occurrence}`, ...(identity ? { identity } : {}), + content: interactionSurfaceContent(node, identity), x: Math.round(node.rect.x), y: Math.round(node.rect.y), width: Math.round(node.rect.width), @@ -566,6 +566,11 @@ function buildInteractionSurfaceEntry( }; } +/** What the element is without the state it is in: its identity, else the type and role of an anonymous node. */ +function interactionSurfaceContent(node: SnapshotNode, identity: string | undefined): string { + return identity ?? `${node.type ?? ''}|${node.role ?? ''}`; +} + /** * What the element IS — never where it sits, and never volatile state a gesture * is expected to change. `interactionSurfaceSemanticKey` deliberately folds in diff --git a/src/daemon/session-state.ts b/src/daemon/session-state.ts index d7ef7e626d..354b928636 100644 --- a/src/daemon/session-state.ts +++ b/src/daemon/session-state.ts @@ -53,6 +53,13 @@ export type InteractionSurfaceEntry = { * comparison entirely. */ identity?: string; + /** + * What the element is without the state it is in: `identity` where the node + * has one, else its type and role. A within-container movement check compares + * entries on this, in document order when repeated, because a state flip + * changes `key` at the same rect and is not movement. + */ + content: string; x: number; y: number; width: number; From 6dae0a126c19bac6cfc24c9b6b683aa2998f64ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oskar=20Kwas=CC=81niewski?= Date: Thu, 24 Sep 2026 19:14:44 +0200 Subject: [PATCH 5/5] fix(maestro): the snapshot signature weighs checked, so a toggle tap is not a no-op to retap maestroSnapshotSignature hashed label, value, enabled, selected, focused and bounds; a switch, checkbox or radio whose only observable effect is its checked state produced the same signature before and after the tap, so the settle and the no-change retry read the tap as a no-op and could tap it back. checked joins the hashed object the way the interaction outcome key already weighs it. --- .../daemon-runtime-port-observation.test.ts | 14 ++++++++++++++ .../daemon-port/daemon-runtime-port-observation.ts | 1 + 2 files changed, 15 insertions(+) diff --git a/packages/maestro/src/daemon-port/__tests__/daemon-runtime-port-observation.test.ts b/packages/maestro/src/daemon-port/__tests__/daemon-runtime-port-observation.test.ts index 8d58d117d0..22e6b64b19 100644 --- a/packages/maestro/src/daemon-port/__tests__/daemon-runtime-port-observation.test.ts +++ b/packages/maestro/src/daemon-port/__tests__/daemon-runtime-port-observation.test.ts @@ -359,12 +359,26 @@ test('normalizes absent attributes like Maestro iOS hierarchy mapping', () => { enabled: false, selected: false, focused: false, + checked: false, }, ]); expect(maestroSnapshotSignature(first)).toBe(maestroSnapshotSignature(second)); }); +test('a checked-only flip changes the snapshot signature', () => { + // An Android switch tapped on: nothing about it moves or renames, only `checked` flips. The + // same signature before and after would read the tap as a no-op and retap it off again. + const off = makeSnapshot([ + { index: 0, type: 'android.widget.Switch', label: 'Wi-Fi', checked: false }, + ]); + const on = makeSnapshot([ + { index: 0, type: 'android.widget.Switch', label: 'Wi-Fi', checked: true }, + ]); + + expect(maestroSnapshotSignature(off)).not.toBe(maestroSnapshotSignature(on)); +}); + test('excludes agent-device presentation metadata from Maestro hierarchy signatures', () => { const first = makeSnapshot([ { diff --git a/packages/maestro/src/daemon-port/daemon-runtime-port-observation.ts b/packages/maestro/src/daemon-port/daemon-runtime-port-observation.ts index c90f173917..f6ce5ffd6d 100644 --- a/packages/maestro/src/daemon-port/daemon-runtime-port-observation.ts +++ b/packages/maestro/src/daemon-port/daemon-runtime-port-observation.ts @@ -408,6 +408,7 @@ export function maestroSnapshotSignature(snapshot: SnapshotState): string { enabled: node.enabled ?? false, selected: node.selected ?? false, focused: node.focused ?? false, + checked: node.checked ?? false, bounds: maestroSnapshotBounds(node.rect), })), ),