Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,12 @@ static void appendNode(
// unselected control reports selected=false and a helper older than this attribute reports
// nothing at all.
appendAttribute(xml, "selected", Boolean.toString(node.isSelected()));
// Present only on a checkable control, with both answers: an unchecked switch reports
// checked=false, while a node that cannot be checked reports nothing, like a helper older than
// this attribute.
if (node.isCheckable()) {
appendAttribute(xml, "checked", Boolean.toString(node.isChecked()));
}
boolean scrollable = node.isScrollable();
if (scrollable) {
appendAttribute(xml, "scrollable", "true");
Expand Down Expand Up @@ -164,9 +170,8 @@ private static boolean isHeading(AccessibilityNodeInfo node) {
return Build.VERSION.SDK_INT >= Build.VERSION_CODES.P && node.isHeading();
}

// Declared residue (agent-device #1832): checked / checkable / long-clickable are not serialized,
// so toggle state is invisible to agents. Adding them is a helper protocol change (new attributes
// + host parser + fields on the wire node), tracked there.
// Declared residue (agent-device #1832): long-clickable is not serialized. Adding it is a helper
// protocol change (new attribute + host parser + field on the wire node).
private static void appendDrawingOrderAttribute(StringBuilder xml, AccessibilityNodeInfo node) {
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) {
appendAttribute(xml, "drawing-order", Integer.toString(node.getDrawingOrder()));
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
import assert from 'node:assert/strict';
import { test } from 'vitest';
import type { SnapshotNode } from '@agent-device/kernel/snapshot';
import { buildSnapshotDiff } from '../snapshot-diff.ts';

function toggle(checked?: boolean): SnapshotNode {
return {
ref: 'e12',
index: 0,
depth: 0,
type: 'android.widget.Switch',
label: 'Wi-Fi switch',
enabled: true,
hittable: true,
...(checked === undefined ? {} : { checked }),
};
}

test('a checked-only flip diffs as changed lines that read differently', () => {
const diff = buildSnapshotDiff([toggle(false)], [toggle(true)]);
const changed = diff.lines.filter((line) => line.kind !== 'unchanged');

assert.equal(diff.summary.unchanged, 0);
assert.equal(changed.length > 0, true);
// The comparable key carries the checked state, so the rendered line has to as well, and both
// answers render: the pair reads `[unchecked]` -> `[checked]`, not two identical lines.
assert.match(changed[0]!.text, /\[unchecked\]/);
assert.match(changed.at(-1)!.text, /\[checked\]/);
});

test.each([
['both unchecked', [toggle(false)], [toggle(false)]],
['both unreported', [toggle()], [toggle()]],
])('a still toggle with %s diffs as unchanged', (_label, previous, current) => {
const diff = buildSnapshotDiff(previous, current);

assert.equal(
diff.lines.every((line) => line.kind === 'unchanged'),
true,
);
});

test('a node that cannot be checked never reads as unchecked', () => {
const diff = buildSnapshotDiff([toggle()], [toggle()]);
assert.equal(
diff.lines.every((line) => !line.text.includes('checked')),
true,
);
});
10 changes: 5 additions & 5 deletions packages/capture-kit/src/snapshot/snapshot-diff.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import {
displayLabel,
formatRole,
formatSnapshotLine,
stateMarkers,
} from './snapshot-lines.ts';

export type SnapshotDiffResult = {
Expand All @@ -27,18 +28,17 @@ type SnapshotComparableLine = {
function snapshotNodeToComparableLine(node: SnapshotNode, depthOverride?: number): string {
const role = formatRole(node.type ?? 'Element');
const textPart = displayLabel(node, role);
const enabledPart = node.enabled === false ? 'disabled' : 'enabled';
const selectedPart = node.selected === true ? 'selected' : 'unselected';
// The states the rendered line prints are the states the key compares, from one function, so a
// fact the diff weighs is always visible in the line it prints.
const statePart = stateMarkers(node).join(',');
const hittablePart = node.hittable === true ? 'hittable' : 'not-hittable';
const depthPart = String(depthOverride ?? node.depth ?? 0);
// The rendered line carries the actions list, so the comparable key has to as
// well: otherwise an action-only change diffs as 'unchanged' while its text
// silently differs from the baseline's. JSON-encoded because the names are
// app-authored and may contain the field separator.
const actionsPart = node.actions ? JSON.stringify(node.actions) : '';
return [depthPart, role, textPart, enabledPart, selectedPart, hittablePart, actionsPart].join(
'|',
);
return [depthPart, role, textPart, statePart, hittablePart, actionsPart].join('|');
}

export function buildSnapshotDiff(
Expand Down
11 changes: 6 additions & 5 deletions packages/capture-kit/src/snapshot/snapshot-freshness/android.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,11 +28,12 @@ export function isNavigationSensitiveAction(command: string): boolean {

/**
* Route signature of an Android snapshot, from the fields the Android backend actually carries.
* The helper serializes no `role`, `checked` or `long-clickable` (declared residue,
* #1832), so a signature keying on them would compare constants and claim discrimination it does
* not have. `selected` is left out as a judgement call rather than an inability: a tab-bar tap
* flips it on two nodes, which the 90%-identical threshold below absorbs at every tree size this
* check runs on, so keying on it would only add capture retries.
* The helper serializes no `role` or `long-clickable` (declared residue, #1832), so a signature
* keying on them would compare constants and claim discrimination it does not have. `selected`
* and `checked` are left out as a judgement call rather than an inability: a tab-bar tap flips
* selection on two nodes and a toggle tap flips one checked state, which the 90%-identical
* threshold below absorbs at every tree size this check runs on, so keying on them would only add
* capture retries.
*/
export function buildSnapshotSignatures(nodes: SnapshotState['nodes']): string[] {
return nodes.map((node) =>
Expand Down
23 changes: 16 additions & 7 deletions packages/capture-kit/src/snapshot/snapshot-lines.ts
Original file line number Diff line number Diff line change
Expand Up @@ -195,6 +195,21 @@ export function formatRole(type: string): string {
return lookupRoleLabel(normalized) || normalized || 'element';
}

/**
* The state markers every rendering path prints, and the states a snapshot diff compares: the diff
* renders its lines without text-surface summarizing, and a fact it weighs has to be visible in the
* line it prints, or a flip reads as a changed pair whose two lines look identical. Both checked
* answers render, since a checkable control shown plain would hide that it toggles; a node that
* cannot be checked carries neither.
*/
export function stateMarkers(node: SnapshotNode): string[] {
const markers: string[] = [];
if (node.enabled === false) markers.push('disabled');
if (node.selected === true) markers.push('selected');
if (node.checked !== undefined) markers.push(node.checked ? 'checked' : 'unchecked');

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

stateMarkers now declares which states a comparison weighs, and this PR applies that answer in snapshot-unchanged.ts and the Android freshness comment. A third state signature still disagrees: interactionSurfaceSemanticKey in src/daemon/interaction-outcome-policy.ts:595 folds enabled, selected, and hittable but not checked. Android is checked's only producer, so a tap whose only effect is a toggle can be changed on the diff lane but unchanged on the interaction-outcome lane, causing retry or delayed-recheck work for a gesture that actually worked. Please either fold checked into that key, or make the exclusion explicit and test it. If including it, also account for discriminatingSurfaceChangedWithinRect in src/daemon/scroll-movement.ts:385, where an unrelated in-container toggle now counts as scroll movement.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Folded in, e5bba1b. interactionSurfaceSemanticKey now spreads stateMarkers(node) for its state segments, so the outcome lane, the unchanged-snapshot comparison, and the diff weigh one list; a checked-only flip classifies changed instead of feeding a no-change retry that would tap the switch back.

For the scroll lane: the flip alone never reaches discriminatingSurfaceChangedWithinRect, since classifyBaselineSurfaceEvidence matches on the flip-tolerant identity and reads it as unchanged. It does reach it beside an unrelated change, such as the status clock ticking, and there a key-matched view read the flipped switch as content moving inside the container. The function now matches entries on identity where one exists, told apart by document order when repeated, and on key otherwise, so a state flip at the same rect is not movement. haveIdenticalDiscriminatingSurfaces keeps its key-matched veto, as its comment asks.

Tests: a checked-only flip is changed on the outcome lane; a flip at the same rect is no movement while a moved row is; the scroll claim is withheld (change-outside-container) for a flipped toggle inside the container beside a clock tick; both checked answers invalidate snapshot reuse and survive the selector digest.

return markers;
}

function lookupRoleLabel(normalized: string): string | undefined {
return Object.prototype.hasOwnProperty.call(ROLE_LABELS, normalized)
? ROLE_LABELS[normalized]
Expand Down Expand Up @@ -238,13 +253,7 @@ function buildLineMetadata(
options: SnapshotLineFormatOptions,
textSurface: { text: string; isLargeSurface: boolean; shouldSummarize: boolean },
): string[] {
const metadata: string[] = [];
if (node.enabled === false) metadata.push('disabled');
// Selection is a state a snapshot diff can report as changed, and the diff renders its lines
// without text-surface summarizing. A fact the diff compares has to be visible in the line it
// prints, or a selection flip reads as a changed pair whose two lines look identical.
if (node.selected === true) metadata.push('selected');
metadata.push(...(node.presentationHints ?? []));
const metadata = [...stateMarkers(node), ...(node.presentationHints ?? [])];
if (!options.summarizeTextSurfaces) {
return uniqueMetadata(metadata);
}
Expand Down
2 changes: 2 additions & 0 deletions packages/kernel/src/snapshot.ts
Original file line number Diff line number Diff line change
Expand Up @@ -251,6 +251,8 @@ export type RawSnapshotNode = {
rect?: Rect;
enabled?: boolean;
selected?: boolean;
/** Checked state of a checkable control (switch, checkbox, radio); absent means not checkable or unavailable. */
checked?: boolean;
Comment on lines +254 to +255
focused?: boolean;
/** Accessibility heading flag an app set on the node; absent means not a heading or unavailable. */
heading?: boolean;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -359,12 +359,26 @@ test('normalizes absent attributes like Maestro iOS hierarchy mapping', () => {
enabled: false,
selected: false,
focused: false,
checked: false,
},
]);

expect(maestroSnapshotSignature(first)).toBe(maestroSnapshotSignature(second));
});

test('a checked-only flip changes the snapshot signature', () => {
// An Android switch tapped on: nothing about it moves or renames, only `checked` flips. The
// same signature before and after would read the tap as a no-op and retap it off again.
const off = makeSnapshot([
{ index: 0, type: 'android.widget.Switch', label: 'Wi-Fi', checked: false },
]);
const on = makeSnapshot([
{ index: 0, type: 'android.widget.Switch', label: 'Wi-Fi', checked: true },
]);

expect(maestroSnapshotSignature(off)).not.toBe(maestroSnapshotSignature(on));
});

test('excludes agent-device presentation metadata from Maestro hierarchy signatures', () => {
const first = makeSnapshot([
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -408,6 +408,7 @@ export function maestroSnapshotSignature(snapshot: SnapshotState): string {
enabled: node.enabled ?? false,
selected: node.selected ?? false,
focused: node.focused ?? false,
checked: node.checked ?? false,
bounds: maestroSnapshotBounds(node.rect),
})),
),
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
import { expect, test } from 'vitest';
import { buildUiHierarchySnapshot, parseUiHierarchyTree } from '../ui-hierarchy.ts';

// A settings row: a switch the helper reports as checkable beside a label that is not. The helper
// writes `checked` only on the checkable node, with both answers.
function togglesXml(wifiChecked: boolean): string {
return `<hierarchy>
<node class="android.widget.LinearLayout" resource-id="toggles" bounds="[0,0][400,200]"
window-index="0" window-type="1" window-layer="1" window-active="true" window-focused="true"
window-bounds="[0,0][400,800]" visible-to-user="true" enabled="true">
<node class="android.widget.TextView" resource-id="wifi-label" text="Wi-Fi"
bounds="[0,0][200,100]" enabled="true" visible-to-user="true" />
<node class="android.widget.Switch" resource-id="wifi-switch" content-desc="Wi-Fi switch"
bounds="[200,0][400,100]" clickable="true" enabled="true" visible-to-user="true"
checked="${wifiChecked}" />
<node class="android.widget.RadioButton" resource-id="size-small" text="Small"
bounds="[0,100][400,200]" clickable="true" enabled="true" visible-to-user="true"
checked="false" />
</node>
</hierarchy>`;
}

// A helper older than the `checked` attribute, or a switch it did not report as checkable.
const UNREPORTED_CHECKED_XML =
'<hierarchy><node class="android.widget.Switch" resource-id="legacy-switch" content-desc="Wi-Fi"' +
' bounds="[200,0][400,100]" clickable="true" enabled="true" visible-to-user="true" /></hierarchy>';

function toggleNodes(raw: boolean, interactiveOnly = false, wifiChecked = true) {
const { nodes } = buildUiHierarchySnapshot(
parseUiHierarchyTree(togglesXml(wifiChecked)),
undefined,
{ raw, interactiveOnly },
);
const byId = (identifier: string) => nodes.find((node) => node.identifier === identifier);
return { label: byId('wifi-label'), wifi: byId('wifi-switch'), small: byId('size-small') };
}

test.each([
{ raw: false, interactiveOnly: false },
{ raw: false, interactiveOnly: true },
{ raw: true, interactiveOnly: false },
{ raw: true, interactiveOnly: true },
])(
'checked state reaches snapshot nodes in every projection (raw=$raw, -i=$interactiveOnly)',
({ raw, interactiveOnly }) => {
const { wifi, small } = toggleNodes(raw, interactiveOnly);
expect(wifi?.checked).toBe(true);
expect(small?.checked).toBe(false);
},
);

test.each([true, false])('the switch answers the state the helper observed (%s)', (wifiChecked) => {
expect(toggleNodes(false, false, wifiChecked).wifi?.checked).toBe(wifiChecked);
});

test('attrs answer explicit false where a node that cannot be checked answers nothing', () => {
// Serialized, because that is the answer an agent reads: an unavailable fact drops the key
// while JSON encodes an observed `false`.
const { label, small } = toggleNodes(false);
expect(JSON.parse(JSON.stringify(small)).checked).toBe(false);
expect(JSON.parse(JSON.stringify(label))).not.toHaveProperty('checked');
});

test('an unreported checked state stays unknown instead of becoming false', () => {
const node = buildUiHierarchySnapshot(parseUiHierarchyTree(UNREPORTED_CHECKED_XML), undefined, {
raw: false,
}).nodes.find((node) => node.identifier === 'legacy-switch');
expect(node).toBeDefined();
expect(node?.checked).toBeUndefined();
});
1 change: 1 addition & 0 deletions packages/platform-android/src/ui-hierarchy-builder.ts
Original file line number Diff line number Diff line change
Expand Up @@ -356,6 +356,7 @@ function createAndroidRawSnapshotNode(
selected: node.selected,
heading: node.heading,
roleDescription: node.roleDescription,
checked: node.checked,
editable: node.editable,
password: node.password,
hintShowing: node.hintShowing,
Expand Down
1 change: 1 addition & 0 deletions packages/platform-android/src/ui-hierarchy-node.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ export type AndroidUiHierarchy = {
selected?: boolean;
heading?: boolean;
roleDescription?: string;
checked?: boolean;
editable?: boolean;
password?: boolean;
hintShowing?: boolean;
Expand Down
4 changes: 4 additions & 0 deletions packages/platform-android/src/ui-hierarchy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@ export type AndroidUiNodeMetadata = {
heading?: boolean;
/** Helper-only: the localized role description an app set beside the class, verbatim. */
roleDescription?: string;
/** Helper-only, present on a checkable control: the checked state of a switch, checkbox, or radio. */
checked?: boolean;
password?: boolean;
editable?: boolean;
selectionStart?: number;
Expand Down Expand Up @@ -175,6 +177,7 @@ function readNodeAttributes(node: string): Omit<AndroidUiNodeMetadata, 'rect'> {
...optionalBoolAttr('selected', 'selected'),
...optionalBoolAttr('heading', 'heading'),
...optionalStringAttr('roleDescription', 'role-description'),
...optionalBoolAttr('checked', 'checked'),
...optionalNumberAttr('drawingOrder', 'drawing-order'),
...optionalBoolAttr('scrollable', 'scrollable'),
...optionalBoolAttr('canScrollForward', 'can-scroll-forward'),
Expand Down Expand Up @@ -331,6 +334,7 @@ function normalizeAndroidUiHierarchyNode(
selected: attrs.selected,
heading: attrs.heading,
roleDescription: attrs.roleDescription,
checked: attrs.checked,
editable: attrs.editable,
password: attrs.password,
hintShowing: attrs.hintShowing,
Expand Down
23 changes: 23 additions & 0 deletions src/__tests__/android-ui-hierarchy.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -272,6 +272,29 @@ test('a published Android snapshot carries the heading flag and the role descrip
);
});

// A settings screen: the helper writes `checked` on the checkable switch and radio, with both
// answers, and nothing on the label beside them.
const ANDROID_TOGGLES_XML = `<hierarchy>
<node class="android.widget.LinearLayout" resource-id="com.example.app:id/toggles" bounds="[0,0][390,200]" enabled="true" visible-to-user="true">
<node class="android.widget.TextView" resource-id="com.example.app:id/wifi-label" text="Wi-Fi" bounds="[0,0][195,100]" enabled="true" visible-to-user="true"/>
<node class="android.widget.Switch" resource-id="com.example.app:id/wifi-switch" content-desc="Wi-Fi switch" bounds="[195,0][390,100]" clickable="true" enabled="true" visible-to-user="true" checked="true"/>
<node class="android.widget.RadioButton" resource-id="com.example.app:id/size-small" text="Small" bounds="[0,100][390,200]" clickable="true" enabled="true" visible-to-user="true" checked="false"/>
</node>
</hierarchy>`;

test('a published Android snapshot carries the checked state of checkable controls only', () => {
const nodes = publishUiHierarchy(ANDROID_TOGGLES_XML).nodes;
const byId = (identifier: string) => nodes.find((node) => node.identifier === identifier)!;

assert.equal(byId('com.example.app:id/wifi-switch').checked, true);
assert.equal(byId('com.example.app:id/size-small').checked, false);
assert.equal(byId('com.example.app:id/wifi-label').checked, undefined);
assert.deepEqual(
Array.from(androidUiNodes(ANDROID_TOGGLES_XML)).map((node) => node.checked),
[undefined, undefined, true, false],
);
});

test('parseUiHierarchy discards stale inactive Android application windows', () => {
const xml = `<hierarchy>
<node class="android.widget.FrameLayout" package="com.example.app" bounds="[0,0][390,844]" window-index="0" window-type="1" window-layer="10" window-active="true" window-focused="true" window-bounds="[0,0][390,844]">
Expand Down
2 changes: 2 additions & 0 deletions src/commands/capture/runtime/snapshot-unchanged.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,8 @@ test.each<Partial<SnapshotNode>>([
{ contentDescription: 'Create a draft' },
{ enabled: false },
{ selected: true },
{ checked: true },
{ checked: false },
{ focused: true },
{ placeholder: 'Key echo' },
{ hittable: false },
Expand Down
1 change: 1 addition & 0 deletions src/commands/capture/runtime/snapshot-unchanged.ts
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,7 @@ const PRESENTATION_SCALAR_FIELDS = {
identifier: true,
enabled: true,
selected: true,
checked: true,

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Added in e5bba1b: { checked: true } and { checked: false } sit in the "detects presentation changes" table, so an absent-to-false flip is a change too.

focused: true,
heading: true,
roleDescription: true,
Expand Down
25 changes: 25 additions & 0 deletions src/commands/output/snapshot.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -820,3 +820,28 @@ test('formatSnapshotLine marks selection anywhere, and keeps text-surface metada
assert.doesNotMatch(line, /\[editable\]/);
assert.doesNotMatch(line, /\[scrollable\]/);
});

test.each([
[true, /\[checked\]/],
[false, /\[unchecked\]/],
])(
'formatSnapshotLine renders both checked answers on the default path (%s)',
(checked, marker) => {
const line = formatSnapshotLine(
{
ref: 'e2',
index: 0,
depth: 0,
type: 'Switch',
label: 'Wi-Fi switch',
enabled: true,
checked,
},
0,
false,
);
// A checkable control rendered as plain would hide that it toggles, and the diff compares the
// state, so the line prints whichever answer the helper observed.
assert.match(line, marker);
},
);
Loading
Loading