diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a238a9932..7ee85fc461 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ ## Unreleased +- Fixed (android): snapshot nodes and `get attrs` carry the accessibility `heading` flag and the + `roleDescription` an app set on a node. React Native puts a header, a tab, a tab list, a link, or a + menu on a plain `android.view.View` and tells the accessibility tree what it is through these two + facts; the helper never serialized either, so every one of them was a nameless `View` to an agent. + The helper now writes `heading` when the node reports it (API 28 or later) and `role-description` + when the app set one, and the parser, the Android hierarchy node, and the published snapshot node + carry them to `get attrs` and the selector digest. The class stays the `type`. - Fixed (ios): `perf cpu profile report --kind xctrace` on Xcode 27 no longer fails with `Apple xctrace CPU report contained no samples` on a trace that holds thousands of samples. Xcode 27 exports each `time-profile` sample stack as `` instead of ``, and diff --git a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeXml.java b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeXml.java index 0753f242d8..802cd38bb1 100644 --- a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeXml.java +++ b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeXml.java @@ -53,6 +53,8 @@ static void appendNode( appendAttribute(xml, "class", node.getClassName()); appendNonEmptyAttribute(xml, "package", node.getPackageName()); appendNonEmptyAttribute(xml, "content-desc", node.getContentDescription()); + appendNonEmptyAttribute(xml, "role-description", roleDescription(node)); + appendTrueAttribute(xml, "heading", isHeading(node)); appendAttribute(xml, "visible-to-user", Boolean.toString(node.isVisibleToUser())); appendDrawingOrderAttribute(xml, node); appendTrueAttribute(xml, "clickable", node.isClickable()); @@ -146,6 +148,19 @@ private static void appendTrueAttribute(StringBuilder xml, String name, boolean } } + // The platform node has no role description getter: androidx writes the value an app set + // (AccessibilityNodeInfoCompat.setRoleDescription) into the node extras under this key, and + // TalkBack reads it from there. + private static CharSequence roleDescription(AccessibilityNodeInfo node) { + return node.getExtras().getCharSequence("AccessibilityNodeInfo.roleDescription"); + } + + // isHeading() arrived in API 28. Older releases keep the compat flag in an extras bit this + // helper does not read, so a heading on API 23-27 reports nothing. + private static boolean isHeading(AccessibilityNodeInfo node) { + return Build.VERSION.SDK_INT >= Build.VERSION_CODES.P && node.isHeading(); + } + // Declared residue (agent-device #1832): checked / checkable / long-clickable are not serialized, // so toggle state is invisible to agents. Adding them is a helper protocol change (new attributes // + host parser + fields on the wire node), tracked there. diff --git a/packages/kernel/src/snapshot.ts b/packages/kernel/src/snapshot.ts index b0f38a3233..eace66eba8 100644 --- a/packages/kernel/src/snapshot.ts +++ b/packages/kernel/src/snapshot.ts @@ -238,6 +238,10 @@ export type RawSnapshotNode = { enabled?: boolean; selected?: boolean; focused?: boolean; + /** Accessibility heading flag an app set on the node; absent means not a heading or unavailable. */ + heading?: boolean; + /** Localized role description an app set beside the native class, verbatim (`Tab`, `Tab List`, `Link`). */ + roleDescription?: string; /** Native accessibility facts; absent means unavailable, not false. */ editable?: boolean; password?: boolean; diff --git a/packages/platform-android/src/__tests__/ui-hierarchy-role-facts.test.ts b/packages/platform-android/src/__tests__/ui-hierarchy-role-facts.test.ts new file mode 100644 index 0000000000..494920afa9 --- /dev/null +++ b/packages/platform-android/src/__tests__/ui-hierarchy-role-facts.test.ts @@ -0,0 +1,60 @@ +import { expect, test } from 'vitest'; +import { buildUiHierarchySnapshot, parseUiHierarchyTree } from '../ui-hierarchy.ts'; + +// A React Native screen: `accessibilityRole="header"` is a plain View the helper flags as a heading, +// a tab bar is a View with the role description the app set, and a label carries neither. +const ROLE_FACTS_XML = ` + + + + + + + +`; + +function nodesById(raw: boolean, interactiveOnly = false) { + const { nodes } = buildUiHierarchySnapshot(parseUiHierarchyTree(ROLE_FACTS_XML), undefined, { + raw, + interactiveOnly, + }); + return (identifier: string) => nodes.find((node) => node.identifier === identifier); +} + +test.each([ + { raw: false, interactiveOnly: false }, + { raw: true, interactiveOnly: false }, + { raw: true, interactiveOnly: true }, +])( + 'the heading flag and the role description reach snapshot nodes (raw=$raw, -i=$interactiveOnly)', + ({ raw, interactiveOnly }) => { + const byId = nodesById(raw, interactiveOnly); + expect(byId('inventory-header')?.heading).toBe(true); + expect(byId('section-tabs')?.roleDescription).toBe('tab list'); + expect(byId('tab-fields')?.roleDescription).toBe('tab'); + }, +); + +test('a node without either fact carries neither key once serialized', () => { + const byId = nodesById(false); + const label = JSON.parse(JSON.stringify(byId('plain-label'))); + expect(label).not.toHaveProperty('heading'); + expect(label).not.toHaveProperty('roleDescription'); + // A heading is not a role description and a role description is not a heading. + expect(JSON.parse(JSON.stringify(byId('inventory-header')))).not.toHaveProperty( + 'roleDescription', + ); + expect(JSON.parse(JSON.stringify(byId('tab-fields')))).not.toHaveProperty('heading'); +}); + +test('the class stays the type: a role description refines nothing on its own', () => { + const byId = nodesById(false); + expect(byId('tab-fields')?.type).toBe('android.view.View'); + expect(byId('inventory-header')?.label).toBe('Inventory'); +}); diff --git a/packages/platform-android/src/ui-hierarchy-builder.ts b/packages/platform-android/src/ui-hierarchy-builder.ts index ea608a2c5e..a3aed983f0 100644 --- a/packages/platform-android/src/ui-hierarchy-builder.ts +++ b/packages/platform-android/src/ui-hierarchy-builder.ts @@ -354,6 +354,8 @@ function createAndroidRawSnapshotNode( enabled: node.enabled, focused: node.focused, selected: node.selected, + heading: node.heading, + roleDescription: node.roleDescription, editable: node.editable, password: node.password, hintShowing: node.hintShowing, diff --git a/packages/platform-android/src/ui-hierarchy-node.ts b/packages/platform-android/src/ui-hierarchy-node.ts index c6e3138c32..4ac4a47a1f 100644 --- a/packages/platform-android/src/ui-hierarchy-node.ts +++ b/packages/platform-android/src/ui-hierarchy-node.ts @@ -17,6 +17,8 @@ export type AndroidUiHierarchy = { visibleToUser?: boolean; focused?: boolean; selected?: boolean; + heading?: boolean; + roleDescription?: string; editable?: boolean; password?: boolean; hintShowing?: boolean; diff --git a/packages/platform-android/src/ui-hierarchy.ts b/packages/platform-android/src/ui-hierarchy.ts index cf5933cec7..006502d9b3 100644 --- a/packages/platform-android/src/ui-hierarchy.ts +++ b/packages/platform-android/src/ui-hierarchy.ts @@ -35,6 +35,10 @@ export type AndroidUiNodeMetadata = { focusable?: boolean; focused?: boolean; selected?: boolean; + /** Helper-only: the accessibility heading flag an app set on the node (API 28 or later). */ + heading?: boolean; + /** Helper-only: the localized role description an app set beside the class, verbatim. */ + roleDescription?: string; password?: boolean; editable?: boolean; selectionStart?: number; @@ -132,6 +136,15 @@ function readNodeAttributes(node: string): Omit { const value = parseBounds(getAttr(name)); return value === undefined ? {} : ({ [key]: value } as Pick); }; + const optionalStringAttr = ( + key: Key, + name: string, + ): Partial> => { + const value = getAttr(name); + return value === null || value === '' + ? {} + : ({ [key]: value } as Pick); + }; const optionalBoolAttr = ( key: Key, name: string, @@ -157,6 +170,8 @@ function readNodeAttributes(node: string): Omit { ...optionalBoolAttr('hintShowing', 'hint-showing'), ...optionalBoolAttr('visibleToUser', 'visible-to-user'), ...optionalBoolAttr('selected', 'selected'), + ...optionalBoolAttr('heading', 'heading'), + ...optionalStringAttr('roleDescription', 'role-description'), ...optionalNumberAttr('drawingOrder', 'drawing-order'), ...optionalBoolAttr('scrollable', 'scrollable'), ...optionalBoolAttr('canScrollForward', 'can-scroll-forward'), @@ -311,6 +326,8 @@ function normalizeAndroidUiHierarchyNode( enabled: attrs.enabled, focused: attrs.focused, selected: attrs.selected, + heading: attrs.heading, + roleDescription: attrs.roleDescription, editable: attrs.editable, password: attrs.password, hintShowing: attrs.hintShowing, diff --git a/src/__tests__/android-ui-hierarchy.test.ts b/src/__tests__/android-ui-hierarchy.test.ts index 62a41f90c9..d9edc4296d 100644 --- a/src/__tests__/android-ui-hierarchy.test.ts +++ b/src/__tests__/android-ui-hierarchy.test.ts @@ -236,6 +236,42 @@ test('a published Android snapshot answers a selected-qualified read (#2462)', a ); }); +// A React Native screen: the header role is a View flagged as a heading, the tab bar and its tabs +// are Views with the role description the app set, and the helper writes neither on a plain label. +const ANDROID_ROLE_FACTS_XML = ` + + + + + + + +`; + +test('a published Android snapshot carries the heading flag and the role description', () => { + const nodes = publishUiHierarchy(ANDROID_ROLE_FACTS_XML).nodes; + const byId = (identifier: string) => nodes.find((node) => node.identifier === identifier)!; + + assert.equal(byId('com.example.app:id/header').heading, true); + assert.equal(byId('com.example.app:id/tabs').roleDescription, 'tab list'); + assert.equal(byId('com.example.app:id/tab-fields').roleDescription, 'tab'); + assert.equal(byId('com.example.app:id/label').heading, undefined); + assert.equal(byId('com.example.app:id/label').roleDescription, undefined); + assert.deepEqual( + Array.from(androidUiNodes(ANDROID_ROLE_FACTS_XML)).map((node) => [ + node.heading, + node.roleDescription, + ]), + [ + [undefined, undefined], + [true, undefined], + [undefined, 'tab list'], + [undefined, 'tab'], + [undefined, undefined], + ], + ); +}); + test('parseUiHierarchy discards stale inactive Android application windows', () => { const xml = ` diff --git a/src/commands/capture/runtime/snapshot-unchanged.ts b/src/commands/capture/runtime/snapshot-unchanged.ts index 61929117be..1ec07b70ee 100644 --- a/src/commands/capture/runtime/snapshot-unchanged.ts +++ b/src/commands/capture/runtime/snapshot-unchanged.ts @@ -112,6 +112,8 @@ const PRESENTATION_SCALAR_FIELDS = { enabled: true, selected: true, focused: true, + heading: true, + roleDescription: true, hittable: true, bundleId: true, appName: true, diff --git a/src/daemon/response-views.ts b/src/daemon/response-views.ts index 62ad99e599..2a21a34a12 100644 --- a/src/daemon/response-views.ts +++ b/src/daemon/response-views.ts @@ -115,6 +115,8 @@ const SELECTOR_DIGEST_NODE_FIELDS = [ 'enabled', 'selected', 'focused', + 'heading', + 'roleDescription', 'editable', 'password', 'hintShowing', diff --git a/website/docs/docs/snapshots.md b/website/docs/docs/snapshots.md index 9a4a1a2492..f9ef3f34e0 100644 --- a/website/docs/docs/snapshots.md +++ b/website/docs/docs/snapshots.md @@ -117,15 +117,21 @@ the strategy owns which tiers it may use. ## Android node metadata Android snapshot nodes and `get attrs` (including the digest response) carry the native -`selected`, `editable`, `password`, `hintShowing`, `selectionStart`, and `selectionEnd` facts -whenever the accessibility tree reports them. Explicit `false` and `0` are kept; an absent field -means the fact was unavailable, not false. `hintShowing` needs Android API 26 or later. +`selected`, `heading`, `roleDescription`, `editable`, `password`, `hintShowing`, `selectionStart`, +and `selectionEnd` facts whenever the accessibility tree reports them. Explicit `false` and `0` are +kept; an absent field means the fact was unavailable, not false. `hintShowing` needs Android API 26 +or later, `heading` API 28 or later. - `selected` is the accessibility selected state an app sets on a control — the active bottom-tab or segmented-control item, or the chosen row of a list. Android reports it explicitly as `true` or `false`; an older helper APK omits the field, which means the answer is unavailable rather than unselected. Snapshot text marks the node `[selected]`, and `is selected`, a `selected=true` selector, and a Maestro `selected:` qualifier all match on it. +- `heading` is the accessibility heading flag an app sets on a node, the way React Native's + `accessibilityRole="header"` does on a plain `View`; it is present only as `true`. +- `roleDescription` is the localized role description an app sets beside the native class, verbatim + (React Native writes `Tab`, `Tab List`, `Radio Group`, `Link`, `Menu`), when the class alone would + not say what the control is. The `type` stays the class; a consumer maps the description to a role. - `value: ""` is an explicitly empty accessibility text; a missing `value` means no text was reported. The text of an empty field is its hint on modern Android, so check `hintShowing` before reading `value` as the entered contents.