From 80aa8df61ee1437d1c5d5c990241455a6d97c403 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oskar=20Kwas=CC=81niewski?= Date: Fri, 25 Sep 2026 11:20:20 +0200 Subject: [PATCH 1/3] feat(ios): the snapshot node carries the field's placeholder Android nodes carry the field's hint as placeholder since #2927; iOS nodes carried nothing, so a consumer could neither query a field by its placeholder nor tell an empty field (XCTest reports the placeholder as its value) from one holding that text. XCTest exposes placeholderValue on every element and snapshot, and the AX server exposes XC_kAXXCAttributePlaceholderValue. Every iOS producer now publishes placeholder on RawAXNode and PresentedNode: the XCTest tree and query sweeps read placeholderValue off the snapshot or element, the runner's private-AX reader asks for the keypath beside value, and the Simulator AX bridge requests the attribute (source v1.8.0, one recompile per host). An empty placeholder reads as none. The docs name the fact for both platforms. --- .../RunnerAXSnapshotBridge.m | 6 +++-- .../RunnerTests+PrivateAXPresentation.swift | 3 +++ .../RunnerTests+Snapshot.swift | 2 ++ .../RunnerTests+SnapshotAcquisition.swift | 13 +++++++++++ ...nnerTests+PrivateAXPresentationTests.swift | 23 +++++++++++++++++++ ...unnerTests+SnapshotPresentationTests.swift | 3 ++- apple/snapshot-bridge/SnapshotBridgeRuntime.m | 4 +++- .../SnapshotModels.swift | 6 +++++ .../SnapshotPresentationProjection.swift | 1 + .../SnapshotVisibilityFold.swift | 1 + .../SnapshotVisibilityFoldProjection.swift | 1 + .../fixtures/wire-vocabulary.json | 3 ++- .../src/snapshot-source/protocol.test.ts | 2 +- .../src/snapshot-source/protocol.ts | 3 ++- .../src/snapshot-source/tree.test.ts | 20 ++++++++++++++++ .../src/snapshot-source/tree.ts | 4 ++++ website/docs/docs/snapshots.md | 5 +++- 17 files changed, 92 insertions(+), 8 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.m index 243c280523..81d1692e13 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.m @@ -365,6 +365,7 @@ + (NSArray *)snapshotAttributes @"identifier", @"label", @"value", + @"placeholderValue", @"frame", @"enabled", @"selected", @@ -388,8 +389,8 @@ + (NSArray *)snapshotAttributes // The mapper expands keypaths with extra attributes (automation type, window display // id, base type) that are disproportionately expensive for the AX server to compute // on large React Native trees. Keep only the attributes we actually consume. - NSArray *needed = @[ @"ElementType", @"Identifier", @"Label", @"Value", @"Frame", - @"Enabled", @"Selected", @"Focus" ]; + NSArray *needed = @[ @"ElementType", @"Identifier", @"Label", @"Value", @"PlaceholderValue", + @"Frame", @"Enabled", @"Selected", @"Focus" ]; NSMutableArray *filtered = [NSMutableArray array]; for (id attribute in (NSArray *)mapped) { NSString *name = [attribute description]; @@ -764,6 +765,7 @@ + (nullable NSMutableDictionary *)dictionaryForSnapshot:(id)snapshot result[@"identifier"] = [self stringValueForKey:@"identifier" snapshot:snapshot] ?: @""; result[@"label"] = [self stringValueForKey:@"label" snapshot:snapshot] ?: @""; result[@"value"] = [self stringValueForKey:@"value" snapshot:snapshot] ?: @""; + result[@"placeholder"] = [self stringValueForKey:@"placeholderValue" snapshot:snapshot] ?: @""; result[@"frame"] = [self frameValueForSnapshot:snapshot]; result[@"enabled"] = [self boolNumberForKey:@"enabled" snapshot:snapshot defaultValue:YES]; result[@"selected"] = [self boolNumberForKey:@"selected" snapshot:snapshot defaultValue:NO]; diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+PrivateAXPresentation.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+PrivateAXPresentation.swift index b95b1dc5a1..f6365d592c 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+PrivateAXPresentation.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+PrivateAXPresentation.swift @@ -8,6 +8,7 @@ struct PrivateAXFields { let label: String let identifier: String let value: String + let placeholder: String let rawType: Int let elementType: XCUIElement.ElementType? let enabled: Bool @@ -52,6 +53,7 @@ extension RunnerTests { label: privateAXPresentationString(raw["label"]), identifier: privateAXPresentationString(raw["identifier"]), value: privateAXPresentationString(raw["value"]), + placeholder: privateAXPresentationString(raw["placeholder"]), rawType: rawType, elementType: privateAXElementType(rawElementType: rawType), enabled: privateAXPresentationBool(raw["enabled"]) ?? true, @@ -70,6 +72,7 @@ extension RunnerTests { label: fields.label.isEmpty ? nil : fields.label, identifier: fields.identifier.isEmpty ? nil : fields.identifier, value: fields.value.isEmpty ? nil : fields.value, + placeholder: fields.placeholder.isEmpty ? nil : fields.placeholder, rect: SnapshotRect(fields.rect), enabled: fields.enabled, focused: fields.focused, selected: fields.selected, hittable: false, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift index b21d55b1da..965ba3623d 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift @@ -35,6 +35,7 @@ extension RunnerTests { let label: String let identifier: String let valueText: String? + let placeholder: String? let focused: Bool let selected: Bool } @@ -500,6 +501,7 @@ extension RunnerTests { label: candidate.label, identifier: candidate.identifier, value: candidate.value, + placeholder: candidate.placeholder, rect: candidate.rect, enabled: candidate.enabled, focused: candidate.focused, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift index 434b9f9f0e..e481f9f223 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift @@ -189,6 +189,7 @@ extension RunnerTests { label: label, identifier: identifier, valueText: valueText, + placeholder: placeholderText(snapshot.placeholderValue), focused: snapshotHasFocus(snapshot), selected: snapshotIsSelected(snapshot) ) @@ -209,6 +210,7 @@ extension RunnerTests { label: evaluation.label.isEmpty ? nil : evaluation.label, identifier: evaluation.identifier.isEmpty ? nil : evaluation.identifier, value: evaluation.valueText, + placeholder: evaluation.placeholder, rect: SnapshotRect(snapshot.frame), enabled: snapshot.isEnabled, focused: evaluation.focused ? true : nil, @@ -235,6 +237,14 @@ extension RunnerTests { return text.isEmpty ? nil : text } + /// The placeholder as the node publishes it: XCTest answers `placeholderValue` for a text field + /// whether or not the field is empty, and an empty string for everything else, which reads as + /// no placeholder. The private-AX bridge asks the AX server the same attribute. + func placeholderText(_ placeholderValue: String?) -> String? { + let text = placeholderValue?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + return text.isEmpty ? nil : text + } + private func snapshotAppFrame(app: XCUIApplication) -> CGRect { #if os(iOS) return onScreenWindowFrame(app: app) @@ -341,6 +351,7 @@ extension RunnerTests { label: node.label, identifier: node.identifier, value: node.value, + placeholder: node.placeholder, rect: node.rect, enabled: node.enabled, focused: node.focused, @@ -395,6 +406,7 @@ extension RunnerTests { label: label.isEmpty ? nil : label, identifier: identifier.isEmpty ? nil : identifier, value: valueText, + placeholder: placeholderText(element.placeholderValue), rect: SnapshotRect(frame), enabled: element.isEnabled, focused: elementHasFocus(element) ? true : nil, @@ -558,6 +570,7 @@ extension RunnerTests { label: label.isEmpty ? nil : label, identifier: identifier.isEmpty ? nil : identifier, value: valueText, + placeholder: placeholderText(element.placeholderValue), rect: SnapshotRect(frame), enabled: enabled, focused: elementHasFocus(element) ? true : nil, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+PrivateAXPresentationTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+PrivateAXPresentationTests.swift index 3dcd2908dd..6bb92417d9 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+PrivateAXPresentationTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+PrivateAXPresentationTests.swift @@ -117,6 +117,29 @@ extension RunnerTests { ) } + /// The private-AX reader hands over `placeholderValue` beside `value`; the acquisition publishes + /// it on the node and reads an empty one as no placeholder, the way every other producer does. + func testPrivateAXAcquisitionCarriesTheFieldPlaceholder() { + let frame = Self.privateAXFrame + let nodes = privateAXNormalizedAcquisition( + rawRoot: [ + "type": Int(XCUIElement.ElementType.application.rawValue), "frame": frame(0, 0, 402, 874), + "children": [ + ["type": Int(XCUIElement.ElementType.textField.rawValue), "value": "Type your name", + "placeholder": "Type your name", "frame": frame(16, 200, 370, 44), "children": []], + ["type": Int(XCUIElement.ElementType.button.rawValue), "label": "Save", + "placeholder": "", "frame": frame(16, 300, 370, 44), "children": []] + ] + ], + hint: CaptureHint( + projection: .raw, depth: nil, regularPresentedDepth: nil, + interactiveOnly: false, customActions: false), + viewport: CGRect(x: 0, y: 0, width: 402, height: 874), + interfaceOrientation: RunnerInterfaceOrientation.portrait) + + XCTAssertEqual(nodes.map(\.placeholder), [nil, "Type your name", nil]) + } + func testPrivateAXRegularPresentationProjectsToViewportAndKeepsScrollHint() throws { let nodes = try privateAXRegularPresentation( rawRoot: Self.privateAXScrolledFixture, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationTests.swift index 462cdfd667..7617c350e5 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationTests.swift @@ -12,6 +12,7 @@ extension RunnerTests { label: "Continue", identifier: "continue-button", value: "Ready", + placeholder: "Type here", rect: SnapshotRect(x: 10, y: 20, width: 100, height: 44), enabled: true, focused: true, @@ -54,7 +55,7 @@ extension RunnerTests { XCTAssertEqual( String(decoding: encoded, as: UTF8.self), - #"[{"actions":["Open menu"],"depth":2,"enabled":true,"focused":true,"hiddenContentAbove":true,"hiddenContentBelow":true,"hittable":true,"identifier":"continue-button","index":3,"label":"Continue","parentIndex":1,"rect":{"height":44,"width":100,"x":10,"y":20},"selected":true,"type":"Button","value":"Ready"}]"# + #"[{"actions":["Open menu"],"depth":2,"enabled":true,"focused":true,"hiddenContentAbove":true,"hiddenContentBelow":true,"hittable":true,"identifier":"continue-button","index":3,"label":"Continue","parentIndex":1,"placeholder":"Type here","rect":{"height":44,"width":100,"x":10,"y":20},"selected":true,"type":"Button","value":"Ready"}]"# ) XCTAssertEqual(capture.truncated, true) XCTAssertEqual(capture.effectiveDepth, 4) diff --git a/apple/snapshot-bridge/SnapshotBridgeRuntime.m b/apple/snapshot-bridge/SnapshotBridgeRuntime.m index a4e8da8eb3..e50e95ebb4 100644 --- a/apple/snapshot-bridge/SnapshotBridgeRuntime.m +++ b/apple/snapshot-bridge/SnapshotBridgeRuntime.m @@ -18,7 +18,7 @@ NSString *const kProtocolVersionKey = @"protocolVersion"; NSString *const kSourceVersionKey = @"sourceVersion"; NSString *const kRequestIdKey = @"requestId"; -NSString *const kSourceVersion = @"agent-device-simulator-ax-v1.7.0"; +NSString *const kSourceVersion = @"agent-device-simulator-ax-v1.8.0"; const NSUInteger kProtocolVersion = 1; const uint32_t kMaximumFrameBytes = 16 * 1024 * 1024; const NSUInteger kMaximumDepth = 128; @@ -29,6 +29,7 @@ static NSString *const kAttributeElementBaseType = @"XC_kAXXCAttributeElementBaseType"; static NSString *const kAttributeLabel = @"XC_kAXXCAttributeLabel"; static NSString *const kAttributeValue = @"XC_kAXXCAttributeValue"; +static NSString *const kAttributePlaceholderValue = @"XC_kAXXCAttributePlaceholderValue"; static NSString *const kAttributeIdentifier = @"XC_kAXXCAttributeIdentifier"; static NSString *const kAttributeFrame = @"XC_kAXXCAttributeFrame"; static NSString *const kAttributeAutomationType = @"XC_kAXXCAttributeAutomationType"; @@ -330,6 +331,7 @@ - (nullable NSDictionary *)snapshotForProcess:(pid_t)pid kAttributeElementBaseType, kAttributeLabel, kAttributeValue, + kAttributePlaceholderValue, kAttributeIdentifier, kAttributeFrame, kAttributeAutomationType, diff --git a/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotModels.swift b/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotModels.swift index 7723411050..65339c60e7 100644 --- a/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotModels.swift +++ b/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotModels.swift @@ -34,6 +34,8 @@ public struct RawAXNode: Equatable { public let label: String? public let identifier: String? public let value: String? + /// The text a text field shows while empty (`placeholderValue`); `nil` for every other node. + public let placeholder: String? public var rect: SnapshotRect public let enabled: Bool public let focused: Bool? @@ -52,6 +54,7 @@ public struct RawAXNode: Equatable { label: String?, identifier: String?, value: String?, + placeholder: String? = nil, rect: SnapshotRect, enabled: Bool, focused: Bool?, @@ -68,6 +71,7 @@ public struct RawAXNode: Equatable { self.label = label self.identifier = identifier self.value = value + self.placeholder = placeholder self.rect = rect self.enabled = enabled self.focused = focused @@ -272,6 +276,7 @@ public struct PresentedNode: Codable, Equatable { public let label: String? public let identifier: String? public let value: String? + public let placeholder: String? public let rect: SnapshotRect public let enabled: Bool public let focused: Bool? @@ -295,6 +300,7 @@ public struct PresentedNode: Codable, Equatable { self.label = raw.label self.identifier = raw.identifier self.value = raw.value + self.placeholder = raw.placeholder self.rect = rect ?? raw.rect self.enabled = raw.enabled self.focused = raw.focused diff --git a/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotPresentationProjection.swift b/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotPresentationProjection.swift index 644e0810bc..80a0ddfca2 100644 --- a/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotPresentationProjection.swift +++ b/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotPresentationProjection.swift @@ -140,6 +140,7 @@ extension SnapshotPresentation { label: raw.label, identifier: raw.identifier, value: raw.value, + placeholder: raw.placeholder, rect: raw.rect, enabled: raw.enabled, focused: raw.focused, diff --git a/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotVisibilityFold.swift b/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotVisibilityFold.swift index 0a3503f770..91f3dac9d3 100644 --- a/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotVisibilityFold.swift +++ b/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotVisibilityFold.swift @@ -199,6 +199,7 @@ public enum SnapshotVisibilityFold { label: node.label, identifier: node.identifier, value: node.value, + placeholder: node.placeholder, rect: node.rect, enabled: node.enabled, focused: node.focused, diff --git a/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotVisibilityFoldProjection.swift b/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotVisibilityFoldProjection.swift index af2dd9e6af..7db50748d4 100644 --- a/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotVisibilityFoldProjection.swift +++ b/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotVisibilityFoldProjection.swift @@ -79,6 +79,7 @@ extension SnapshotVisibilityFold { label: node.label, identifier: node.identifier, value: node.value, + placeholder: node.placeholder, rect: node.rect, enabled: node.enabled, focused: node.focused, diff --git a/packages/platform-apple/src/snapshot-source/fixtures/wire-vocabulary.json b/packages/platform-apple/src/snapshot-source/fixtures/wire-vocabulary.json index 6eacf669d6..9896f768c1 100644 --- a/packages/platform-apple/src/snapshot-source/fixtures/wire-vocabulary.json +++ b/packages/platform-apple/src/snapshot-source/fixtures/wire-vocabulary.json @@ -1,6 +1,6 @@ { "protocolVersion": 1, - "sourceVersion": "agent-device-simulator-ax-v1.7.0", + "sourceVersion": "agent-device-simulator-ax-v1.8.0", "requestKeys": [ "verb", "requestId", @@ -34,6 +34,7 @@ "XC_kAXXCAttributeElementBaseType", "XC_kAXXCAttributeLabel", "XC_kAXXCAttributeValue", + "XC_kAXXCAttributePlaceholderValue", "XC_kAXXCAttributeIdentifier", "XC_kAXXCAttributeFrame", "XC_kAXXCAttributeAutomationType", diff --git a/packages/platform-apple/src/snapshot-source/protocol.test.ts b/packages/platform-apple/src/snapshot-source/protocol.test.ts index 4fda71db5f..4069d3e558 100644 --- a/packages/platform-apple/src/snapshot-source/protocol.test.ts +++ b/packages/platform-apple/src/snapshot-source/protocol.test.ts @@ -142,7 +142,7 @@ test('wire vocabulary guard keeps TS and Objective-C literals aligned', async () assert.deepEqual(wireVocabulary.responseKeys, SNAPSHOT_SOURCE_RESPONSE_KEYS); assert.deepEqual(wireVocabulary.attributeKeys, SNAPSHOT_SOURCE_ATTRIBUTE_KEYS); assert.match(nativeSource, /kProtocolVersion = 1/); - assert.match(nativeSource, /kSourceVersion = @"agent-device-simulator-ax-v1\.7\.0"/); + assert.match(nativeSource, /kSourceVersion = @"agent-device-simulator-ax-v1\.8\.0"/); for (const key of [ ...wireVocabulary.requestKeys, ...wireVocabulary.responseKeys, diff --git a/packages/platform-apple/src/snapshot-source/protocol.ts b/packages/platform-apple/src/snapshot-source/protocol.ts index ce663f86ec..6bd22177b1 100644 --- a/packages/platform-apple/src/snapshot-source/protocol.ts +++ b/packages/platform-apple/src/snapshot-source/protocol.ts @@ -3,7 +3,7 @@ import { snapshotSourceError } from './errors.ts'; import type { SnapshotSourceLimits } from './types.ts'; export const SNAPSHOT_SOURCE_PROTOCOL_VERSION = 1; -export const SNAPSHOT_SOURCE_VERSION = 'agent-device-simulator-ax-v1.7.0'; +export const SNAPSHOT_SOURCE_VERSION = 'agent-device-simulator-ax-v1.8.0'; const FRAME_HEADER_BYTES = 4; export const SNAPSHOT_SOURCE_WIRE_KEYS = Object.freeze([ @@ -41,6 +41,7 @@ export const SNAPSHOT_SOURCE_ATTRIBUTE_KEYS = Object.freeze([ 'XC_kAXXCAttributeElementBaseType', 'XC_kAXXCAttributeLabel', 'XC_kAXXCAttributeValue', + 'XC_kAXXCAttributePlaceholderValue', 'XC_kAXXCAttributeIdentifier', 'XC_kAXXCAttributeFrame', 'XC_kAXXCAttributeAutomationType', diff --git a/packages/platform-apple/src/snapshot-source/tree.test.ts b/packages/platform-apple/src/snapshot-source/tree.test.ts index 6d7e01773b..37e7eb0ba2 100644 --- a/packages/platform-apple/src/snapshot-source/tree.test.ts +++ b/packages/platform-apple/src/snapshot-source/tree.test.ts @@ -464,6 +464,26 @@ test('a window reporting the app box quarter-turned is counted as an unresolved ); }); +test('the bridge tree carries a text field placeholder and omits an empty one', () => { + const field = (placeholder?: unknown) => ({ + [application]: 'UITextField', + [frame]: { X: 16, Y: 200, Width: 370, Height: 44 }, + XC_kAXXCAttributeValue: 'Type your name', + ...(placeholder === undefined ? {} : { XC_kAXXCAttributePlaceholderValue: placeholder }), + [children]: [], + }); + const decode = (placeholder?: unknown) => + decodeSnapshotBridgeTree( + { [application]: 'Application', [children]: [field(placeholder)] }, + { truncated: false }, + limits, + ).nodes[1]; + + assert.equal(decode('Type your name')?.placeholder, 'Type your name'); + assert.equal(decode('')?.placeholder, undefined, 'no placeholder reads as none, not as ""'); + assert.equal(decode()?.placeholder, undefined, 'an unread fact stays unknown'); +}); + test('the bridge tree publishes whether a dimming view takes touches', () => { const dimming = (enabled?: unknown) => ({ [application]: 'UIDimmingView', diff --git a/packages/platform-apple/src/snapshot-source/tree.ts b/packages/platform-apple/src/snapshot-source/tree.ts index 8fe1f6518f..fce6c89d8b 100644 --- a/packages/platform-apple/src/snapshot-source/tree.ts +++ b/packages/platform-apple/src/snapshot-source/tree.ts @@ -16,6 +16,7 @@ const ATTRIBUTE = Object.freeze({ elementBaseType: 'XC_kAXXCAttributeElementBaseType', label: 'XC_kAXXCAttributeLabel', value: 'XC_kAXXCAttributeValue', + placeholder: 'XC_kAXXCAttributePlaceholderValue', identifier: 'XC_kAXXCAttributeIdentifier', frame: 'XC_kAXXCAttributeFrame', automationType: 'XC_kAXXCAttributeAutomationType', @@ -261,6 +262,9 @@ function nodeFacts( ...(optionalScalar(value[ATTRIBUTE.value]) ? { value: optionalScalar(value[ATTRIBUTE.value]) } : {}), + ...(optionalString(value[ATTRIBUTE.placeholder]) + ? { placeholder: optionalString(value[ATTRIBUTE.placeholder]) } + : {}), ...(optionalString(value[ATTRIBUTE.identifier]) ? { identifier: optionalString(value[ATTRIBUTE.identifier]) } : {}), diff --git a/website/docs/docs/snapshots.md b/website/docs/docs/snapshots.md index 31662eaaac..dfd454fb30 100644 --- a/website/docs/docs/snapshots.md +++ b/website/docs/docs/snapshots.md @@ -149,7 +149,10 @@ them. Explicit `false` and `0` are kept; an absent field means the fact was unav before reading `value` as the entered contents. - `placeholder` is the field's hint text itself, present whether the field is empty or filled: an empty field shows it (`hintShowing: true`, and `value` repeats it), a filled field no longer does. - A field without a hint omits it. + A field without a hint omits it. iOS nodes carry the same fact from the field's + `placeholderValue`, on every producer (the XCTest tree, the Simulator AX bridge, and the runner's + private-AX reader); XCTest reports an empty field's placeholder as its `value` too, so a `value` + equal to `placeholder` is an empty field. - `selectionStart`/`selectionEnd` are accessibility selection offsets. They are independent of `editable` (read-only selectable text exposes them too), they are not a character count, and they do not prove that a masked or secure value equals expected text. From fd3ae054ae53f19db8253de434cf0ee55dc12fde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oskar=20Kwas=CC=81niewski?= Date: Fri, 25 Sep 2026 11:45:47 +0200 Subject: [PATCH 2/3] fix(ios): count a placeholder as content, keep value and placeholder apart in the tests Review follow-ups: the collapsed-tab producer read the placeholder only after its content check, so a placeholder-only field inside a collapsed container was dropped; the placeholder now counts as content. The bridge test used one string for value and placeholder, so it could not tell the two attributes apart; it uses distinct strings and asserts both. A wire-shape test pins that a node without a placeholder encodes no key. The docs stop calling a value equal to the placeholder an empty field, since a user can type that text. --- .../RunnerTests+SnapshotAcquisition.swift | 5 ++- ...unnerTests+SnapshotPresentationTests.swift | 38 +++++++++++++++++++ .../src/snapshot-source/tree.test.ts | 6 ++- website/docs/docs/snapshots.md | 5 ++- 4 files changed, 48 insertions(+), 6 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift index e481f9f223..ef1367720f 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift @@ -387,7 +387,8 @@ extension RunnerTests { let label = element.label.trimmingCharacters(in: .whitespacesAndNewlines) let identifier = element.identifier.trimmingCharacters(in: .whitespacesAndNewlines) let valueText = snapshotValueText(element) - let hasContent = !label.isEmpty || !identifier.isEmpty || valueText != nil + let placeholder = placeholderText(element.placeholderValue) + let hasContent = !label.isEmpty || !identifier.isEmpty || valueText != nil || placeholder != nil if !hasContent { return } if sameSemanticElement( containerSnapshot: containerSnapshot, @@ -406,7 +407,7 @@ extension RunnerTests { label: label.isEmpty ? nil : label, identifier: identifier.isEmpty ? nil : identifier, value: valueText, - placeholder: placeholderText(element.placeholderValue), + placeholder: placeholder, rect: SnapshotRect(frame), enabled: element.isEnabled, focused: elementHasFocus(element) ? true : nil, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationTests.swift index 7617c350e5..8138eb41fb 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationTests.swift @@ -63,6 +63,44 @@ extension RunnerTests { XCTAssertEqual(capture.customActions?.candidates, 2) } + /// A node without a placeholder omits the key: the wire contract is "absent when empty", which + /// the synthesized `encodeIfPresent` provides today and a hand-written encoder must keep. + func testPresentedNodeOmitsAnAbsentPlaceholder() throws { + let raw = RawAXNode( + index: 0, + type: "Button", + label: "Save", + identifier: nil, + value: nil, + rect: SnapshotRect(x: 0, y: 0, width: 100, height: 44), + enabled: true, + focused: nil, + selected: nil, + hittable: true, + depth: 0, + parentIndex: nil, + hiddenContentAbove: nil, + hiddenContentBelow: nil + ) + let capture = try XCTUnwrap(try SnapshotPresentation.present( + SnapshotAcquisition( + hint: CaptureHint( + projection: .raw, depth: nil, regularPresentedDepth: nil, + interactiveOnly: false, customActions: false), + nodes: [raw], + truncated: false, + effectiveDepth: nil, + customActions: nil, + viewport: .reported(box: CGRect(x: 0, y: 0, width: 100, height: 100)) + ), + options: PresentationOptions(interactiveOnly: false, depth: nil, scope: nil, raw: true) + )) + let encoded = String(decoding: try JSONEncoder().encode(capture.nodes), as: UTF8.self) + + XCTAssertNil(capture.nodes.first?.placeholder) + XCTAssertFalse(encoded.contains("placeholder"), encoded) + } + func testSnapshotPresentationOwnsBackendNeutralEligibility() throws { // Non-vacuity: forcing `isEligibleForRegularPresentation` to return true made this test execute // once and fail exactly four membership/index/depth/parent assertions before restoration. diff --git a/packages/platform-apple/src/snapshot-source/tree.test.ts b/packages/platform-apple/src/snapshot-source/tree.test.ts index 37e7eb0ba2..f98e892df9 100644 --- a/packages/platform-apple/src/snapshot-source/tree.test.ts +++ b/packages/platform-apple/src/snapshot-source/tree.test.ts @@ -468,7 +468,7 @@ test('the bridge tree carries a text field placeholder and omits an empty one', const field = (placeholder?: unknown) => ({ [application]: 'UITextField', [frame]: { X: 16, Y: 200, Width: 370, Height: 44 }, - XC_kAXXCAttributeValue: 'Type your name', + XC_kAXXCAttributeValue: 'Ada Lovelace', ...(placeholder === undefined ? {} : { XC_kAXXCAttributePlaceholderValue: placeholder }), [children]: [], }); @@ -479,7 +479,9 @@ test('the bridge tree carries a text field placeholder and omits an empty one', limits, ).nodes[1]; - assert.equal(decode('Type your name')?.placeholder, 'Type your name'); + const filled = decode('Type your name'); + assert.equal(filled?.placeholder, 'Type your name'); + assert.equal(filled?.value, 'Ada Lovelace', 'the value and the placeholder are separate facts'); assert.equal(decode('')?.placeholder, undefined, 'no placeholder reads as none, not as ""'); assert.equal(decode()?.placeholder, undefined, 'an unread fact stays unknown'); }); diff --git a/website/docs/docs/snapshots.md b/website/docs/docs/snapshots.md index dfd454fb30..2bb8af5c07 100644 --- a/website/docs/docs/snapshots.md +++ b/website/docs/docs/snapshots.md @@ -151,8 +151,9 @@ them. Explicit `false` and `0` are kept; an absent field means the fact was unav empty field shows it (`hintShowing: true`, and `value` repeats it), a filled field no longer does. A field without a hint omits it. iOS nodes carry the same fact from the field's `placeholderValue`, on every producer (the XCTest tree, the Simulator AX bridge, and the runner's - private-AX reader); XCTest reports an empty field's placeholder as its `value` too, so a `value` - equal to `placeholder` is an empty field. + private-AX reader). XCTest reports an empty field's placeholder as its `value` too, so a `value` + equal to `placeholder` is either an empty field or one holding exactly that text; equality alone + cannot tell them apart. - `selectionStart`/`selectionEnd` are accessibility selection offsets. They are independent of `editable` (read-only selectable text exposes them too), they are not a character count, and they do not prove that a masked or secure value equals expected text. From 9ef008834fc4ce5800b292626a18f5e5ad5381e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oskar=20Kwas=CC=81niewski?= Date: Fri, 25 Sep 2026 12:15:58 +0200 Subject: [PATCH 3/3] fix(ios): make the placeholder attribute optional in the bridge vocabulary, read it on text entry only Review follow-ups. The Simulator AX bridge asked for the placeholder attribute in its required set, so a runtime whose vocabulary lacks it would have failed every capture; a mismatch now retries the mapping without the attribute and the capture serves without placeholders. The bridge trims the placeholder the way the runner does, so whitespace reads as none. The element sweeps read a live XCUIElement attribute per element inside their deadline, so they read placeholderValue on text-entry types only; the snapshot producer keeps reading it off the snapshot it already holds. --- .../RunnerTests+SnapshotAcquisition.swift | 16 ++++++++++++++-- apple/snapshot-bridge/SnapshotBridgeRuntime.m | 8 ++++++++ .../src/snapshot-source/tree.test.ts | 5 +++++ .../platform-apple/src/snapshot-source/tree.ts | 6 +++--- 4 files changed, 30 insertions(+), 5 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift index ef1367720f..0cbc1ea311 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift @@ -245,6 +245,18 @@ extension RunnerTests { return text.isEmpty ? nil : text } + /// The element types whose `placeholderValue` the element sweeps read. On a live `XCUIElement` + /// every attribute is one more lookup inside the sweep's deadline, so only text entry pays for + /// it; a snapshot-based producer reads the attribute off the snapshot it already holds. + static let placeholderElementTypes: Set = [ + .textField, .secureTextField, .searchField, .textView, + ] + + func elementPlaceholderText(_ element: XCUIElement, type: XCUIElement.ElementType) -> String? { + guard Self.placeholderElementTypes.contains(type) else { return nil } + return placeholderText(element.placeholderValue) + } + private func snapshotAppFrame(app: XCUIApplication) -> CGRect { #if os(iOS) return onScreenWindowFrame(app: app) @@ -387,7 +399,7 @@ extension RunnerTests { let label = element.label.trimmingCharacters(in: .whitespacesAndNewlines) let identifier = element.identifier.trimmingCharacters(in: .whitespacesAndNewlines) let valueText = snapshotValueText(element) - let placeholder = placeholderText(element.placeholderValue) + let placeholder = elementPlaceholderText(element, type: elementType) let hasContent = !label.isEmpty || !identifier.isEmpty || valueText != nil || placeholder != nil if !hasContent { return } if sameSemanticElement( @@ -571,7 +583,7 @@ extension RunnerTests { label: label.isEmpty ? nil : label, identifier: identifier.isEmpty ? nil : identifier, value: valueText, - placeholder: placeholderText(element.placeholderValue), + placeholder: elementPlaceholderText(element, type: elementType), rect: SnapshotRect(frame), enabled: enabled, focused: elementHasFocus(element) ? true : nil, diff --git a/apple/snapshot-bridge/SnapshotBridgeRuntime.m b/apple/snapshot-bridge/SnapshotBridgeRuntime.m index e50e95ebb4..4b3bd28d49 100644 --- a/apple/snapshot-bridge/SnapshotBridgeRuntime.m +++ b/apple/snapshot-bridge/SnapshotBridgeRuntime.m @@ -339,6 +339,14 @@ - (nullable NSDictionary *)snapshotForProcess:(pid_t)pid kAttributeChildren, ]; NSArray *numbers = _attributeNumbersForNames(names); + if (![numbers isKindOfClass:NSArray.class] || numbers.count != names.count) { + // The placeholder attribute is optional: a runtime whose vocabulary lacks it serves the capture + // without placeholders rather than failing every capture over a fact no consumer depends on. + NSMutableArray *required = [names mutableCopy]; + [required removeObject:kAttributePlaceholderValue]; + names = required; + numbers = _attributeNumbersForNames(names); + } if (![numbers isKindOfClass:NSArray.class] || numbers.count != names.count) { if (error) *error = failureResponse(requestId, @"reader_unavailable", @"attribute-vocabulary-mismatch", @"AX attribute vocabulary is incompatible"); finishRequestWatchdog(watchdog, watchdogState); diff --git a/packages/platform-apple/src/snapshot-source/tree.test.ts b/packages/platform-apple/src/snapshot-source/tree.test.ts index f98e892df9..e4f2c78d64 100644 --- a/packages/platform-apple/src/snapshot-source/tree.test.ts +++ b/packages/platform-apple/src/snapshot-source/tree.test.ts @@ -483,6 +483,11 @@ test('the bridge tree carries a text field placeholder and omits an empty one', assert.equal(filled?.placeholder, 'Type your name'); assert.equal(filled?.value, 'Ada Lovelace', 'the value and the placeholder are separate facts'); assert.equal(decode('')?.placeholder, undefined, 'no placeholder reads as none, not as ""'); + assert.equal( + decode(' ')?.placeholder, + undefined, + 'a whitespace placeholder is none, as on the runner', + ); assert.equal(decode()?.placeholder, undefined, 'an unread fact stays unknown'); }); diff --git a/packages/platform-apple/src/snapshot-source/tree.ts b/packages/platform-apple/src/snapshot-source/tree.ts index fce6c89d8b..d0b31724e0 100644 --- a/packages/platform-apple/src/snapshot-source/tree.ts +++ b/packages/platform-apple/src/snapshot-source/tree.ts @@ -248,6 +248,8 @@ function nodeFacts( // shape the XCTest tree produces, so a `selected:` selector cannot tell the producers apart. const selected = traits === undefined || (traits & SELECTED_TRAIT) === 0n ? undefined : true; const userInteractionEnabled = optionalBoolean(value[ATTRIBUTE.userInteractionEnabled]); + // Trimmed like the runner's `placeholderText`: a whitespace placeholder is no placeholder. + const placeholder = optionalString(value[ATTRIBUTE.placeholder])?.trim(); return { index, ...(parentIndex === undefined ? {} : { parentIndex }), @@ -262,9 +264,7 @@ function nodeFacts( ...(optionalScalar(value[ATTRIBUTE.value]) ? { value: optionalScalar(value[ATTRIBUTE.value]) } : {}), - ...(optionalString(value[ATTRIBUTE.placeholder]) - ? { placeholder: optionalString(value[ATTRIBUTE.placeholder]) } - : {}), + ...(placeholder ? { placeholder } : {}), ...(optionalString(value[ATTRIBUTE.identifier]) ? { identifier: optionalString(value[ATTRIBUTE.identifier]) } : {}),