feat(ios): the snapshot node carries the field's placeholder

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.m

@@ -365,6 +365,7 @@ + (NSArray *)snapshotAttributes
     @"identifier",
     @"label",
     @"value",
+    @"placeholderValue",
     @"frame",
     @"enabled",
     @"selected",
@@ -388,8 +389,8 @@ + (NSArray *)snapshotAttributes
       // The mapper expands keypaths with extra attributes (automation type, window display
       // id, base type) that are disproportionately expensive for the AX server to compute
       // on large React Native trees. Keep only the attributes we actually consume.
-      NSArray *needed = @[ @"ElementType", @"Identifier", @"Label", @"Value", @"Frame",
-                           @"Enabled", @"Selected", @"Focus" ];
+      NSArray *needed = @[ @"ElementType", @"Identifier", @"Label", @"Value", @"PlaceholderValue",
+                           @"Frame", @"Enabled", @"Selected", @"Focus" ];
       NSMutableArray *filtered = [NSMutableArray array];
       for (id attribute in (NSArray *)mapped) {
         NSString *name = [attribute description];
@@ -764,6 +765,7 @@ + (nullable NSMutableDictionary *)dictionaryForSnapshot:(id)snapshot
   result[@"identifier"] = [self stringValueForKey:@"identifier" snapshot:snapshot] ?: @"";
   result[@"label"] = [self stringValueForKey:@"label" snapshot:snapshot] ?: @"";
   result[@"value"] = [self stringValueForKey:@"value" snapshot:snapshot] ?: @"";
+  result[@"placeholder"] = [self stringValueForKey:@"placeholderValue" snapshot:snapshot] ?: @"";
   result[@"frame"] = [self frameValueForSnapshot:snapshot];
   result[@"enabled"] = [self boolNumberForKey:@"enabled" snapshot:snapshot defaultValue:YES];
   result[@"selected"] = [self boolNumberForKey:@"selected" snapshot:snapshot defaultValue:NO];

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+PrivateAXPresentation.swift

@@ -8,6 +8,7 @@ struct PrivateAXFields {
   let label: String
   let identifier: String
   let value: String
+  let placeholder: String
   let rawType: Int
   let elementType: XCUIElement.ElementType?
   let enabled: Bool
@@ -52,6 +53,7 @@ extension RunnerTests {
       label: privateAXPresentationString(raw["label"]),
       identifier: privateAXPresentationString(raw["identifier"]),
       value: privateAXPresentationString(raw["value"]),
+      placeholder: privateAXPresentationString(raw["placeholder"]),
       rawType: rawType,
       elementType: privateAXElementType(rawElementType: rawType),
       enabled: privateAXPresentationBool(raw["enabled"]) ?? true,
@@ -70,6 +72,7 @@ extension RunnerTests {
       label: fields.label.isEmpty ? nil : fields.label,
       identifier: fields.identifier.isEmpty ? nil : fields.identifier,
       value: fields.value.isEmpty ? nil : fields.value,
+      placeholder: fields.placeholder.isEmpty ? nil : fields.placeholder,
       rect: SnapshotRect(fields.rect), enabled: fields.enabled,
       focused: fields.focused, selected: fields.selected,
       hittable: false,

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift

@@ -35,6 +35,7 @@ extension RunnerTests {
     let label: String
     let identifier: String
     let valueText: String?
+    let placeholder: String?
     let focused: Bool
     let selected: Bool
   }
@@ -500,6 +501,7 @@ extension RunnerTests {
           label: candidate.label,
           identifier: candidate.identifier,
           value: candidate.value,
+          placeholder: candidate.placeholder,
           rect: candidate.rect,
           enabled: candidate.enabled,
           focused: candidate.focused,

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift

@@ -189,6 +189,7 @@ extension RunnerTests {
       label: label,
       identifier: identifier,
       valueText: valueText,
+      placeholder: placeholderText(snapshot.placeholderValue),
       focused: snapshotHasFocus(snapshot),
       selected: snapshotIsSelected(snapshot)
     )
@@ -209,6 +210,7 @@ extension RunnerTests {
       label: evaluation.label.isEmpty ? nil : evaluation.label,
       identifier: evaluation.identifier.isEmpty ? nil : evaluation.identifier,
       value: evaluation.valueText,
+      placeholder: evaluation.placeholder,
       rect: SnapshotRect(snapshot.frame),
       enabled: snapshot.isEnabled,
       focused: evaluation.focused ? true : nil,
@@ -235,6 +237,26 @@ extension RunnerTests {
     return text.isEmpty ? nil : text
   }
 
+  /// The placeholder as the node publishes it: XCTest answers `placeholderValue` for a text field
+  /// whether or not the field is empty, and an empty string for everything else, which reads as
+  /// no placeholder. The private-AX bridge asks the AX server the same attribute.
+  func placeholderText(_ placeholderValue: String?) -> String? {
+    let text = placeholderValue?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
+    return text.isEmpty ? nil : text
+  }
+
+  /// The element types whose `placeholderValue` the element sweeps read. On a live `XCUIElement`
+  /// every attribute is one more lookup inside the sweep's deadline, so only text entry pays for
+  /// it; a snapshot-based producer reads the attribute off the snapshot it already holds.
+  static let placeholderElementTypes: Set<XCUIElement.ElementType> = [
+    .textField, .secureTextField, .searchField, .textView,
+  ]
+
+  func elementPlaceholderText(_ element: XCUIElement, type: XCUIElement.ElementType) -> String? {
+    guard Self.placeholderElementTypes.contains(type) else { return nil }
+    return placeholderText(element.placeholderValue)
+  }
+
   private func snapshotAppFrame(app: XCUIApplication) -> CGRect {
 #if os(iOS)
     return onScreenWindowFrame(app: app)
@@ -341,6 +363,7 @@ extension RunnerTests {
         label: node.label,
         identifier: node.identifier,
         value: node.value,
+        placeholder: node.placeholder,
         rect: node.rect,
         enabled: node.enabled,
         focused: node.focused,
@@ -376,7 +399,8 @@ extension RunnerTests {
       let label = element.label.trimmingCharacters(in: .whitespacesAndNewlines)
       let identifier = element.identifier.trimmingCharacters(in: .whitespacesAndNewlines)
       let valueText = snapshotValueText(element)
-      let hasContent = !label.isEmpty || !identifier.isEmpty || valueText != nil
+      let placeholder = elementPlaceholderText(element, type: elementType)
+      let hasContent = !label.isEmpty || !identifier.isEmpty || valueText != nil || placeholder != nil
       if !hasContent { return }
       if sameSemanticElement(
         containerSnapshot: containerSnapshot,
@@ -395,6 +419,7 @@ extension RunnerTests {
         label: label.isEmpty ? nil : label,
         identifier: identifier.isEmpty ? nil : identifier,
         value: valueText,
+        placeholder: placeholder,
         rect: SnapshotRect(frame),
         enabled: element.isEnabled,
         focused: elementHasFocus(element) ? true : nil,
@@ -558,6 +583,7 @@ extension RunnerTests {
         label: label.isEmpty ? nil : label,
         identifier: identifier.isEmpty ? nil : identifier,
         value: valueText,
+        placeholder: elementPlaceholderText(element, type: elementType),
         rect: SnapshotRect(frame),
         enabled: enabled,
         focused: elementHasFocus(element) ? true : nil,

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+PrivateAXPresentationTests.swift

@@ -117,6 +117,29 @@ extension RunnerTests {
     )
   }
 
+  /// The private-AX reader hands over `placeholderValue` beside `value`; the acquisition publishes
+  /// it on the node and reads an empty one as no placeholder, the way every other producer does.
+  func testPrivateAXAcquisitionCarriesTheFieldPlaceholder() {
+    let frame = Self.privateAXFrame
+    let nodes = privateAXNormalizedAcquisition(
+      rawRoot: [
+        "type": Int(XCUIElement.ElementType.application.rawValue), "frame": frame(0, 0, 402, 874),
+        "children": [
+          ["type": Int(XCUIElement.ElementType.textField.rawValue), "value": "Type your name",
+            "placeholder": "Type your name", "frame": frame(16, 200, 370, 44), "children": []],
+          ["type": Int(XCUIElement.ElementType.button.rawValue), "label": "Save",
+            "placeholder": "", "frame": frame(16, 300, 370, 44), "children": []]
+        ]
+      ],
+      hint: CaptureHint(
+        projection: .raw, depth: nil, regularPresentedDepth: nil,
+        interactiveOnly: false, customActions: false),
+      viewport: CGRect(x: 0, y: 0, width: 402, height: 874),
+      interfaceOrientation: RunnerInterfaceOrientation.portrait)
+
+    XCTAssertEqual(nodes.map(\.placeholder), [nil, "Type your name", nil])
+  }
+
   func testPrivateAXRegularPresentationProjectsToViewportAndKeepsScrollHint() throws {
     let nodes = try privateAXRegularPresentation(
       rawRoot: Self.privateAXScrolledFixture,

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationTests.swift

@@ -12,6 +12,7 @@ extension RunnerTests {
       label: "Continue",
       identifier: "continue-button",
       value: "Ready",
+      placeholder: "Type here",
       rect: SnapshotRect(x: 10, y: 20, width: 100, height: 44),
       enabled: true,
       focused: true,
@@ -54,14 +55,52 @@ extension RunnerTests {
 
     XCTAssertEqual(
       String(decoding: encoded, as: UTF8.self),
-      #"[{"actions":["Open menu"],"depth":2,"enabled":true,"focused":true,"hiddenContentAbove":true,"hiddenContentBelow":true,"hittable":true,"identifier":"continue-button","index":3,"label":"Continue","parentIndex":1,"rect":{"height":44,"width":100,"x":10,"y":20},"selected":true,"type":"Button","value":"Ready"}]"#
+      #"[{"actions":["Open menu"],"depth":2,"enabled":true,"focused":true,"hiddenContentAbove":true,"hiddenContentBelow":true,"hittable":true,"identifier":"continue-button","index":3,"label":"Continue","parentIndex":1,"placeholder":"Type here","rect":{"height":44,"width":100,"x":10,"y":20},"selected":true,"type":"Button","value":"Ready"}]"#
     )
     XCTAssertEqual(capture.truncated, true)
     XCTAssertEqual(capture.effectiveDepth, 4)
     XCTAssertEqual(capture.customActions?.read, 1)
     XCTAssertEqual(capture.customActions?.candidates, 2)
   }
 
+  /// A node without a placeholder omits the key: the wire contract is "absent when empty", which
+  /// the synthesized `encodeIfPresent` provides today and a hand-written encoder must keep.
+  func testPresentedNodeOmitsAnAbsentPlaceholder() throws {
+    let raw = RawAXNode(
+      index: 0,
+      type: "Button",
+      label: "Save",
+      identifier: nil,
+      value: nil,
+      rect: SnapshotRect(x: 0, y: 0, width: 100, height: 44),
+      enabled: true,
+      focused: nil,
+      selected: nil,
+      hittable: true,
+      depth: 0,
+      parentIndex: nil,
+      hiddenContentAbove: nil,
+      hiddenContentBelow: nil
+    )
+    let capture = try XCTUnwrap(try SnapshotPresentation.present(
+      SnapshotAcquisition(
+        hint: CaptureHint(
+          projection: .raw, depth: nil, regularPresentedDepth: nil,
+          interactiveOnly: false, customActions: false),
+        nodes: [raw],
+        truncated: false,
+        effectiveDepth: nil,
+        customActions: nil,
+        viewport: .reported(box: CGRect(x: 0, y: 0, width: 100, height: 100))
+      ),
+      options: PresentationOptions(interactiveOnly: false, depth: nil, scope: nil, raw: true)
+    ))
+    let encoded = String(decoding: try JSONEncoder().encode(capture.nodes), as: UTF8.self)
+
+    XCTAssertNil(capture.nodes.first?.placeholder)
+    XCTAssertFalse(encoded.contains("placeholder"), encoded)
+  }
+
   func testSnapshotPresentationOwnsBackendNeutralEligibility() throws {
     // Non-vacuity: forcing `isEligibleForRegularPresentation` to return true made this test execute
     // once and fail exactly four membership/index/depth/parent assertions before restoration.

apple/snapshot-bridge/SnapshotBridgeRuntime.m

@@ -18,7 +18,7 @@
 NSString *const kProtocolVersionKey = @"protocolVersion";
 NSString *const kSourceVersionKey = @"sourceVersion";
 NSString *const kRequestIdKey = @"requestId";
-NSString *const kSourceVersion = @"agent-device-simulator-ax-v1.7.0";
+NSString *const kSourceVersion = @"agent-device-simulator-ax-v1.8.0";
 const NSUInteger kProtocolVersion = 1;
 const uint32_t kMaximumFrameBytes = 16 * 1024 * 1024;
 const NSUInteger kMaximumDepth = 128;
@@ -29,6 +29,7 @@
 static NSString *const kAttributeElementBaseType = @"XC_kAXXCAttributeElementBaseType";
 static NSString *const kAttributeLabel = @"XC_kAXXCAttributeLabel";
 static NSString *const kAttributeValue = @"XC_kAXXCAttributeValue";
+static NSString *const kAttributePlaceholderValue = @"XC_kAXXCAttributePlaceholderValue";
 static NSString *const kAttributeIdentifier = @"XC_kAXXCAttributeIdentifier";
 static NSString *const kAttributeFrame = @"XC_kAXXCAttributeFrame";
 static NSString *const kAttributeAutomationType = @"XC_kAXXCAttributeAutomationType";
@@ -330,13 +331,22 @@ - (nullable NSDictionary *)snapshotForProcess:(pid_t)pid
     kAttributeElementBaseType,
     kAttributeLabel,
     kAttributeValue,
+    kAttributePlaceholderValue,
     kAttributeIdentifier,
     kAttributeFrame,
     kAttributeAutomationType,
     kAttributeTraits,
     kAttributeChildren,
   ];
   NSArray<NSNumber *> *numbers = _attributeNumbersForNames(names);
+  if (![numbers isKindOfClass:NSArray.class] || numbers.count != names.count) {
+    // The placeholder attribute is optional: a runtime whose vocabulary lacks it serves the capture
+    // without placeholders rather than failing every capture over a fact no consumer depends on.
+    NSMutableArray<NSString *> *required = [names mutableCopy];
+    [required removeObject:kAttributePlaceholderValue];
+    names = required;
+    numbers = _attributeNumbersForNames(names);
+  }
   if (![numbers isKindOfClass:NSArray.class] || numbers.count != names.count) {
     if (error) *error = failureResponse(requestId, @"reader_unavailable", @"attribute-vocabulary-mismatch", @"AX attribute vocabulary is incompatible");
     finishRequestWatchdog(watchdog, watchdogState);

apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotModels.swift

@@ -34,6 +34,8 @@ public struct RawAXNode: Equatable {
   public let label: String?
   public let identifier: String?
   public let value: String?
+  /// The text a text field shows while empty (`placeholderValue`); `nil` for every other node.
+  public let placeholder: String?
   public var rect: SnapshotRect
   public let enabled: Bool
   public let focused: Bool?
@@ -52,6 +54,7 @@ public struct RawAXNode: Equatable {
     label: String?,
     identifier: String?,
     value: String?,
+    placeholder: String? = nil,
     rect: SnapshotRect,
     enabled: Bool,
     focused: Bool?,
@@ -68,6 +71,7 @@ public struct RawAXNode: Equatable {
     self.label = label
     self.identifier = identifier
     self.value = value
+    self.placeholder = placeholder
     self.rect = rect
     self.enabled = enabled
     self.focused = focused
@@ -272,6 +276,7 @@ public struct PresentedNode: Codable, Equatable {
   public let label: String?
   public let identifier: String?
   public let value: String?
+  public let placeholder: String?
   public let rect: SnapshotRect
   public let enabled: Bool
   public let focused: Bool?
@@ -295,6 +300,7 @@ public struct PresentedNode: Codable, Equatable {
     self.label = raw.label
     self.identifier = raw.identifier
     self.value = raw.value
+    self.placeholder = raw.placeholder
     self.rect = rect ?? raw.rect
     self.enabled = raw.enabled
     self.focused = raw.focused

apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotPresentationProjection.swift

@@ -140,6 +140,7 @@ extension SnapshotPresentation {
           label: raw.label,
           identifier: raw.identifier,
           value: raw.value,
+          placeholder: raw.placeholder,
           rect: raw.rect,
           enabled: raw.enabled,
           focused: raw.focused,

apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotVisibilityFold.swift

@@ -199,6 +199,7 @@ public enum SnapshotVisibilityFold {
               label: node.label,
               identifier: node.identifier,
               value: node.value,
+              placeholder: node.placeholder,
               rect: node.rect,
               enabled: node.enabled,
               focused: node.focused,

apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotVisibilityFoldProjection.swift

@@ -79,6 +79,7 @@ extension SnapshotVisibilityFold {
           label: node.label,
           identifier: node.identifier,
           value: node.value,
+          placeholder: node.placeholder,
           rect: node.rect,
           enabled: node.enabled,
           focused: node.focused,

packages/platform-apple/src/snapshot-source/fixtures/wire-vocabulary.json

@@ -1,6 +1,6 @@
 {
   "protocolVersion": 1,
-  "sourceVersion": "agent-device-simulator-ax-v1.7.0",
+  "sourceVersion": "agent-device-simulator-ax-v1.8.0",
   "requestKeys": [
     "verb",
     "requestId",
@@ -34,6 +34,7 @@
     "XC_kAXXCAttributeElementBaseType",
     "XC_kAXXCAttributeLabel",
     "XC_kAXXCAttributeValue",
+    "XC_kAXXCAttributePlaceholderValue",
     "XC_kAXXCAttributeIdentifier",
     "XC_kAXXCAttributeFrame",
     "XC_kAXXCAttributeAutomationType",

packages/platform-apple/src/snapshot-source/protocol.test.ts

@@ -142,7 +142,7 @@ test('wire vocabulary guard keeps TS and Objective-C literals aligned', async ()
   assert.deepEqual(wireVocabulary.responseKeys, SNAPSHOT_SOURCE_RESPONSE_KEYS);
   assert.deepEqual(wireVocabulary.attributeKeys, SNAPSHOT_SOURCE_ATTRIBUTE_KEYS);
   assert.match(nativeSource, /kProtocolVersion = 1/);
-  assert.match(nativeSource, /kSourceVersion = @"agent-device-simulator-ax-v1\.7\.0"/);
+  assert.match(nativeSource, /kSourceVersion = @"agent-device-simulator-ax-v1\.8\.0"/);
   for (const key of [
     ...wireVocabulary.requestKeys,
     ...wireVocabulary.responseKeys,

packages/platform-apple/src/snapshot-source/protocol.ts

@@ -3,7 +3,7 @@ import { snapshotSourceError } from './errors.ts';
 import type { SnapshotSourceLimits } from './types.ts';
 
 export const SNAPSHOT_SOURCE_PROTOCOL_VERSION = 1;
-export const SNAPSHOT_SOURCE_VERSION = 'agent-device-simulator-ax-v1.7.0';
+export const SNAPSHOT_SOURCE_VERSION = 'agent-device-simulator-ax-v1.8.0';
 const FRAME_HEADER_BYTES = 4;
 
 export const SNAPSHOT_SOURCE_WIRE_KEYS = Object.freeze([
@@ -41,6 +41,7 @@ export const SNAPSHOT_SOURCE_ATTRIBUTE_KEYS = Object.freeze([
   'XC_kAXXCAttributeElementBaseType',
   'XC_kAXXCAttributeLabel',
   'XC_kAXXCAttributeValue',
+  'XC_kAXXCAttributePlaceholderValue',
   'XC_kAXXCAttributeIdentifier',
   'XC_kAXXCAttributeFrame',
   'XC_kAXXCAttributeAutomationType',