fix(ios-runner): one synthesize-then-fallback helper; sequence taps use the standalone tap policy

.github/workflows/ios.yml

@@ -158,6 +158,7 @@ jobs:
             -xctestrun "$XCTESTRUN_PATH" \
             -destination "platform=iOS Simulator,id=${{ steps.ios-simulator.outputs.simulator-udid }}" \
             -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSinglePointerFlingFallsBackToXCTestCoordinateDragWhenPrivateSynthesisFails \
+            -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSynthesizedSequenceTapFallsBackToXCTestCoordinateTapWhenAccessibilityIsUnavailable \
             -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRecordStartThrowsTheCaptureRefusalItReceived \
             -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAlertDispatchResolvesItsOwnModalWithoutCoordinateTapRoutingProbe \
             -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAlertResolutionCannotBypassRequestedDeadline \

CHANGELOG.md

@@ -35,6 +35,16 @@
   with `observation: "unsettled"`, `wait absent` keeps polling, and the next read captures afresh.
   A failed read also carries `targetActivation` in `error.details`, and a failed interaction now
   keeps the disclosure sentences in its hint.
+  passed. That capture now carries `unsettledGesture`: `is`, `get`, `find`, and `wait` report it (in
+  `error.details` or `data`) with an appended warning, `snapshot` appends the warning, `is absent`
+  refuses with `observation: "unsettled"`, `wait absent` keeps polling, and the next read captures
+  afresh. Click, press, and fill by selector do not disclose it yet. A failed read now also carries
+  `targetActivation` in `error.details`, the same place as `unsettledGesture`.
+- Fixed (ios): a synthesized tap step inside a runner `sequence` (for example `press x y --count N`)
+  now follows the standalone tap's policy instead of its own. When accessibility is unavailable or no
+  app window resolves, the step now falls back to an XCTest coordinate tap instead of failing the
+  step with `UNSUPPORTED_OPERATION`. One helper now owns the synthesize-then-fallback decision at
+  every synthesized tap site (#2788).
 - Fixed (ios): `open` on a local Simulator now waits for the launched app's discovery before it
   decides whether the app is observable. On a loaded host `simctl spawn launchctl list` outlasts one
   1.5 s discovery wait slice, and the launch observation read that slice as an unobservable app, so

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift

@@ -78,20 +78,15 @@ extension RunnerTests {
               app: activeApp,
               policy: synthesizedGesturePolicy(policyKind)
             )
-            let (timing, outcome) = performGesture(activeApp, idleTimeout: false) {
+            switch performSynthesizedGesture(activeApp, kind: policyKind, context: context, synthesize: {
               synthesizedTapAt(
                 app: activeApp,
                 x: touchPoint.x,
                 y: touchPoint.y,
                 context: context
               )
-            }
-            if case .performed = outcome {
-              logSynthesizedGesturePolicyDecision(
-                kind: policyKind,
-                context: context,
-                fallbackAttempted: false
-              )
+            }) {
+            case .performed(let timing):
               if isTextEntry {
                 waitForTextEntryReadinessAfterTap(app: activeApp, element: element)
               }
@@ -107,13 +102,12 @@ extension RunnerTests {
                   ? match.usedNonHittableFallback
                   : nil
               )
+            case .xctestFallback(let message, let hint):
+              fallback = GestureFallback(strategy: "xctest-coordinate-tap", message: message, hint: hint)
+            case .refused(_, let message, let hint):
+              clearRememberedTextEntryTap()
+              return unsupportedResponse(message: message, hint: hint)
             }
-            logSynthesizedGesturePolicyDecision(
-              kind: policyKind,
-              context: context,
-              fallbackAttempted: true
-            )
-            fallback = gestureFallback(strategy: "xctest-coordinate-tap", from: outcome)
           }
           let (timing, outcome) = performGesture(activeApp) {
             if expectedPoint != nil || match.usedNonHittableFallback {
@@ -172,16 +166,18 @@ extension RunnerTests {
             app: activeApp,
             policy: synthesizedGesturePolicy(policyKind)
           )
-          let (timing, outcome) = performGesture(activeApp, idleTimeout: false) {
+          switch performSynthesizedGesture(activeApp, kind: policyKind, context: context, synthesize: {
             synthesizedTapAt(app: activeApp, x: x, y: y, context: context)
-          }
-          if case .performed = outcome {
-            logSynthesizedGesturePolicyDecision(kind: policyKind, context: context, fallbackAttempted: false)
+          }) {
+          case .performed(let timing):
             rememberTextEntryTap(textInput)
             return gestureResponse(message: "tapped", timing: timing)
+          case .xctestFallback(let message, let hint):
+            fallback = GestureFallback(strategy: "xctest-coordinate-tap", message: message, hint: hint)
+          case .refused(_, let message, let hint):
+            clearRememberedTextEntryTap()
+            return unsupportedResponse(message: message, hint: hint)
           }
-          logSynthesizedGesturePolicyDecision(kind: policyKind, context: context, fallbackAttempted: true)
-          fallback = gestureFallback(strategy: "xctest-coordinate-tap", from: outcome)
         }
         let touchFrame = resolvedTouchVisualizationFrame(app: activeApp, x: x, y: y)
         let (timing, outcome) = performGesture(activeApp) { tapAt(app: activeApp, x: x, y: y) }

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+GestureExecution.swift

@@ -25,13 +25,17 @@ extension RunnerTests {
     case .performed:
       return nil
     case .unsupported(let message, let hint):
-      return Response(
-        ok: false,
-        error: ErrorPayload(code: "UNSUPPORTED_OPERATION", message: message, hint: hint)
-      )
+      return unsupportedResponse(message: message, hint: hint)
     }
   }
 
+  func unsupportedResponse(message: String, hint: String?) -> Response {
+    Response(
+      ok: false,
+      error: ErrorPayload(code: "UNSUPPORTED_OPERATION", message: message, hint: hint)
+    )
+  }
+
   /// Optional visualization frame returned with a gesture response.
   enum GestureFrame {
     case none
@@ -45,15 +49,6 @@ extension RunnerTests {
     let hint: String?
   }
 
-  func gestureFallback(strategy: String, from outcome: RunnerInteractionOutcome) -> GestureFallback? {
-    switch outcome {
-    case .performed:
-      return nil
-    case .unsupported(let message, let hint):
-      return GestureFallback(strategy: strategy, message: message, hint: hint)
-    }
-  }
-
 
   /// Runs a gesture action with uniform timing capture. Touch gestures pass `idleTimeout: true`
   /// (the default) to run inside the scroll idle-timeout + quiescence-skip wrapper; synthesis

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift

@@ -159,17 +159,20 @@ extension RunnerTests {
       app: app,
       policy: synthesizedGesturePolicy(.coordinateTap)
     )?.withReferenceFrame(frame)
-    let synthesized = performGesture(app, idleTimeout: false) {
+    switch performSynthesizedGesture(app, kind: .coordinateTap, context: context, synthesize: {
       synthesizedTapAt(app: app, x: point.x, y: point.y, context: context)
-    }
-    if case .performed = synthesized.outcome {
-      return verifyNavigationFallbackOutcome(app: app, before: before)
-    }
-    let fallback = performGesture(app) {
-      tapAt(app: app, x: point.x, y: point.y)
-    }
-    if case .performed = fallback.outcome {
+    }) {
+    case .performed:
       return verifyNavigationFallbackOutcome(app: app, before: before)
+    case .refused:
+      return .unavailable
+    case .xctestFallback:
+      let fallback = performGesture(app) {
+        tapAt(app: app, x: point.x, y: point.y)
+      }
+      if case .performed = fallback.outcome {
+        return verifyNavigationFallbackOutcome(app: app, before: before)
+      }
     }
 #endif
     return .unavailable

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollDragExecution.swift

@@ -149,19 +149,6 @@ extension RunnerTests {
       context: context
     )
     else {
-      if context?.allowsXCTestCoordinateFallback == true {
-        logSynthesizedGesturePolicyDecision(kind: policyKind, context: context, fallbackAttempted: true)
-        return executeCoordinateDragFallback(
-          activeApp: activeApp,
-          x: x,
-          y: y,
-          x2: x2,
-          y2: y2,
-          durationMs: durationMs,
-          message: message,
-          fallback: nil
-        )
-      }
       logSynthesizedGesturePolicyDecision(kind: policyKind, context: context, fallbackAttempted: false)
       return Response(
         ok: false,
@@ -179,7 +166,7 @@ extension RunnerTests {
       y2: plan.points.y2,
       referenceFrame: plan.referenceFrame
     )
-    let (timing, outcome) = performGesture(activeApp, idleTimeout: false) {
+    switch performSynthesizedGesture(activeApp, kind: policyKind, context: plan.context, synthesize: {
       synthesizedDragAt(
         app: activeApp,
         x: plan.points.x,
@@ -190,13 +177,10 @@ extension RunnerTests {
         profile: profile,
         context: plan.context
       )
-    }
-    if case .performed = outcome {
-      logSynthesizedGesturePolicyDecision(kind: policyKind, context: plan.context, fallbackAttempted: false)
+    }) {
+    case .performed(let timing):
       return gestureResponse(message: message, timing: timing, frame: .drag(dragFrame))
-    }
-    if plan.context.allowsXCTestCoordinateFallback {
-      logSynthesizedGesturePolicyDecision(kind: policyKind, context: plan.context, fallbackAttempted: true)
+    case .xctestFallback(let fallbackMessage, let hint):
       return executeCoordinateDragFallback(
         activeApp: activeApp,
         x: plan.points.x,
@@ -205,11 +189,11 @@ extension RunnerTests {
         y2: plan.points.y2,
         durationMs: durationMs,
         message: message,
-        fallback: gestureFallback(strategy: "xctest-coordinate-drag", from: outcome)
+        fallback: GestureFallback(strategy: "xctest-coordinate-drag", message: fallbackMessage, hint: hint)
       )
+    case .refused(_, let refusalMessage, let hint):
+      return unsupportedResponse(message: refusalMessage, hint: hint)
     }
-    logSynthesizedGesturePolicyDecision(kind: policyKind, context: plan.context, fallbackAttempted: false)
-    return unsupportedResponse(for: outcome)
 #else
     return nil
 #endif
@@ -223,7 +207,7 @@ extension RunnerTests {
     y2: Double,
     durationMs: Double,
     message: String,
-    fallback: GestureFallback?
+    fallback: GestureFallback
   ) -> Response {
     let dragPoints = keyboardAvoidingDragPoints(app: activeApp, x: x, y: y, x2: x2, y2: y2)
     let dragFrame = resolvedDragVisualizationFrame(

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift

@@ -143,51 +143,21 @@ extension RunnerTests {
   ) -> SequenceStepOutcome {
     let x = step.x ?? 0
     let y = step.y ?? 0
-    // Synthesized HID tap fast path mirrors the individual `tap` command (idleTimeout:false, with
-    // a tapAt fallback when synthesis is unsupported), so fusing a jittered tap series does not
-    // change the touch mechanism for these inputs.
-    if step.kind == "tap", step.synthesized == true {
-      let policyKind = SynthesizedGesturePolicyKind.synthesizedDrag
-#if os(iOS)
-      guard let synthesizedContext else {
-        let nowMs = ProcessInfo.processInfo.systemUptime * 1000
-        logSynthesizedGesturePolicyDecision(kind: policyKind, context: nil, fallbackAttempted: false)
-        return SequenceStepOutcome(
-          outcome: .unsupported(
-            message: "synthesized coordinate tap could not resolve a finite coordinate frame",
-            hint: "Retry after the app is foregrounded, or use a plain screenshot to choose coordinates."
-          ),
-          gestureStartUptimeMs: nowMs,
-          gestureEndUptimeMs: nowMs
-        )
-      }
-#endif
-      let (timing, outcome) = performGesture(activeApp, idleTimeout: false) {
+    if let policyKind = synthesizedPolicyKind(forSequenceStep: step) {
+      switch performSynthesizedGesture(activeApp, kind: policyKind, context: synthesizedContext, synthesize: {
         synthesizedTapAt(app: activeApp, x: x, y: y, context: synthesizedContext)
-      }
-      if case .performed = outcome {
-        logSynthesizedGesturePolicyDecision(kind: policyKind, context: synthesizedContext, fallbackAttempted: false)
-        if let pauseMs = step.pauseMs, pauseMs > 0 {
-          sleepFor(min(max(pauseMs, 0), 10000) / 1000.0)
-        }
-        return SequenceStepOutcome(
-          outcome: outcome,
-          gestureStartUptimeMs: timing.gestureStartUptimeMs,
-          gestureEndUptimeMs: timing.gestureEndUptimeMs
-        )
-      }
-#if os(iOS)
-      guard synthesizedContext.allowsXCTestCoordinateFallback else {
-        logSynthesizedGesturePolicyDecision(kind: policyKind, context: synthesizedContext, fallbackAttempted: false)
-        return SequenceStepOutcome(
-          outcome: outcome,
-          gestureStartUptimeMs: timing.gestureStartUptimeMs,
-          gestureEndUptimeMs: timing.gestureEndUptimeMs
+      }) {
+      case .performed(let timing):
+        return finishedSequenceStep(step, timing: timing, outcome: .performed)
+      case .refused(let timing, let message, let hint):
+        return finishedSequenceStep(
+          step,
+          timing: timing,
+          outcome: .unsupported(message: message, hint: hint)
         )
+      case .xctestFallback:
+        break
       }
-      logSynthesizedGesturePolicyDecision(kind: policyKind, context: synthesizedContext, fallbackAttempted: true)
-#endif
-      // Synthesis unsupported (e.g. macOS) — fall through to the drag-based tapAt below.
     }
     let (timing, outcome) = performGesture(activeApp) {
       switch step.kind {
@@ -201,6 +171,14 @@ extension RunnerTests {
         return tapAt(app: activeApp, x: x, y: y)
       }
     }
+    return finishedSequenceStep(step, timing: timing, outcome: outcome)
+  }
+
+  private func finishedSequenceStep(
+    _ step: SequenceStep,
+    timing: (gestureStartUptimeMs: Double, gestureEndUptimeMs: Double),
+    outcome: RunnerInteractionOutcome
+  ) -> SequenceStepOutcome {
     // Sleep AFTER the step — pauseMs is the inter-step gap — but only when the step performed.
     // assembleSequenceExecution stops at the first unsupported outcome, so pausing after a failed
     // step would burn up to 10s of watchdog budget with no following step to separate from.