From fc2623775091d2e376b63cee70d8f8a78847d5b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 23 Sep 2026 21:48:50 +0200 Subject: [PATCH 1/3] fix(ios-runner): route every synthesized gesture through one fallback helper A synthesized tap step inside `sequence` resolved its context with the synthesized-drag policy, so it refused the XCTest coordinate fallback when accessibility was unavailable and failed outright when no window frame resolved. A standalone synthesized tap falls back in both cases. `performSynthesizedGesture` now performs the synthesized gesture, decides the XCTest coordinate fallback from the gesture kind's policy, and logs the decision. Selector tap, coordinate tap, sequence tap step, type focus and drag use it; each site keeps its own XCTest gesture and bookkeeping. A sequence tap step takes the coordinate-tap policy, and the fallback decision reads the kind rather than a policy copy on the coordinate context. Remove the unused `whenAccessibilityHealthy` keyboard policy, the drag fallback branch that a nil frame could never reach, and the text-entry focus predicate the helper replaces. Scroll keeps `privateSynthesisRequired`. Fixes #2788 --- .../RunnerTests+CommandExecution.swift | 36 ++++---- .../RunnerTests+GestureExecution.swift | 21 ++--- .../RunnerTests+ScrollDragExecution.swift | 32 ++----- .../RunnerTests+SequenceExecution.swift | 62 +++++--------- ...RunnerTests+SynthesizedGesturePolicy.swift | 67 ++++++++++----- .../RunnerTests+SynthesizedInteraction.swift | 1 - .../RunnerTests+SynthesizedTextEntry.swift | 7 -- .../RunnerTests+TypeExecution.swift | 21 ++--- .../RunnerTests+SequenceExecutionTests.swift | 19 +---- ...rTests+SynthesizedGesturePolicyTests.swift | 83 ++++++++++++++++++- .../RunnerTests+TextEntryPolicyTests.swift | 9 -- .../0011-interaction-guarantee-contract.md | 10 +-- 12 files changed, 193 insertions(+), 175 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index 77bdd1a17f..491b7962ad 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -78,20 +78,15 @@ extension RunnerTests { app: activeApp, policy: synthesizedGesturePolicy(policyKind) ) - let (timing, outcome) = performGesture(activeApp, idleTimeout: false) { + switch performSynthesizedGesture(activeApp, kind: policyKind, context: context, synthesize: { synthesizedTapAt( app: activeApp, x: touchPoint.x, y: touchPoint.y, context: context ) - } - if case .performed = outcome { - logSynthesizedGesturePolicyDecision( - kind: policyKind, - context: context, - fallbackAttempted: false - ) + }) { + case .performed(let timing): if isTextEntry { waitForTextEntryReadinessAfterTap(app: activeApp, element: element) } @@ -107,13 +102,12 @@ extension RunnerTests { ? match.usedNonHittableFallback : nil ) + case .xctestFallback(let message, let hint): + fallback = GestureFallback(strategy: "xctest-coordinate-tap", message: message, hint: hint) + case .refused(_, let message, let hint): + clearRememberedTextEntryTap() + return unsupportedResponse(message: message, hint: hint) } - logSynthesizedGesturePolicyDecision( - kind: policyKind, - context: context, - fallbackAttempted: true - ) - fallback = gestureFallback(strategy: "xctest-coordinate-tap", from: outcome) } let (timing, outcome) = performGesture(activeApp) { if expectedPoint != nil || match.usedNonHittableFallback { @@ -172,16 +166,18 @@ extension RunnerTests { app: activeApp, policy: synthesizedGesturePolicy(policyKind) ) - let (timing, outcome) = performGesture(activeApp, idleTimeout: false) { + switch performSynthesizedGesture(activeApp, kind: policyKind, context: context, synthesize: { synthesizedTapAt(app: activeApp, x: x, y: y, context: context) - } - if case .performed = outcome { - logSynthesizedGesturePolicyDecision(kind: policyKind, context: context, fallbackAttempted: false) + }) { + case .performed(let timing): rememberTextEntryTap(textInput) return gestureResponse(message: "tapped", timing: timing) + case .xctestFallback(let message, let hint): + fallback = GestureFallback(strategy: "xctest-coordinate-tap", message: message, hint: hint) + case .refused(_, let message, let hint): + clearRememberedTextEntryTap() + return unsupportedResponse(message: message, hint: hint) } - logSynthesizedGesturePolicyDecision(kind: policyKind, context: context, fallbackAttempted: true) - fallback = gestureFallback(strategy: "xctest-coordinate-tap", from: outcome) } let touchFrame = resolvedTouchVisualizationFrame(app: activeApp, x: x, y: y) let (timing, outcome) = performGesture(activeApp) { tapAt(app: activeApp, x: x, y: y) } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+GestureExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+GestureExecution.swift index 46d14d17f7..7959acbf94 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+GestureExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+GestureExecution.swift @@ -25,13 +25,17 @@ extension RunnerTests { case .performed: return nil case .unsupported(let message, let hint): - return Response( - ok: false, - error: ErrorPayload(code: "UNSUPPORTED_OPERATION", message: message, hint: hint) - ) + return unsupportedResponse(message: message, hint: hint) } } + func unsupportedResponse(message: String, hint: String?) -> Response { + Response( + ok: false, + error: ErrorPayload(code: "UNSUPPORTED_OPERATION", message: message, hint: hint) + ) + } + /// Optional visualization frame returned with a gesture response. enum GestureFrame { case none @@ -45,15 +49,6 @@ extension RunnerTests { let hint: String? } - func gestureFallback(strategy: String, from outcome: RunnerInteractionOutcome) -> GestureFallback? { - switch outcome { - case .performed: - return nil - case .unsupported(let message, let hint): - return GestureFallback(strategy: strategy, message: message, hint: hint) - } - } - /// Runs a gesture action with uniform timing capture. Touch gestures pass `idleTimeout: true` /// (the default) to run inside the scroll idle-timeout + quiescence-skip wrapper; synthesis diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollDragExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollDragExecution.swift index 022a23d61c..77c896c5ba 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollDragExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollDragExecution.swift @@ -149,19 +149,6 @@ extension RunnerTests { context: context ) else { - if context?.allowsXCTestCoordinateFallback == true { - logSynthesizedGesturePolicyDecision(kind: policyKind, context: context, fallbackAttempted: true) - return executeCoordinateDragFallback( - activeApp: activeApp, - x: x, - y: y, - x2: x2, - y2: y2, - durationMs: durationMs, - message: message, - fallback: nil - ) - } logSynthesizedGesturePolicyDecision(kind: policyKind, context: context, fallbackAttempted: false) return Response( ok: false, @@ -179,7 +166,7 @@ extension RunnerTests { y2: plan.points.y2, referenceFrame: plan.referenceFrame ) - let (timing, outcome) = performGesture(activeApp, idleTimeout: false) { + switch performSynthesizedGesture(activeApp, kind: policyKind, context: plan.context, synthesize: { synthesizedDragAt( app: activeApp, x: plan.points.x, @@ -190,13 +177,10 @@ extension RunnerTests { profile: profile, context: plan.context ) - } - if case .performed = outcome { - logSynthesizedGesturePolicyDecision(kind: policyKind, context: plan.context, fallbackAttempted: false) + }) { + case .performed(let timing): return gestureResponse(message: message, timing: timing, frame: .drag(dragFrame)) - } - if plan.context.allowsXCTestCoordinateFallback { - logSynthesizedGesturePolicyDecision(kind: policyKind, context: plan.context, fallbackAttempted: true) + case .xctestFallback(let fallbackMessage, let hint): return executeCoordinateDragFallback( activeApp: activeApp, x: plan.points.x, @@ -205,11 +189,11 @@ extension RunnerTests { y2: plan.points.y2, durationMs: durationMs, message: message, - fallback: gestureFallback(strategy: "xctest-coordinate-drag", from: outcome) + fallback: GestureFallback(strategy: "xctest-coordinate-drag", message: fallbackMessage, hint: hint) ) + case .refused(_, let refusalMessage, let hint): + return unsupportedResponse(message: refusalMessage, hint: hint) } - logSynthesizedGesturePolicyDecision(kind: policyKind, context: plan.context, fallbackAttempted: false) - return unsupportedResponse(for: outcome) #else return nil #endif @@ -223,7 +207,7 @@ extension RunnerTests { y2: Double, durationMs: Double, message: String, - fallback: GestureFallback? + fallback: GestureFallback ) -> Response { let dragPoints = keyboardAvoidingDragPoints(app: activeApp, x: x, y: y, x2: x2, y2: y2) let dragFrame = resolvedDragVisualizationFrame( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift index b6ea551520..9bc0481861 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift @@ -143,51 +143,21 @@ extension RunnerTests { ) -> SequenceStepOutcome { let x = step.x ?? 0 let y = step.y ?? 0 - // Synthesized HID tap fast path mirrors the individual `tap` command (idleTimeout:false, with - // a tapAt fallback when synthesis is unsupported), so fusing a jittered tap series does not - // change the touch mechanism for these inputs. - if step.kind == "tap", step.synthesized == true { - let policyKind = SynthesizedGesturePolicyKind.synthesizedDrag -#if os(iOS) - guard let synthesizedContext else { - let nowMs = ProcessInfo.processInfo.systemUptime * 1000 - logSynthesizedGesturePolicyDecision(kind: policyKind, context: nil, fallbackAttempted: false) - return SequenceStepOutcome( - outcome: .unsupported( - message: "synthesized coordinate tap could not resolve a finite coordinate frame", - hint: "Retry after the app is foregrounded, or use a plain screenshot to choose coordinates." - ), - gestureStartUptimeMs: nowMs, - gestureEndUptimeMs: nowMs - ) - } -#endif - let (timing, outcome) = performGesture(activeApp, idleTimeout: false) { + if let policyKind = synthesizedPolicyKind(forSequenceStep: step) { + switch performSynthesizedGesture(activeApp, kind: policyKind, context: synthesizedContext, synthesize: { synthesizedTapAt(app: activeApp, x: x, y: y, context: synthesizedContext) - } - if case .performed = outcome { - logSynthesizedGesturePolicyDecision(kind: policyKind, context: synthesizedContext, fallbackAttempted: false) - if let pauseMs = step.pauseMs, pauseMs > 0 { - sleepFor(min(max(pauseMs, 0), 10000) / 1000.0) - } - return SequenceStepOutcome( - outcome: outcome, - gestureStartUptimeMs: timing.gestureStartUptimeMs, - gestureEndUptimeMs: timing.gestureEndUptimeMs - ) - } -#if os(iOS) - guard synthesizedContext.allowsXCTestCoordinateFallback else { - logSynthesizedGesturePolicyDecision(kind: policyKind, context: synthesizedContext, fallbackAttempted: false) - return SequenceStepOutcome( - outcome: outcome, - gestureStartUptimeMs: timing.gestureStartUptimeMs, - gestureEndUptimeMs: timing.gestureEndUptimeMs + }) { + case .performed(let timing): + return finishedSequenceStep(step, timing: timing, outcome: .performed) + case .refused(let timing, let message, let hint): + return finishedSequenceStep( + step, + timing: timing, + outcome: .unsupported(message: message, hint: hint) ) + case .xctestFallback: + break } - logSynthesizedGesturePolicyDecision(kind: policyKind, context: synthesizedContext, fallbackAttempted: true) -#endif - // Synthesis unsupported (e.g. macOS) — fall through to the drag-based tapAt below. } let (timing, outcome) = performGesture(activeApp) { switch step.kind { @@ -201,6 +171,14 @@ extension RunnerTests { return tapAt(app: activeApp, x: x, y: y) } } + return finishedSequenceStep(step, timing: timing, outcome: outcome) + } + + private func finishedSequenceStep( + _ step: SequenceStep, + timing: (gestureStartUptimeMs: Double, gestureEndUptimeMs: Double), + outcome: RunnerInteractionOutcome + ) -> SequenceStepOutcome { // Sleep AFTER the step — pauseMs is the inter-step gap — but only when the step performed. // assembleSequenceExecution stops at the first unsupported outcome, so pausing after a failed // step would burn up to 10s of watchdog budget with no following step to separate from. diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift index 74723a7f44..212d136781 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift @@ -14,15 +14,12 @@ enum RunnerAccessibilityHealth: String, Equatable { enum SynthesizedKeyboardPolicy: String, Equatable, Hashable { case never - case whenAccessibilityHealthy case requiredWhenAvailable func allowsProbe(accessibilityHealth: RunnerAccessibilityHealth) -> Bool { switch self { case .never: return false - case .whenAccessibilityHealthy: - return accessibilityHealth == .healthy case .requiredWhenAvailable: return accessibilityHealth != .unavailable } @@ -63,7 +60,6 @@ struct SynthesizedCoordinateContext { /// through this same window so geometry and routing can never name different windows. let resolvedWindow: XCUIElement let keyboardPolicy: SynthesizedKeyboardPolicy - let fallbackPolicy: SynthesizedFallbackPolicy let accessibilityHealth: RunnerAccessibilityHealth func withReferenceFrame(_ frame: CGRect) -> SynthesizedCoordinateContext { @@ -71,15 +67,10 @@ struct SynthesizedCoordinateContext { referenceFrame: frame, resolvedWindow: resolvedWindow, keyboardPolicy: keyboardPolicy, - fallbackPolicy: fallbackPolicy, accessibilityHealth: accessibilityHealth ) } - var allowsXCTestCoordinateFallback: Bool { - fallbackPolicy.allowsXCTestCoordinateFallback(accessibilityHealth: accessibilityHealth) - } - var allowsKeyboardProbe: Bool { keyboardPolicy.allowsProbe(accessibilityHealth: accessibilityHealth) } @@ -115,10 +106,22 @@ func shouldProbeCoordinateTapTextInput(xCTestChannelPenalized: Bool) -> Bool { !xCTestChannelPenalized } -func sequenceHasSynthesizedCoordinateStep(_ steps: [SequenceStep]) -> Bool { - steps.contains { step in - step.synthesized == true && step.kind == "tap" - } +/// A synthesized `tap` step in a `sequence` is a standalone coordinate tap and follows its policy. +func synthesizedPolicyKind(forSequenceStep step: SequenceStep) -> SynthesizedGesturePolicyKind? { + step.synthesized == true && step.kind == "tap" ? .coordinateTap : nil +} + +/// A synthesized gesture's result for its call site, which owns the XCTest coordinate gesture. +enum SynthesizedGestureAttempt { + case performed(timing: (gestureStartUptimeMs: Double, gestureEndUptimeMs: Double)) + /// Synthesis failed and the policy allows the call site's XCTest coordinate gesture. + case xctestFallback(message: String, hint: String?) + /// Synthesis failed and the policy refuses an XCTest coordinate gesture. + case refused( + timing: (gestureStartUptimeMs: Double, gestureEndUptimeMs: Double), + message: String, + hint: String? + ) } extension RunnerTests { @@ -126,11 +129,36 @@ extension RunnerTests { steps: [SequenceStep], app: XCUIApplication ) -> SynthesizedCoordinateContext? { - guard sequenceHasSynthesizedCoordinateStep(steps) else { return nil } - return synthesizedCoordinateContext( - app: app, - policy: synthesizedGesturePolicy(.synthesizedDrag) + guard let kind = steps.lazy.compactMap(synthesizedPolicyKind(forSequenceStep:)).first else { + return nil + } + return synthesizedCoordinateContext(app: app, policy: synthesizedGesturePolicy(kind)) + } + + /// `context` is nil when no window frame resolved; `kind`'s fallback policy then reads the + /// runner's current accessibility health. + func performSynthesizedGesture( + _ app: XCUIApplication, + kind: SynthesizedGesturePolicyKind, + context: SynthesizedCoordinateContext?, + synthesize: () -> RunnerInteractionOutcome + ) -> SynthesizedGestureAttempt { + let (timing, outcome) = performGesture(app, idleTimeout: false, synthesize) + guard case .unsupported(let message, let hint) = outcome else { + logSynthesizedGesturePolicyDecision(kind: kind, context: context, fallbackAttempted: false) + return .performed(timing: timing) + } + let fallbackAllowed = synthesizedGesturePolicy(kind).fallbackPolicy.allowsXCTestCoordinateFallback( + accessibilityHealth: context?.accessibilityHealth ?? runnerAccessibilityHealth + ) + logSynthesizedGesturePolicyDecision( + kind: kind, + context: context, + fallbackAttempted: fallbackAllowed ) + return fallbackAllowed + ? .xctestFallback(message: message, hint: hint) + : .refused(timing: timing, message: message, hint: hint) } func logSynthesizedGesturePolicyDecision( @@ -162,11 +190,12 @@ extension RunnerTests { return "AGENT_DEVICE_RUNNER_SYNTHESIZED_GESTURE_POLICY kind=\(kind.rawValue)" + " context=unavailable fallbackAttempted=\(fallbackAttempted)" } + let fallbackPolicy = synthesizedGesturePolicy(kind).fallbackPolicy return "AGENT_DEVICE_RUNNER_SYNTHESIZED_GESTURE_POLICY kind=\(kind.rawValue)" + " axHealth=\(context.accessibilityHealth.rawValue) frameSource=window" + " keyboardPolicy=\(context.keyboardPolicy.rawValue)" - + " fallbackPolicy=\(context.fallbackPolicy.rawValue)" - + " fallbackAllowed=\(context.allowsXCTestCoordinateFallback)" + + " fallbackPolicy=\(fallbackPolicy.rawValue)" + + " fallbackAllowed=\(fallbackPolicy.allowsXCTestCoordinateFallback(accessibilityHealth: context.accessibilityHealth))" + " fallbackAttempted=\(fallbackAttempted)" } } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift index 1d307ddd91..369f588cf9 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift @@ -384,7 +384,6 @@ extension RunnerTests { referenceFrame: referenceFrame, resolvedWindow: window, keyboardPolicy: policy.keyboardPolicy, - fallbackPolicy: policy.fallbackPolicy, accessibilityHealth: health ) #else diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift index 59086f3bd6..0ea1c6e65e 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift @@ -270,11 +270,4 @@ extension RunnerTests { ) -> Bool { repairMode == .replacement && hasX && hasY && xCTestChannelPenalized } - - static func shouldFallbackFromSynthesizedTextEntryFocus( - _ outcome: RunnerInteractionOutcome - ) -> Bool { - if case .unsupported = outcome { return true } - return false - } } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift index 2150047869..5ec6f27826 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift @@ -31,27 +31,20 @@ extension RunnerTests { app: activeApp, policy: synthesizedGesturePolicy(policyKind) ) - let (_, outcome) = performGesture(activeApp, idleTimeout: false) { + switch performSynthesizedGesture(activeApp, kind: policyKind, context: context, synthesize: { synthesizedTapAt(app: activeApp, x: x, y: y, context: context) - } - if Self.shouldFallbackFromSynthesizedTextEntryFocus(outcome) { - logSynthesizedGesturePolicyDecision( - kind: policyKind, - context: context, - fallbackAttempted: true - ) - } else { - logSynthesizedGesturePolicyDecision( - kind: policyKind, - context: context, - fallbackAttempted: false - ) + }) { + case .performed: resolvedCoordinateContext = context resolvedCoordinateTarget = TextEntryTarget( element: nil, refreshPoint: CGPoint(x: x, y: y), prefersFocusedElement: false ) + case .xctestFallback: + break + case .refused(_, let message, let hint): + return unsupportedResponse(message: message, hint: hint) } } #else diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SequenceExecutionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SequenceExecutionTests.swift index a2e0327df6..5412382539 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SequenceExecutionTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SequenceExecutionTests.swift @@ -57,20 +57,6 @@ extension RunnerTests { XCTAssertTrue(response.error?.message.contains("at most 20") ?? false) } - func testSequenceHasSynthesizedCoordinateStep() { - XCTAssertTrue( - sequenceHasSynthesizedCoordinateStep([ - sequenceStep(kind: "tap", x: 1, y: 2, synthesized: true), - ]) - ) - XCTAssertFalse( - sequenceHasSynthesizedCoordinateStep([ - sequenceStep(kind: "tap", x: 1, y: 2), - sequenceStep(kind: "doubleTap", x: 1, y: 2, synthesized: true), - ]) - ) - } - func testAssembleSequencePreservesOrderOnSuccess() { let steps = [ sequenceStep(kind: "tap", x: 1, y: 1), @@ -151,8 +137,7 @@ extension RunnerTests { private func sequenceStep( kind: String, x: Double?, - y: Double? = nil, - synthesized: Bool? = nil + y: Double? = nil ) -> SequenceStep { SequenceStep( kind: kind, @@ -160,7 +145,7 @@ extension RunnerTests { y: y, durationMs: nil, pauseMs: nil, - synthesized: synthesized + synthesized: nil ) } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGesturePolicyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGesturePolicyTests.swift index acb3d9b032..7f1646568b 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGesturePolicyTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGesturePolicyTests.swift @@ -58,10 +58,7 @@ extension RunnerTests { /// Keyboard-policy semantics only. Which command gets which policy is the table below; a probe /// that is merely permitted still costs a live AX fetch, so the two questions stay separate. func testSynthesizedKeyboardPolicyAllowsProbeOnlyWhenAccessibilityPermitsIt() { - XCTAssertFalse( - SynthesizedKeyboardPolicy.whenAccessibilityHealthy - .allowsProbe(accessibilityHealth: .unknown) - ) + XCTAssertFalse(SynthesizedKeyboardPolicy.never.allowsProbe(accessibilityHealth: .healthy)) XCTAssertTrue( SynthesizedKeyboardPolicy.requiredWhenAvailable .allowsProbe(accessibilityHealth: .unknown) @@ -101,5 +98,83 @@ extension RunnerTests { XCTAssertFalse(shouldProbeCoordinateTapTextInput(xCTestChannelPenalized: true)) } + func testOnlySynthesizedSequenceTapStepsTakeTheCoordinateTapPolicy() { + XCTAssertEqual(synthesizedPolicyKind(forSequenceStep: sequenceStep("tap", synthesized: true)), .coordinateTap) + XCTAssertNil(synthesizedPolicyKind(forSequenceStep: sequenceStep("tap", synthesized: nil))) + XCTAssertNil(synthesizedPolicyKind(forSequenceStep: sequenceStep("doubleTap", synthesized: true))) + XCTAssertNil(synthesizedPolicyKind(forSequenceStep: sequenceStep("longPress", synthesized: true))) + } + + func testSequenceTapStepAndStandaloneTapChooseTheSameFallbackForEveryAccessibilityHealth() throws { + let sequenceKind = try XCTUnwrap( + synthesizedPolicyKind(forSequenceStep: sequenceStep("tap", synthesized: true)) + ) + for health: RunnerAccessibilityHealth in [.unknown, .healthy, .unavailable] { + runnerAccessibilityHealth = health + for context in [nil, synthesizedGestureTestContext(accessibilityHealth: health)] { + let label = "axHealth=\(health.rawValue) context=\(context == nil ? "unresolved" : "resolved")" + let standalone = synthesizedGestureRoute( + performSynthesizedGesture(app, kind: .coordinateTap, context: context, synthesize: failedSynthesis) + ) + let sequenceStep = synthesizedGestureRoute( + performSynthesizedGesture(app, kind: sequenceKind, context: context, synthesize: failedSynthesis) + ) + XCTAssertEqual(sequenceStep, standalone, label) + XCTAssertEqual(standalone, "xctestFallback", label) + } + } + } + + func testSynthesizedGestureFallbackFollowsItsKindAndTheResolvedAccessibilityHealth() { + runnerAccessibilityHealth = .healthy + let unavailable = synthesizedGestureTestContext(accessibilityHealth: .unavailable) + let unknown = synthesizedGestureTestContext(accessibilityHealth: .unknown) + let cases: [(SynthesizedGesturePolicyKind, SynthesizedCoordinateContext?, String)] = [ + (.scroll, synthesizedGestureTestContext(accessibilityHealth: .healthy), "refused"), + (.synthesizedDrag, unavailable, "refused"), + (.synthesizedDrag, unknown, "xctestFallback"), + (.coordinateTap, unavailable, "xctestFallback"), + ] + for (kind, context, expected) in cases { + let attempt = performSynthesizedGesture(app, kind: kind, context: context, synthesize: failedSynthesis) + XCTAssertEqual(synthesizedGestureRoute(attempt), expected, "kind=\(kind.rawValue)") + switch attempt { + case .xctestFallback(let message, let hint), .refused(_, let message, let hint): + XCTAssertEqual(message, "forced private synthesis failure") + XCTAssertEqual(hint, "forced hint") + case .performed: + break + } + let performed = performSynthesizedGesture(app, kind: kind, context: context) { .performed } + XCTAssertEqual(synthesizedGestureRoute(performed), "performed", "kind=\(kind.rawValue)") + } + } + + private func failedSynthesis() -> RunnerInteractionOutcome { + .unsupported(message: "forced private synthesis failure", hint: "forced hint") + } + + private func synthesizedGestureRoute(_ attempt: SynthesizedGestureAttempt) -> String { + switch attempt { + case .performed: return "performed" + case .xctestFallback: return "xctestFallback" + case .refused: return "refused" + } + } + + private func synthesizedGestureTestContext( + accessibilityHealth: RunnerAccessibilityHealth + ) -> SynthesizedCoordinateContext { + SynthesizedCoordinateContext( + referenceFrame: CGRect(x: 0, y: 0, width: 390, height: 844), + resolvedWindow: app.windows.firstMatch, + keyboardPolicy: .never, + accessibilityHealth: accessibilityHealth + ) + } + + private func sequenceStep(_ kind: String, synthesized: Bool?) -> SequenceStep { + SequenceStep(kind: kind, x: 10, y: 20, durationMs: nil, pauseMs: nil, synthesized: synthesized) + } } #endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift index 556dd1aa65..f0563c6476 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift @@ -204,15 +204,6 @@ extension RunnerTests { } #endif - func testResolvedCoordinateTextEntryFallsBackWhenSynthesizedFocusIsUnavailable() { - XCTAssertFalse(Self.shouldFallbackFromSynthesizedTextEntryFocus(.performed)) - XCTAssertTrue( - Self.shouldFallbackFromSynthesizedTextEntryFocus( - .unsupported(message: "private synthesis unavailable", hint: "use XCTest") - ) - ) - } - func testResolvedCoordinateTextEntryRouteRequiresReplacementCoordinatesAndPenalizedXCTest() { let cases: [(TextTypingRepairMode, Bool, Bool, Bool, Bool)] = [ (.replacement, true, true, false, false), diff --git a/docs/adr/0011-interaction-guarantee-contract.md b/docs/adr/0011-interaction-guarantee-contract.md index 51665d3613..48a5db131a 100644 --- a/docs/adr/0011-interaction-guarantee-contract.md +++ b/docs/adr/0011-interaction-guarantee-contract.md @@ -301,13 +301,13 @@ XCTest-coordinate fallback rules stay runner-local in `RunnerTests+SynthesizedGesturePolicy.swift`. That Swift policy is the source of truth because the current table has only three behaviors: -- coordinate synthesized tap never probes keyboards and may use the coordinate - fallback; +- coordinate synthesized tap, standalone or as a `sequence` step, never probes + keyboards and may use the coordinate fallback; - default iOS scroll probes keyboards only after AX is known healthy and must not fall back to `XCUICoordinate`; -- synthesized one-contact `gesture` plans and synthesized sequence tap steps may - still use the coordinate fallback before AX health is known, but stop using it - once a snapshot stamps AX unavailable. +- synthesized one-contact `gesture` plans may still use the coordinate fallback + before AX health is known, but stop using it once a snapshot stamps AX + unavailable. The non-obvious parts are covered by gated XCTest policy tests instead of a cross-language mirror. A future sibling registry should only be introduced once From 97ae088d630ab0dce44bff59f3696c29125ec2ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 23 Sep 2026 21:58:31 +0200 Subject: [PATCH 2/3] fix(ios-runner): route the in-app back tap through the synthesized gesture helper Pin the sequence tap step fallback at the sequence command response with a simulator test that forces private tap synthesis to fail while accessibility is unavailable. --- .github/workflows/ios.yml | 1 + .../RunnerTests+Navigation.swift | 21 ++++++++------ .../RunnerTests+CommandExecutionTests.swift | 28 +++++++++--------- .../RunnerTests+SequenceExecutionTests.swift | 29 +++++++++++++++++++ ...rTests+SynthesizedGesturePolicyTests.swift | 18 +++++------- .../0011-interaction-guarantee-contract.md | 5 ++-- 6 files changed, 66 insertions(+), 36 deletions(-) diff --git a/.github/workflows/ios.yml b/.github/workflows/ios.yml index d04fa8f140..c115474c75 100644 --- a/.github/workflows/ios.yml +++ b/.github/workflows/ios.yml @@ -158,6 +158,7 @@ jobs: -xctestrun "$XCTESTRUN_PATH" \ -destination "platform=iOS Simulator,id=${{ steps.ios-simulator.outputs.simulator-udid }}" \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSinglePointerFlingFallsBackToXCTestCoordinateDragWhenPrivateSynthesisFails \ + -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSynthesizedSequenceTapFallsBackToXCTestCoordinateTapWhenAccessibilityIsUnavailable \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRecordStartThrowsTheCaptureRefusalItReceived \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAlertDispatchResolvesItsOwnModalWithoutCoordinateTapRoutingProbe \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAlertResolutionCannotBypassRequestedDeadline \ diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift index a4df398737..da185b037e 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift @@ -159,17 +159,20 @@ extension RunnerTests { app: app, policy: synthesizedGesturePolicy(.coordinateTap) )?.withReferenceFrame(frame) - let synthesized = performGesture(app, idleTimeout: false) { + switch performSynthesizedGesture(app, kind: .coordinateTap, context: context, synthesize: { synthesizedTapAt(app: app, x: point.x, y: point.y, context: context) - } - if case .performed = synthesized.outcome { - return verifyNavigationFallbackOutcome(app: app, before: before) - } - let fallback = performGesture(app) { - tapAt(app: app, x: point.x, y: point.y) - } - if case .performed = fallback.outcome { + }) { + case .performed: return verifyNavigationFallbackOutcome(app: app, before: before) + case .refused: + return .unavailable + case .xctestFallback: + let fallback = performGesture(app) { + tapAt(app: app, x: point.x, y: point.y) + } + if case .performed = fallback.outcome { + return verifyNavigationFallbackOutcome(app: app, before: before) + } } #endif return .unavailable diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandExecutionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandExecutionTests.swift index 33d3a82a25..af9a99939e 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandExecutionTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandExecutionTests.swift @@ -10,30 +10,30 @@ private final class RunnerSynthesizedTapFailureStub: NSObject { "forced private synthesis failure" } } + +extension RunnerTests { + /// Makes private tap synthesis fail until the returned closure restores it. + func forceSynthesizedTapFailure() throws -> () -> Void { + let selector = NSSelectorFromString("synthesizeTapWithApplication:resolvedWindow:x:y:") + let synthesizedTapMethod = try XCTUnwrap(class_getClassMethod(RunnerSynthesizedGesture.self, selector)) + let failureStubMethod = try XCTUnwrap(class_getClassMethod(RunnerSynthesizedTapFailureStub.self, selector)) + let originalImplementation = method_getImplementation(synthesizedTapMethod) + method_setImplementation(synthesizedTapMethod, method_getImplementation(failureStubMethod)) + return { method_setImplementation(synthesizedTapMethod, originalImplementation) } + } +} #endif #if AGENT_DEVICE_RUNNER_UNIT_TESTS extension RunnerTests { #if os(iOS) func testSelectorTapFallsBackToXCTestCoordinateWhenPrivateSynthesisFails() throws { - let selector = NSSelectorFromString("synthesizeTapWithApplication:resolvedWindow:x:y:") - guard - let synthesizedTapMethod = class_getClassMethod(RunnerSynthesizedGesture.self, selector), - let failureStubMethod = class_getClassMethod(RunnerSynthesizedTapFailureStub.self, selector) - else { - XCTFail("unable to install synthesized tap failure stub") - return - } - let originalImplementation = method_getImplementation(synthesizedTapMethod) - method_setImplementation( - synthesizedTapMethod, - method_getImplementation(failureStubMethod) - ) + let restoreSynthesizedTap = try forceSynthesizedTapFailure() app.launch() currentApp = app runnerAccessibilityHealth = .healthy defer { - method_setImplementation(synthesizedTapMethod, originalImplementation) + restoreSynthesizedTap() invalidateCachedTarget(reason: "unit_test_cleanup") app.terminate() } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SequenceExecutionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SequenceExecutionTests.swift index 5412382539..8535c29ebb 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SequenceExecutionTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SequenceExecutionTests.swift @@ -169,3 +169,32 @@ extension RunnerTests { } } #endif + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS) +extension RunnerTests { + func testSynthesizedSequenceTapFallsBackToXCTestCoordinateTapWhenAccessibilityIsUnavailable() throws { + let restoreSynthesizedTap = try forceSynthesizedTapFailure() + app.launch() + currentApp = app + defer { + restoreSynthesizedTap() + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } + let label = app.staticTexts["Agent Device Runner"] + XCTAssertTrue(label.waitForExistence(timeout: appExistenceTimeout)) + let point = CGPoint(x: label.frame.midX, y: label.frame.midY) + runnerAccessibilityHealth = .unavailable + let command = try runnerCommandFixture( + #"{"command":"sequence","commandId":"sequence-synthesized-tap-fallback","steps":[{"kind":"tap","x":\#(point.x),"y":\#(point.y),"synthesized":true}]}"# + ) + + let response = try executeOnMainPrepared(command: command, activeApp: app) + + XCTAssertTrue(response.ok) + XCTAssertEqual(response.data?.completedSteps, 1) + XCTAssertNil(response.data?.failedStepIndex) + XCTAssertEqual(response.data?.sequenceResults?.first?.ok, true) + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGesturePolicyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGesturePolicyTests.swift index 7f1646568b..ed7d2cb4ab 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGesturePolicyTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGesturePolicyTests.swift @@ -105,22 +105,18 @@ extension RunnerTests { XCTAssertNil(synthesizedPolicyKind(forSequenceStep: sequenceStep("longPress", synthesized: true))) } - func testSequenceTapStepAndStandaloneTapChooseTheSameFallbackForEveryAccessibilityHealth() throws { - let sequenceKind = try XCTUnwrap( - synthesizedPolicyKind(forSequenceStep: sequenceStep("tap", synthesized: true)) - ) + func testFailedCoordinateTapSynthesisFallsBackToXCTestAtEveryAccessibilityHealth() { for health: RunnerAccessibilityHealth in [.unknown, .healthy, .unavailable] { runnerAccessibilityHealth = health for context in [nil, synthesizedGestureTestContext(accessibilityHealth: health)] { let label = "axHealth=\(health.rawValue) context=\(context == nil ? "unresolved" : "resolved")" - let standalone = synthesizedGestureRoute( - performSynthesizedGesture(app, kind: .coordinateTap, context: context, synthesize: failedSynthesis) - ) - let sequenceStep = synthesizedGestureRoute( - performSynthesizedGesture(app, kind: sequenceKind, context: context, synthesize: failedSynthesis) + let attempt = performSynthesizedGesture( + app, + kind: .coordinateTap, + context: context, + synthesize: failedSynthesis ) - XCTAssertEqual(sequenceStep, standalone, label) - XCTAssertEqual(standalone, "xctestFallback", label) + XCTAssertEqual(synthesizedGestureRoute(attempt), "xctestFallback", label) } } } diff --git a/docs/adr/0011-interaction-guarantee-contract.md b/docs/adr/0011-interaction-guarantee-contract.md index 48a5db131a..4f8d9da027 100644 --- a/docs/adr/0011-interaction-guarantee-contract.md +++ b/docs/adr/0011-interaction-guarantee-contract.md @@ -301,8 +301,9 @@ XCTest-coordinate fallback rules stay runner-local in `RunnerTests+SynthesizedGesturePolicy.swift`. That Swift policy is the source of truth because the current table has only three behaviors: -- coordinate synthesized tap, standalone or as a `sequence` step, never probes - keyboards and may use the coordinate fallback; +- coordinate synthesized tap, standalone, as a `sequence` step, or as the + in-app `back` top-leading tap, never probes keyboards and may use the + coordinate fallback; - default iOS scroll probes keyboards only after AX is known healthy and must not fall back to `XCUICoordinate`; - synthesized one-contact `gesture` plans may still use the coordinate fallback From 60c57c67c73c66793ab864a532d5275b8399896e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 24 Sep 2026 09:11:11 +0200 Subject: [PATCH 3/3] docs(changelog): note the synthesized sequence tap fallback --- CHANGELOG.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 63fc667515..8fa3d7b9be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,16 @@ with `observation: "unsettled"`, `wait absent` keeps polling, and the next read captures afresh. A failed read also carries `targetActivation` in `error.details`, and a failed interaction now keeps the disclosure sentences in its hint. + passed. That capture now carries `unsettledGesture`: `is`, `get`, `find`, and `wait` report it (in + `error.details` or `data`) with an appended warning, `snapshot` appends the warning, `is absent` + refuses with `observation: "unsettled"`, `wait absent` keeps polling, and the next read captures + afresh. Click, press, and fill by selector do not disclose it yet. A failed read now also carries + `targetActivation` in `error.details`, the same place as `unsettledGesture`. +- Fixed (ios): a synthesized tap step inside a runner `sequence` (for example `press x y --count N`) + now follows the standalone tap's policy instead of its own. When accessibility is unavailable or no + app window resolves, the step now falls back to an XCTest coordinate tap instead of failing the + step with `UNSUPPORTED_OPERATION`. One helper now owns the synthesize-then-fallback decision at + every synthesized tap site (#2788). - Fixed (ios): `open` on a local Simulator now waits for the launched app's discovery before it decides whether the app is observable. On a loaded host `simctl spawn launchctl list` outlasts one 1.5 s discovery wait slice, and the launch observation read that slice as an unobservable app, so