diff --git a/.github/workflows/ios.yml b/.github/workflows/ios.yml index d04fa8f140..c115474c75 100644 --- a/.github/workflows/ios.yml +++ b/.github/workflows/ios.yml @@ -158,6 +158,7 @@ jobs: -xctestrun "$XCTESTRUN_PATH" \ -destination "platform=iOS Simulator,id=${{ steps.ios-simulator.outputs.simulator-udid }}" \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSinglePointerFlingFallsBackToXCTestCoordinateDragWhenPrivateSynthesisFails \ + -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSynthesizedSequenceTapFallsBackToXCTestCoordinateTapWhenAccessibilityIsUnavailable \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRecordStartThrowsTheCaptureRefusalItReceived \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAlertDispatchResolvesItsOwnModalWithoutCoordinateTapRoutingProbe \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAlertResolutionCannotBypassRequestedDeadline \ diff --git a/CHANGELOG.md b/CHANGELOG.md index 63fc667515..8fa3d7b9be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,16 @@ with `observation: "unsettled"`, `wait absent` keeps polling, and the next read captures afresh. A failed read also carries `targetActivation` in `error.details`, and a failed interaction now keeps the disclosure sentences in its hint. + passed. That capture now carries `unsettledGesture`: `is`, `get`, `find`, and `wait` report it (in + `error.details` or `data`) with an appended warning, `snapshot` appends the warning, `is absent` + refuses with `observation: "unsettled"`, `wait absent` keeps polling, and the next read captures + afresh. Click, press, and fill by selector do not disclose it yet. A failed read now also carries + `targetActivation` in `error.details`, the same place as `unsettledGesture`. +- Fixed (ios): a synthesized tap step inside a runner `sequence` (for example `press x y --count N`) + now follows the standalone tap's policy instead of its own. When accessibility is unavailable or no + app window resolves, the step now falls back to an XCTest coordinate tap instead of failing the + step with `UNSUPPORTED_OPERATION`. One helper now owns the synthesize-then-fallback decision at + every synthesized tap site (#2788). - Fixed (ios): `open` on a local Simulator now waits for the launched app's discovery before it decides whether the app is observable. On a loaded host `simctl spawn launchctl list` outlasts one 1.5 s discovery wait slice, and the launch observation read that slice as an unobservable app, so diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index 77bdd1a17f..491b7962ad 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -78,20 +78,15 @@ extension RunnerTests { app: activeApp, policy: synthesizedGesturePolicy(policyKind) ) - let (timing, outcome) = performGesture(activeApp, idleTimeout: false) { + switch performSynthesizedGesture(activeApp, kind: policyKind, context: context, synthesize: { synthesizedTapAt( app: activeApp, x: touchPoint.x, y: touchPoint.y, context: context ) - } - if case .performed = outcome { - logSynthesizedGesturePolicyDecision( - kind: policyKind, - context: context, - fallbackAttempted: false - ) + }) { + case .performed(let timing): if isTextEntry { waitForTextEntryReadinessAfterTap(app: activeApp, element: element) } @@ -107,13 +102,12 @@ extension RunnerTests { ? match.usedNonHittableFallback : nil ) + case .xctestFallback(let message, let hint): + fallback = GestureFallback(strategy: "xctest-coordinate-tap", message: message, hint: hint) + case .refused(_, let message, let hint): + clearRememberedTextEntryTap() + return unsupportedResponse(message: message, hint: hint) } - logSynthesizedGesturePolicyDecision( - kind: policyKind, - context: context, - fallbackAttempted: true - ) - fallback = gestureFallback(strategy: "xctest-coordinate-tap", from: outcome) } let (timing, outcome) = performGesture(activeApp) { if expectedPoint != nil || match.usedNonHittableFallback { @@ -172,16 +166,18 @@ extension RunnerTests { app: activeApp, policy: synthesizedGesturePolicy(policyKind) ) - let (timing, outcome) = performGesture(activeApp, idleTimeout: false) { + switch performSynthesizedGesture(activeApp, kind: policyKind, context: context, synthesize: { synthesizedTapAt(app: activeApp, x: x, y: y, context: context) - } - if case .performed = outcome { - logSynthesizedGesturePolicyDecision(kind: policyKind, context: context, fallbackAttempted: false) + }) { + case .performed(let timing): rememberTextEntryTap(textInput) return gestureResponse(message: "tapped", timing: timing) + case .xctestFallback(let message, let hint): + fallback = GestureFallback(strategy: "xctest-coordinate-tap", message: message, hint: hint) + case .refused(_, let message, let hint): + clearRememberedTextEntryTap() + return unsupportedResponse(message: message, hint: hint) } - logSynthesizedGesturePolicyDecision(kind: policyKind, context: context, fallbackAttempted: true) - fallback = gestureFallback(strategy: "xctest-coordinate-tap", from: outcome) } let touchFrame = resolvedTouchVisualizationFrame(app: activeApp, x: x, y: y) let (timing, outcome) = performGesture(activeApp) { tapAt(app: activeApp, x: x, y: y) } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+GestureExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+GestureExecution.swift index 46d14d17f7..7959acbf94 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+GestureExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+GestureExecution.swift @@ -25,13 +25,17 @@ extension RunnerTests { case .performed: return nil case .unsupported(let message, let hint): - return Response( - ok: false, - error: ErrorPayload(code: "UNSUPPORTED_OPERATION", message: message, hint: hint) - ) + return unsupportedResponse(message: message, hint: hint) } } + func unsupportedResponse(message: String, hint: String?) -> Response { + Response( + ok: false, + error: ErrorPayload(code: "UNSUPPORTED_OPERATION", message: message, hint: hint) + ) + } + /// Optional visualization frame returned with a gesture response. enum GestureFrame { case none @@ -45,15 +49,6 @@ extension RunnerTests { let hint: String? } - func gestureFallback(strategy: String, from outcome: RunnerInteractionOutcome) -> GestureFallback? { - switch outcome { - case .performed: - return nil - case .unsupported(let message, let hint): - return GestureFallback(strategy: strategy, message: message, hint: hint) - } - } - /// Runs a gesture action with uniform timing capture. Touch gestures pass `idleTimeout: true` /// (the default) to run inside the scroll idle-timeout + quiescence-skip wrapper; synthesis diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift index a4df398737..da185b037e 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift @@ -159,17 +159,20 @@ extension RunnerTests { app: app, policy: synthesizedGesturePolicy(.coordinateTap) )?.withReferenceFrame(frame) - let synthesized = performGesture(app, idleTimeout: false) { + switch performSynthesizedGesture(app, kind: .coordinateTap, context: context, synthesize: { synthesizedTapAt(app: app, x: point.x, y: point.y, context: context) - } - if case .performed = synthesized.outcome { - return verifyNavigationFallbackOutcome(app: app, before: before) - } - let fallback = performGesture(app) { - tapAt(app: app, x: point.x, y: point.y) - } - if case .performed = fallback.outcome { + }) { + case .performed: return verifyNavigationFallbackOutcome(app: app, before: before) + case .refused: + return .unavailable + case .xctestFallback: + let fallback = performGesture(app) { + tapAt(app: app, x: point.x, y: point.y) + } + if case .performed = fallback.outcome { + return verifyNavigationFallbackOutcome(app: app, before: before) + } } #endif return .unavailable diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollDragExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollDragExecution.swift index 022a23d61c..77c896c5ba 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollDragExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollDragExecution.swift @@ -149,19 +149,6 @@ extension RunnerTests { context: context ) else { - if context?.allowsXCTestCoordinateFallback == true { - logSynthesizedGesturePolicyDecision(kind: policyKind, context: context, fallbackAttempted: true) - return executeCoordinateDragFallback( - activeApp: activeApp, - x: x, - y: y, - x2: x2, - y2: y2, - durationMs: durationMs, - message: message, - fallback: nil - ) - } logSynthesizedGesturePolicyDecision(kind: policyKind, context: context, fallbackAttempted: false) return Response( ok: false, @@ -179,7 +166,7 @@ extension RunnerTests { y2: plan.points.y2, referenceFrame: plan.referenceFrame ) - let (timing, outcome) = performGesture(activeApp, idleTimeout: false) { + switch performSynthesizedGesture(activeApp, kind: policyKind, context: plan.context, synthesize: { synthesizedDragAt( app: activeApp, x: plan.points.x, @@ -190,13 +177,10 @@ extension RunnerTests { profile: profile, context: plan.context ) - } - if case .performed = outcome { - logSynthesizedGesturePolicyDecision(kind: policyKind, context: plan.context, fallbackAttempted: false) + }) { + case .performed(let timing): return gestureResponse(message: message, timing: timing, frame: .drag(dragFrame)) - } - if plan.context.allowsXCTestCoordinateFallback { - logSynthesizedGesturePolicyDecision(kind: policyKind, context: plan.context, fallbackAttempted: true) + case .xctestFallback(let fallbackMessage, let hint): return executeCoordinateDragFallback( activeApp: activeApp, x: plan.points.x, @@ -205,11 +189,11 @@ extension RunnerTests { y2: plan.points.y2, durationMs: durationMs, message: message, - fallback: gestureFallback(strategy: "xctest-coordinate-drag", from: outcome) + fallback: GestureFallback(strategy: "xctest-coordinate-drag", message: fallbackMessage, hint: hint) ) + case .refused(_, let refusalMessage, let hint): + return unsupportedResponse(message: refusalMessage, hint: hint) } - logSynthesizedGesturePolicyDecision(kind: policyKind, context: plan.context, fallbackAttempted: false) - return unsupportedResponse(for: outcome) #else return nil #endif @@ -223,7 +207,7 @@ extension RunnerTests { y2: Double, durationMs: Double, message: String, - fallback: GestureFallback? + fallback: GestureFallback ) -> Response { let dragPoints = keyboardAvoidingDragPoints(app: activeApp, x: x, y: y, x2: x2, y2: y2) let dragFrame = resolvedDragVisualizationFrame( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift index b6ea551520..9bc0481861 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift @@ -143,51 +143,21 @@ extension RunnerTests { ) -> SequenceStepOutcome { let x = step.x ?? 0 let y = step.y ?? 0 - // Synthesized HID tap fast path mirrors the individual `tap` command (idleTimeout:false, with - // a tapAt fallback when synthesis is unsupported), so fusing a jittered tap series does not - // change the touch mechanism for these inputs. - if step.kind == "tap", step.synthesized == true { - let policyKind = SynthesizedGesturePolicyKind.synthesizedDrag -#if os(iOS) - guard let synthesizedContext else { - let nowMs = ProcessInfo.processInfo.systemUptime * 1000 - logSynthesizedGesturePolicyDecision(kind: policyKind, context: nil, fallbackAttempted: false) - return SequenceStepOutcome( - outcome: .unsupported( - message: "synthesized coordinate tap could not resolve a finite coordinate frame", - hint: "Retry after the app is foregrounded, or use a plain screenshot to choose coordinates." - ), - gestureStartUptimeMs: nowMs, - gestureEndUptimeMs: nowMs - ) - } -#endif - let (timing, outcome) = performGesture(activeApp, idleTimeout: false) { + if let policyKind = synthesizedPolicyKind(forSequenceStep: step) { + switch performSynthesizedGesture(activeApp, kind: policyKind, context: synthesizedContext, synthesize: { synthesizedTapAt(app: activeApp, x: x, y: y, context: synthesizedContext) - } - if case .performed = outcome { - logSynthesizedGesturePolicyDecision(kind: policyKind, context: synthesizedContext, fallbackAttempted: false) - if let pauseMs = step.pauseMs, pauseMs > 0 { - sleepFor(min(max(pauseMs, 0), 10000) / 1000.0) - } - return SequenceStepOutcome( - outcome: outcome, - gestureStartUptimeMs: timing.gestureStartUptimeMs, - gestureEndUptimeMs: timing.gestureEndUptimeMs - ) - } -#if os(iOS) - guard synthesizedContext.allowsXCTestCoordinateFallback else { - logSynthesizedGesturePolicyDecision(kind: policyKind, context: synthesizedContext, fallbackAttempted: false) - return SequenceStepOutcome( - outcome: outcome, - gestureStartUptimeMs: timing.gestureStartUptimeMs, - gestureEndUptimeMs: timing.gestureEndUptimeMs + }) { + case .performed(let timing): + return finishedSequenceStep(step, timing: timing, outcome: .performed) + case .refused(let timing, let message, let hint): + return finishedSequenceStep( + step, + timing: timing, + outcome: .unsupported(message: message, hint: hint) ) + case .xctestFallback: + break } - logSynthesizedGesturePolicyDecision(kind: policyKind, context: synthesizedContext, fallbackAttempted: true) -#endif - // Synthesis unsupported (e.g. macOS) — fall through to the drag-based tapAt below. } let (timing, outcome) = performGesture(activeApp) { switch step.kind { @@ -201,6 +171,14 @@ extension RunnerTests { return tapAt(app: activeApp, x: x, y: y) } } + return finishedSequenceStep(step, timing: timing, outcome: outcome) + } + + private func finishedSequenceStep( + _ step: SequenceStep, + timing: (gestureStartUptimeMs: Double, gestureEndUptimeMs: Double), + outcome: RunnerInteractionOutcome + ) -> SequenceStepOutcome { // Sleep AFTER the step — pauseMs is the inter-step gap — but only when the step performed. // assembleSequenceExecution stops at the first unsupported outcome, so pausing after a failed // step would burn up to 10s of watchdog budget with no following step to separate from. diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift index 74723a7f44..212d136781 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift @@ -14,15 +14,12 @@ enum RunnerAccessibilityHealth: String, Equatable { enum SynthesizedKeyboardPolicy: String, Equatable, Hashable { case never - case whenAccessibilityHealthy case requiredWhenAvailable func allowsProbe(accessibilityHealth: RunnerAccessibilityHealth) -> Bool { switch self { case .never: return false - case .whenAccessibilityHealthy: - return accessibilityHealth == .healthy case .requiredWhenAvailable: return accessibilityHealth != .unavailable } @@ -63,7 +60,6 @@ struct SynthesizedCoordinateContext { /// through this same window so geometry and routing can never name different windows. let resolvedWindow: XCUIElement let keyboardPolicy: SynthesizedKeyboardPolicy - let fallbackPolicy: SynthesizedFallbackPolicy let accessibilityHealth: RunnerAccessibilityHealth func withReferenceFrame(_ frame: CGRect) -> SynthesizedCoordinateContext { @@ -71,15 +67,10 @@ struct SynthesizedCoordinateContext { referenceFrame: frame, resolvedWindow: resolvedWindow, keyboardPolicy: keyboardPolicy, - fallbackPolicy: fallbackPolicy, accessibilityHealth: accessibilityHealth ) } - var allowsXCTestCoordinateFallback: Bool { - fallbackPolicy.allowsXCTestCoordinateFallback(accessibilityHealth: accessibilityHealth) - } - var allowsKeyboardProbe: Bool { keyboardPolicy.allowsProbe(accessibilityHealth: accessibilityHealth) } @@ -115,10 +106,22 @@ func shouldProbeCoordinateTapTextInput(xCTestChannelPenalized: Bool) -> Bool { !xCTestChannelPenalized } -func sequenceHasSynthesizedCoordinateStep(_ steps: [SequenceStep]) -> Bool { - steps.contains { step in - step.synthesized == true && step.kind == "tap" - } +/// A synthesized `tap` step in a `sequence` is a standalone coordinate tap and follows its policy. +func synthesizedPolicyKind(forSequenceStep step: SequenceStep) -> SynthesizedGesturePolicyKind? { + step.synthesized == true && step.kind == "tap" ? .coordinateTap : nil +} + +/// A synthesized gesture's result for its call site, which owns the XCTest coordinate gesture. +enum SynthesizedGestureAttempt { + case performed(timing: (gestureStartUptimeMs: Double, gestureEndUptimeMs: Double)) + /// Synthesis failed and the policy allows the call site's XCTest coordinate gesture. + case xctestFallback(message: String, hint: String?) + /// Synthesis failed and the policy refuses an XCTest coordinate gesture. + case refused( + timing: (gestureStartUptimeMs: Double, gestureEndUptimeMs: Double), + message: String, + hint: String? + ) } extension RunnerTests { @@ -126,11 +129,36 @@ extension RunnerTests { steps: [SequenceStep], app: XCUIApplication ) -> SynthesizedCoordinateContext? { - guard sequenceHasSynthesizedCoordinateStep(steps) else { return nil } - return synthesizedCoordinateContext( - app: app, - policy: synthesizedGesturePolicy(.synthesizedDrag) + guard let kind = steps.lazy.compactMap(synthesizedPolicyKind(forSequenceStep:)).first else { + return nil + } + return synthesizedCoordinateContext(app: app, policy: synthesizedGesturePolicy(kind)) + } + + /// `context` is nil when no window frame resolved; `kind`'s fallback policy then reads the + /// runner's current accessibility health. + func performSynthesizedGesture( + _ app: XCUIApplication, + kind: SynthesizedGesturePolicyKind, + context: SynthesizedCoordinateContext?, + synthesize: () -> RunnerInteractionOutcome + ) -> SynthesizedGestureAttempt { + let (timing, outcome) = performGesture(app, idleTimeout: false, synthesize) + guard case .unsupported(let message, let hint) = outcome else { + logSynthesizedGesturePolicyDecision(kind: kind, context: context, fallbackAttempted: false) + return .performed(timing: timing) + } + let fallbackAllowed = synthesizedGesturePolicy(kind).fallbackPolicy.allowsXCTestCoordinateFallback( + accessibilityHealth: context?.accessibilityHealth ?? runnerAccessibilityHealth + ) + logSynthesizedGesturePolicyDecision( + kind: kind, + context: context, + fallbackAttempted: fallbackAllowed ) + return fallbackAllowed + ? .xctestFallback(message: message, hint: hint) + : .refused(timing: timing, message: message, hint: hint) } func logSynthesizedGesturePolicyDecision( @@ -162,11 +190,12 @@ extension RunnerTests { return "AGENT_DEVICE_RUNNER_SYNTHESIZED_GESTURE_POLICY kind=\(kind.rawValue)" + " context=unavailable fallbackAttempted=\(fallbackAttempted)" } + let fallbackPolicy = synthesizedGesturePolicy(kind).fallbackPolicy return "AGENT_DEVICE_RUNNER_SYNTHESIZED_GESTURE_POLICY kind=\(kind.rawValue)" + " axHealth=\(context.accessibilityHealth.rawValue) frameSource=window" + " keyboardPolicy=\(context.keyboardPolicy.rawValue)" - + " fallbackPolicy=\(context.fallbackPolicy.rawValue)" - + " fallbackAllowed=\(context.allowsXCTestCoordinateFallback)" + + " fallbackPolicy=\(fallbackPolicy.rawValue)" + + " fallbackAllowed=\(fallbackPolicy.allowsXCTestCoordinateFallback(accessibilityHealth: context.accessibilityHealth))" + " fallbackAttempted=\(fallbackAttempted)" } } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift index 1d307ddd91..369f588cf9 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift @@ -384,7 +384,6 @@ extension RunnerTests { referenceFrame: referenceFrame, resolvedWindow: window, keyboardPolicy: policy.keyboardPolicy, - fallbackPolicy: policy.fallbackPolicy, accessibilityHealth: health ) #else diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift index 59086f3bd6..0ea1c6e65e 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift @@ -270,11 +270,4 @@ extension RunnerTests { ) -> Bool { repairMode == .replacement && hasX && hasY && xCTestChannelPenalized } - - static func shouldFallbackFromSynthesizedTextEntryFocus( - _ outcome: RunnerInteractionOutcome - ) -> Bool { - if case .unsupported = outcome { return true } - return false - } } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift index 2150047869..5ec6f27826 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift @@ -31,27 +31,20 @@ extension RunnerTests { app: activeApp, policy: synthesizedGesturePolicy(policyKind) ) - let (_, outcome) = performGesture(activeApp, idleTimeout: false) { + switch performSynthesizedGesture(activeApp, kind: policyKind, context: context, synthesize: { synthesizedTapAt(app: activeApp, x: x, y: y, context: context) - } - if Self.shouldFallbackFromSynthesizedTextEntryFocus(outcome) { - logSynthesizedGesturePolicyDecision( - kind: policyKind, - context: context, - fallbackAttempted: true - ) - } else { - logSynthesizedGesturePolicyDecision( - kind: policyKind, - context: context, - fallbackAttempted: false - ) + }) { + case .performed: resolvedCoordinateContext = context resolvedCoordinateTarget = TextEntryTarget( element: nil, refreshPoint: CGPoint(x: x, y: y), prefersFocusedElement: false ) + case .xctestFallback: + break + case .refused(_, let message, let hint): + return unsupportedResponse(message: message, hint: hint) } } #else diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandExecutionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandExecutionTests.swift index 33d3a82a25..af9a99939e 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandExecutionTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandExecutionTests.swift @@ -10,30 +10,30 @@ private final class RunnerSynthesizedTapFailureStub: NSObject { "forced private synthesis failure" } } + +extension RunnerTests { + /// Makes private tap synthesis fail until the returned closure restores it. + func forceSynthesizedTapFailure() throws -> () -> Void { + let selector = NSSelectorFromString("synthesizeTapWithApplication:resolvedWindow:x:y:") + let synthesizedTapMethod = try XCTUnwrap(class_getClassMethod(RunnerSynthesizedGesture.self, selector)) + let failureStubMethod = try XCTUnwrap(class_getClassMethod(RunnerSynthesizedTapFailureStub.self, selector)) + let originalImplementation = method_getImplementation(synthesizedTapMethod) + method_setImplementation(synthesizedTapMethod, method_getImplementation(failureStubMethod)) + return { method_setImplementation(synthesizedTapMethod, originalImplementation) } + } +} #endif #if AGENT_DEVICE_RUNNER_UNIT_TESTS extension RunnerTests { #if os(iOS) func testSelectorTapFallsBackToXCTestCoordinateWhenPrivateSynthesisFails() throws { - let selector = NSSelectorFromString("synthesizeTapWithApplication:resolvedWindow:x:y:") - guard - let synthesizedTapMethod = class_getClassMethod(RunnerSynthesizedGesture.self, selector), - let failureStubMethod = class_getClassMethod(RunnerSynthesizedTapFailureStub.self, selector) - else { - XCTFail("unable to install synthesized tap failure stub") - return - } - let originalImplementation = method_getImplementation(synthesizedTapMethod) - method_setImplementation( - synthesizedTapMethod, - method_getImplementation(failureStubMethod) - ) + let restoreSynthesizedTap = try forceSynthesizedTapFailure() app.launch() currentApp = app runnerAccessibilityHealth = .healthy defer { - method_setImplementation(synthesizedTapMethod, originalImplementation) + restoreSynthesizedTap() invalidateCachedTarget(reason: "unit_test_cleanup") app.terminate() } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SequenceExecutionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SequenceExecutionTests.swift index a2e0327df6..8535c29ebb 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SequenceExecutionTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SequenceExecutionTests.swift @@ -57,20 +57,6 @@ extension RunnerTests { XCTAssertTrue(response.error?.message.contains("at most 20") ?? false) } - func testSequenceHasSynthesizedCoordinateStep() { - XCTAssertTrue( - sequenceHasSynthesizedCoordinateStep([ - sequenceStep(kind: "tap", x: 1, y: 2, synthesized: true), - ]) - ) - XCTAssertFalse( - sequenceHasSynthesizedCoordinateStep([ - sequenceStep(kind: "tap", x: 1, y: 2), - sequenceStep(kind: "doubleTap", x: 1, y: 2, synthesized: true), - ]) - ) - } - func testAssembleSequencePreservesOrderOnSuccess() { let steps = [ sequenceStep(kind: "tap", x: 1, y: 1), @@ -151,8 +137,7 @@ extension RunnerTests { private func sequenceStep( kind: String, x: Double?, - y: Double? = nil, - synthesized: Bool? = nil + y: Double? = nil ) -> SequenceStep { SequenceStep( kind: kind, @@ -160,7 +145,7 @@ extension RunnerTests { y: y, durationMs: nil, pauseMs: nil, - synthesized: synthesized + synthesized: nil ) } @@ -184,3 +169,32 @@ extension RunnerTests { } } #endif + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS) +extension RunnerTests { + func testSynthesizedSequenceTapFallsBackToXCTestCoordinateTapWhenAccessibilityIsUnavailable() throws { + let restoreSynthesizedTap = try forceSynthesizedTapFailure() + app.launch() + currentApp = app + defer { + restoreSynthesizedTap() + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } + let label = app.staticTexts["Agent Device Runner"] + XCTAssertTrue(label.waitForExistence(timeout: appExistenceTimeout)) + let point = CGPoint(x: label.frame.midX, y: label.frame.midY) + runnerAccessibilityHealth = .unavailable + let command = try runnerCommandFixture( + #"{"command":"sequence","commandId":"sequence-synthesized-tap-fallback","steps":[{"kind":"tap","x":\#(point.x),"y":\#(point.y),"synthesized":true}]}"# + ) + + let response = try executeOnMainPrepared(command: command, activeApp: app) + + XCTAssertTrue(response.ok) + XCTAssertEqual(response.data?.completedSteps, 1) + XCTAssertNil(response.data?.failedStepIndex) + XCTAssertEqual(response.data?.sequenceResults?.first?.ok, true) + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGesturePolicyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGesturePolicyTests.swift index acb3d9b032..ed7d2cb4ab 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGesturePolicyTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGesturePolicyTests.swift @@ -58,10 +58,7 @@ extension RunnerTests { /// Keyboard-policy semantics only. Which command gets which policy is the table below; a probe /// that is merely permitted still costs a live AX fetch, so the two questions stay separate. func testSynthesizedKeyboardPolicyAllowsProbeOnlyWhenAccessibilityPermitsIt() { - XCTAssertFalse( - SynthesizedKeyboardPolicy.whenAccessibilityHealthy - .allowsProbe(accessibilityHealth: .unknown) - ) + XCTAssertFalse(SynthesizedKeyboardPolicy.never.allowsProbe(accessibilityHealth: .healthy)) XCTAssertTrue( SynthesizedKeyboardPolicy.requiredWhenAvailable .allowsProbe(accessibilityHealth: .unknown) @@ -101,5 +98,79 @@ extension RunnerTests { XCTAssertFalse(shouldProbeCoordinateTapTextInput(xCTestChannelPenalized: true)) } + func testOnlySynthesizedSequenceTapStepsTakeTheCoordinateTapPolicy() { + XCTAssertEqual(synthesizedPolicyKind(forSequenceStep: sequenceStep("tap", synthesized: true)), .coordinateTap) + XCTAssertNil(synthesizedPolicyKind(forSequenceStep: sequenceStep("tap", synthesized: nil))) + XCTAssertNil(synthesizedPolicyKind(forSequenceStep: sequenceStep("doubleTap", synthesized: true))) + XCTAssertNil(synthesizedPolicyKind(forSequenceStep: sequenceStep("longPress", synthesized: true))) + } + + func testFailedCoordinateTapSynthesisFallsBackToXCTestAtEveryAccessibilityHealth() { + for health: RunnerAccessibilityHealth in [.unknown, .healthy, .unavailable] { + runnerAccessibilityHealth = health + for context in [nil, synthesizedGestureTestContext(accessibilityHealth: health)] { + let label = "axHealth=\(health.rawValue) context=\(context == nil ? "unresolved" : "resolved")" + let attempt = performSynthesizedGesture( + app, + kind: .coordinateTap, + context: context, + synthesize: failedSynthesis + ) + XCTAssertEqual(synthesizedGestureRoute(attempt), "xctestFallback", label) + } + } + } + + func testSynthesizedGestureFallbackFollowsItsKindAndTheResolvedAccessibilityHealth() { + runnerAccessibilityHealth = .healthy + let unavailable = synthesizedGestureTestContext(accessibilityHealth: .unavailable) + let unknown = synthesizedGestureTestContext(accessibilityHealth: .unknown) + let cases: [(SynthesizedGesturePolicyKind, SynthesizedCoordinateContext?, String)] = [ + (.scroll, synthesizedGestureTestContext(accessibilityHealth: .healthy), "refused"), + (.synthesizedDrag, unavailable, "refused"), + (.synthesizedDrag, unknown, "xctestFallback"), + (.coordinateTap, unavailable, "xctestFallback"), + ] + for (kind, context, expected) in cases { + let attempt = performSynthesizedGesture(app, kind: kind, context: context, synthesize: failedSynthesis) + XCTAssertEqual(synthesizedGestureRoute(attempt), expected, "kind=\(kind.rawValue)") + switch attempt { + case .xctestFallback(let message, let hint), .refused(_, let message, let hint): + XCTAssertEqual(message, "forced private synthesis failure") + XCTAssertEqual(hint, "forced hint") + case .performed: + break + } + let performed = performSynthesizedGesture(app, kind: kind, context: context) { .performed } + XCTAssertEqual(synthesizedGestureRoute(performed), "performed", "kind=\(kind.rawValue)") + } + } + + private func failedSynthesis() -> RunnerInteractionOutcome { + .unsupported(message: "forced private synthesis failure", hint: "forced hint") + } + + private func synthesizedGestureRoute(_ attempt: SynthesizedGestureAttempt) -> String { + switch attempt { + case .performed: return "performed" + case .xctestFallback: return "xctestFallback" + case .refused: return "refused" + } + } + + private func synthesizedGestureTestContext( + accessibilityHealth: RunnerAccessibilityHealth + ) -> SynthesizedCoordinateContext { + SynthesizedCoordinateContext( + referenceFrame: CGRect(x: 0, y: 0, width: 390, height: 844), + resolvedWindow: app.windows.firstMatch, + keyboardPolicy: .never, + accessibilityHealth: accessibilityHealth + ) + } + + private func sequenceStep(_ kind: String, synthesized: Bool?) -> SequenceStep { + SequenceStep(kind: kind, x: 10, y: 20, durationMs: nil, pauseMs: nil, synthesized: synthesized) + } } #endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift index 556dd1aa65..f0563c6476 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift @@ -204,15 +204,6 @@ extension RunnerTests { } #endif - func testResolvedCoordinateTextEntryFallsBackWhenSynthesizedFocusIsUnavailable() { - XCTAssertFalse(Self.shouldFallbackFromSynthesizedTextEntryFocus(.performed)) - XCTAssertTrue( - Self.shouldFallbackFromSynthesizedTextEntryFocus( - .unsupported(message: "private synthesis unavailable", hint: "use XCTest") - ) - ) - } - func testResolvedCoordinateTextEntryRouteRequiresReplacementCoordinatesAndPenalizedXCTest() { let cases: [(TextTypingRepairMode, Bool, Bool, Bool, Bool)] = [ (.replacement, true, true, false, false), diff --git a/docs/adr/0011-interaction-guarantee-contract.md b/docs/adr/0011-interaction-guarantee-contract.md index 51665d3613..4f8d9da027 100644 --- a/docs/adr/0011-interaction-guarantee-contract.md +++ b/docs/adr/0011-interaction-guarantee-contract.md @@ -301,13 +301,14 @@ XCTest-coordinate fallback rules stay runner-local in `RunnerTests+SynthesizedGesturePolicy.swift`. That Swift policy is the source of truth because the current table has only three behaviors: -- coordinate synthesized tap never probes keyboards and may use the coordinate - fallback; +- coordinate synthesized tap, standalone, as a `sequence` step, or as the + in-app `back` top-leading tap, never probes keyboards and may use the + coordinate fallback; - default iOS scroll probes keyboards only after AX is known healthy and must not fall back to `XCUICoordinate`; -- synthesized one-contact `gesture` plans and synthesized sequence tap steps may - still use the coordinate fallback before AX health is known, but stop using it - once a snapshot stamps AX unavailable. +- synthesized one-contact `gesture` plans may still use the coordinate fallback + before AX health is known, but stop using it once a snapshot stamps AX + unavailable. The non-obvious parts are covered by gated XCTest policy tests instead of a cross-language mirror. A future sibling registry should only be introduced once