From 8226713f04bbbafb26838246eec3c27fbf0290d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 23 Sep 2026 15:57:16 +0200 Subject: [PATCH 1/8] fix(ios): take snapshot target identity on main and hop plan state writes The capture plan runs on the command queue but read currentBundleId, currentAppProcessIdentifier and the penalty warm-up exemption, and wrote runnerAccessibilityHealth and invalidated the cached target directly. Snapshot preparation now takes a SnapshotCaptureTarget on main (app, bundle id, pid, consumed warm-up exemption) and the plan, the modal probe penalty, the XCTest penalty recorder and the private AX tier read only that copy. Health writes and the fail-closed invalidation go through applyMainOwnedSnapshotState, the abandoned-work-guarded main hop that invalidateCachedTargetAfterSnapshotFailure now also uses. Fixes #2781 --- .../RunnerTests+AXSnapshotFallback.swift | 28 +++++--- .../RunnerTests+CommandDispatch.swift | 3 +- .../RunnerTests+Snapshot.swift | 35 ++++++---- .../RunnerTests+SnapshotCapturePlan.swift | 25 ++++--- .../RunnerTests+SnapshotCaptureTarget.swift | 66 +++++++++++++++++++ .../RunnerTests+SnapshotExecution.swift | 47 ++++++------- .../RunnerTests+SnapshotTiming.swift | 3 +- .../RunnerTests.swift | 3 + ...nnerTests+AXRecoveryConformanceTests.swift | 6 +- .../RunnerTests+AXSnapshotFallbackTests.swift | 13 ++-- ...ts+SnapshotCapturePlanOccupancyTests.swift | 5 +- ...nnerTests+SnapshotCaptureTargetTests.swift | 49 ++++++++++++++ 12 files changed, 210 insertions(+), 73 deletions(-) create mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCaptureTarget.swift create mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCaptureTargetTests.swift diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift index ed79919b17..13244aeaf2 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift @@ -110,14 +110,16 @@ extension RunnerTests { /// remembered depth deliberately does NOT refresh the TTL, so expiry re-probes the full /// requested depth once per window instead of capping this screen class forever. func recordPrivateAXAcceptedDepth( + bundleId: String?, + processIdentifier: Int?, exactDepthRequested: Bool, effectiveDepth: Int, attemptDepths: [Int] ) { guard !exactDepthRequested, effectiveDepth != attemptDepths.first else { return } rememberPrivateAXAcceptedDepth( - bundleId: currentBundleId, - processIdentifier: currentAppProcessIdentifier, + bundleId: bundleId, + processIdentifier: processIdentifier, depth: effectiveDepth ) } @@ -160,11 +162,12 @@ extension RunnerTests { } func privateAXSnapshotAcquisition( - app: XCUIApplication, + target: SnapshotCaptureTarget, hint: CaptureHint, deadline: Date = .distantFuture ) -> SnapshotAcquisition? { #if os(iOS) && targetEnvironment(simulator) + let app = target.app let requestedDepth = hint.rawTraversalDepth ?? 64 // An explicit --depth request is honored as asked: no accepted-depth // memory, no frontier extension past it. @@ -173,8 +176,8 @@ extension RunnerTests { exactDepthRequested ? nil : rememberedPrivateAXAcceptedDepth( - bundleId: currentBundleId, - processIdentifier: currentAppProcessIdentifier + bundleId: target.bundleId, + processIdentifier: target.processIdentifier ) let attemptDepths = Self.privateAXAttemptDepths( requestedDepth: requestedDepth, @@ -202,6 +205,8 @@ extension RunnerTests { return nil } recordPrivateAXAcceptedDepth( + bundleId: target.bundleId, + processIdentifier: target.processIdentifier, exactDepthRequested: exactDepthRequested, effectiveDepth: effectiveDepth, attemptDepths: attemptDepths @@ -212,7 +217,11 @@ extension RunnerTests { } let rootFrame = privateAXRect(root["frame"]) - let geometry = privateAXSnapshotGeometry(app: app, rootFrame: rootFrame) + let geometry = privateAXSnapshotGeometry( + app: app, + bundleId: target.bundleId, + rootFrame: rootFrame + ) let viewport = geometry.viewport let nodes = privateAXAcquisition( rawRoot: root, @@ -262,8 +271,8 @@ extension RunnerTests { /// grinding on this screen class. Under penalty it reliably burns its full timeout and /// falls back anyway (~1s added to every private AX capture on the Bluesky bench feed), /// so honor the penalty here the same way capture plans do. - func shouldReadPrivateAXViewportViaXCTest() -> Bool { - !hasAbandonedMainThreadWork() && !isSnapshotXCTestChannelPenalized(bundleId: currentBundleId) + func shouldReadPrivateAXViewportViaXCTest(bundleId: String?) -> Bool { + !hasAbandonedMainThreadWork() && !isSnapshotXCTestChannelPenalized(bundleId: bundleId) } /// The geometry this tier may anchor a rotation on. The bridge's own root frame is one more @@ -272,10 +281,11 @@ extension RunnerTests { /// consumers already treat as geometry they cannot measure (#2612). private func privateAXSnapshotGeometry( app: XCUIApplication, + bundleId: String?, rootFrame: CGRect ) -> (viewport: CGRect, interfaceOrientation: Int) { let fallback = rootFrame.isEmpty ? CGRect.infinite : rootFrame - guard shouldReadPrivateAXViewportViaXCTest() else { + guard shouldReadPrivateAXViewportViaXCTest(bundleId: bundleId) else { return (fallback, RunnerInterfaceOrientation.unknown) } do { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandDispatch.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandDispatch.swift index a791984dc1..da4792455d 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandDispatch.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandDispatch.swift @@ -580,7 +580,8 @@ extension RunnerTests { #endif let probeDeadline = Date().addingTimeInterval(systemModalProbeBudget) return boundedBlockingSystemAlertSnapshot( - deadline: probeDeadline + deadline: probeDeadline, + penaltyBundleId: currentBundleId ) != nil #else return false diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift index 14fe138387..ba412066aa 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift @@ -145,14 +145,17 @@ extension RunnerTests { // `boundedBlockingSystemAlertSnapshot`'s probe closure (see `systemModalProbeOverrideForTesting` // in RunnerTests.swift), so reverting this entry point to bypass the bounded probe fails the // regression test. - func snapshotFast(app: XCUIApplication, options: PresentationOptions) throws -> DataPayload { + func snapshotFast(target: SnapshotCaptureTarget, options: PresentationOptions) throws -> DataPayload { let deadline = Date().addingTimeInterval(Self.snapshotPlanBudget) - if let blocking = boundedBlockingSystemAlertSnapshot(deadline: deadline) { + if let blocking = boundedBlockingSystemAlertSnapshot( + deadline: deadline, + penaltyBundleId: target.bundleId + ) { return blocking } return try runSnapshotCapturePlan( Self.regularVisiblePlan, - app: app, + target: target, options: options, terminal: .sparseWithFatalOnAXFailure, deadline: deadline @@ -267,14 +270,17 @@ extension RunnerTests { } // See `snapshotFast` above: the single production entry point, no unit-test overload. - func snapshotRaw(app: XCUIApplication, options: PresentationOptions) throws -> DataPayload { + func snapshotRaw(target: SnapshotCaptureTarget, options: PresentationOptions) throws -> DataPayload { let deadline = Date().addingTimeInterval(Self.snapshotPlanBudget) - if let blocking = boundedBlockingSystemAlertSnapshot(deadline: deadline) { + if let blocking = boundedBlockingSystemAlertSnapshot( + deadline: deadline, + penaltyBundleId: target.bundleId + ) { return blocking } return try runSnapshotCapturePlan( Self.rawDiagnosticPlan, - app: app, + target: target, options: options, terminal: .throwOnAXFailure, deadline: deadline @@ -283,8 +289,12 @@ extension RunnerTests { /// Runs the pre-plan SpringBoard system-modal probe as a bounded capture tier sharing the plan /// deadline, so a slow alert enumeration cannot bypass the snapshot timeout and stall (#1244). - func boundedBlockingSystemAlertSnapshot(deadline: Date) -> DataPayload? { - boundedBlockingSystemAlertSnapshotBody(deadline: deadline) { probeDeadline in + /// An abandoned probe penalizes the XCTest channel for `penaltyBundleId`. + func boundedBlockingSystemAlertSnapshot(deadline: Date, penaltyBundleId: String?) -> DataPayload? { + boundedBlockingSystemAlertSnapshotBody( + deadline: deadline, + penaltyBundleId: penaltyBundleId + ) { probeDeadline in #if AGENT_DEVICE_RUNNER_UNIT_TESTS if let override = self.systemModalProbeOverrideForTesting { return override(probeDeadline) @@ -301,6 +311,7 @@ extension RunnerTests { /// production runs and what the unit tests exercise. private func boundedBlockingSystemAlertSnapshotBody( deadline: Date, + penaltyBundleId: String?, probe: @escaping (Date) -> DataPayload? ) -> DataPayload? { #if os(macOS) @@ -329,7 +340,7 @@ extension RunnerTests { }, onAbandoned: { self.penalizeSnapshotXCTestChannel( - bundleId: self.currentBundleId, + bundleId: penaltyBundleId, reason: "system_modal_probe_timeout" ) } @@ -503,8 +514,10 @@ extension RunnerTests { func snapshotAccessibilityUnavailable(failure: SnapshotCaptureFailure) -> DataPayload { NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_AX_UNAVAILABLE=%@", failure.message) - runnerAccessibilityHealth = .unavailable - invalidateCachedTarget(reason: Self.axSnapshotUnavailableReason) + applyMainOwnedSnapshotState("ax_unavailable_invalidation") { + self.runnerAccessibilityHealth = .unavailable + self.invalidateCachedTarget(reason: Self.axSnapshotUnavailableReason) + } // This is a planned terminal result, so it carries the structured verdict like every other // planned snapshot — downstream sparse handling keys off the verdict, not node shapes. return sparseTruncatedSnapshotPayload( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift index b6e9946957..27e73a3908 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift @@ -142,6 +142,7 @@ extension RunnerTests { return penalized == bundleId } + /// Main thread only, through `takeSnapshotCaptureTarget`. func consumeSnapshotXCTestPenaltyWarmupExemption() -> Bool { let pending = snapshotXCTestPenaltyWarmupExemptionPending snapshotXCTestPenaltyWarmupExemptionPending = false @@ -260,7 +261,7 @@ extension RunnerTests { func runSnapshotCapturePlan( _ plan: [SnapshotBackendKind], - app: XCUIApplication, + target: SnapshotCaptureTarget, options: PresentationOptions, terminal: SnapshotCaptureTerminalPolicy, deadline: Date? = nil @@ -270,7 +271,6 @@ extension RunnerTests { var axFailure: SnapshotCaptureFailure? // A caller may share the pre-plan system-modal probe's deadline; otherwise own the full budget (#1244). let deadline = deadline ?? Date().addingTimeInterval(Self.snapshotPlanBudget) - let suppressXCTestPenalty = consumeSnapshotXCTestPenaltyWarmupExemption() // Reorder is iOS-only because hostile screens can make XCTest tree/query work grind while // the app remains visually responsive. Simulators can avoid that channel through private AX; @@ -281,7 +281,7 @@ extension RunnerTests { var xCTestChannelPenalized = false var xCTestChannelPenalizedByBreaker = false #if os(iOS) - xCTestChannelPenalizedByBreaker = isSnapshotXCTestChannelPenalized(bundleId: currentBundleId) + xCTestChannelPenalizedByBreaker = isSnapshotXCTestChannelPenalized(bundleId: target.bundleId) xCTestChannelPenalized = Self.snapshotXCTestChannelTreatedAsPenalized( penalized: xCTestChannelPenalizedByBreaker, preferredBackend: options.preferredBackend @@ -305,9 +305,9 @@ extension RunnerTests { case .normal: break case .deferredToIndependentBackend: - NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_XCTEST_CHANNEL_DEFERRED bundle=%@", currentBundleId ?? "") + NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_XCTEST_CHANNEL_DEFERRED bundle=%@", target.bundleId ?? "") case .boundedXCTestProbe: - NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_XCTEST_CHANNEL_PROBE_BOUNDED bundle=%@", currentBundleId ?? "") + NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_XCTEST_CHANNEL_PROBE_BOUNDED bundle=%@", target.bundleId ?? "") } for kind in effectivePlan { @@ -332,7 +332,7 @@ extension RunnerTests { } let attempt = try captureWithBackend( kind, - app: app, + target: target, options: options, deadline: deadline, treeCaptureSliceBudgetOverride: effective.treeCaptureSliceBudgetOverride @@ -340,7 +340,8 @@ extension RunnerTests { recordXCTestSnapshotBackendAttemptIfNeeded( kind, attempt: attempt, - penaltySuppressed: suppressXCTestPenalty + bundleId: target.bundleId, + penaltySuppressed: target.xCTestPenaltyWarmupExempt ) if case let .failed(failure, phase: _) = attempt.outcome { if Self.isAxSnapshotFailure(failure) { axFailure = failure } @@ -412,11 +413,12 @@ extension RunnerTests { private func captureWithBackend( _ kind: SnapshotBackendKind, - app: XCUIApplication, + target: SnapshotCaptureTarget, options: PresentationOptions, deadline: Date, treeCaptureSliceBudgetOverride: TimeInterval? ) throws -> SnapshotBackendAttempt { + let app = target.app let hint = SnapshotPresentation.captureHint(for: options) var timer = SnapshotPhaseTimer() let acquisition: SnapshotAcquisition? @@ -461,7 +463,7 @@ extension RunnerTests { } case .privateAX: return self.privateAXSnapshotAcquisition( - app: app, + target: target, hint: hint, deadline: deadline ) @@ -635,7 +637,10 @@ extension RunnerTests { state: String, reason: (reason: String, code: String)? ) -> DataPayload { - runnerAccessibilityHealth = reason?.code == "ax-rejected" ? .unavailable : .healthy + let health: RunnerAccessibilityHealth = reason?.code == "ax-rejected" ? .unavailable : .healthy + applyMainOwnedSnapshotState("accessibility_health") { + self.runnerAccessibilityHealth = health + } let payload = capture.payload let quality = SnapshotQuality( state: state, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCaptureTarget.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCaptureTarget.swift new file mode 100644 index 0000000000..88a176e41d --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCaptureTarget.swift @@ -0,0 +1,66 @@ +import XCTest + +// MARK: - Snapshot capture target (#2781) +// +// Target identity (`currentApp`, `currentBundleId`, `currentAppProcessIdentifier`), the penalty +// warm-up exemption, and `runnerAccessibilityHealth` are owned by main-thread lifecycle code. A +// capture plan runs on the command queue, so it reads the identity from a `SnapshotCaptureTarget` +// taken on main while the command is prepared, and writes health or invalidates the target only +// through `applyMainOwnedSnapshotState`. + +/// The target one capture plan reads, taken once on the main thread. +struct SnapshotCaptureTarget { + let app: XCUIApplication + let bundleId: String? + let processIdentifier: Int? + /// The first capture of a fresh target process does not penalize the XCTest channel for a slow + /// tier; the pending exemption is consumed when the target is taken. + let xCTestPenaltyWarmupExempt: Bool +} + +/// What snapshot command preparation hands the off-main capture. +enum SnapshotCommandPreparation { + case response(Response) + case capture(SnapshotCaptureTarget, systemSurface: SystemSurfaceHost?) +} + +extension RunnerTests { + /// Main thread only: reads the lifecycle-owned target identity and consumes the warm-up exemption. + func takeSnapshotCaptureTarget(app: XCUIApplication) -> SnapshotCaptureTarget { + SnapshotCaptureTarget( + app: app, + bundleId: currentBundleId, + processIdentifier: currentAppProcessIdentifier, + xCTestPenaltyWarmupExempt: consumeSnapshotXCTestPenaltyWarmupExemption() + ) + } + + /// Runs `write` against main-owned runner state for a capture that may be on the command queue. + /// Abandoned work ahead of the hop cannot be cancelled, so behind it the write queues without + /// waiting: the capture answers now and the next command still observes the write. + func applyMainOwnedSnapshotState(_ operation: String, _ write: @escaping () -> Void) { + if Thread.isMainThread { + write() + return + } + guard !hasAbandonedMainThreadWork() else { + NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_STATE_DEFERRED_XCTEST_OCCUPIED operation=%@", operation) + DispatchQueue.main.async(execute: write) + return + } + do { + try runMainThreadWork( + operation, + timeout: 1, + timeoutError: mainThreadExecutionTimeoutError, + write + ) + } catch { + NSLog( + "AGENT_DEVICE_RUNNER_SNAPSHOT_STATE_FAILED operation=%@ error=%@", + operation, + String(describing: error) + ) + } + } +} diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotExecution.swift index d17d79eeba..ec0b2c8faa 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotExecution.swift @@ -9,21 +9,29 @@ extension RunnerTests { } private func executeSnapshotDispatchedOnce(command: Command) throws -> Response { - let preparation = try runMainThreadWork( + let preparation: SnapshotCommandPreparation = try runMainThreadWork( "command_preparation", timeout: mainThreadExecutionTimeout, timeoutError: mainThreadExecutionTimeoutError - ) { - try self.prepareActiveCommandContextSafely(command: command, routeToSpringboard: false) + ) { () -> SnapshotCommandPreparation in + switch try self.prepareActiveCommandContextSafely(command: command, routeToSpringboard: false) { + case .response(let response): + return .response(response) + case .context(let context): + return .capture( + self.takeSnapshotCaptureTarget(app: context.app), + systemSurface: context.systemSurface + ) + } } switch preparation { case .response(let response): return response - case .context(let context): + case .capture(let target, let systemSurface): return try executeSnapshotPrepared( command: command, - activeApp: context.app, - systemSurface: context.systemSurface + target: target, + systemSurface: systemSurface ) } } @@ -76,16 +84,16 @@ extension RunnerTests { private func executeSnapshotPrepared( command: Command, - activeApp: XCUIApplication, + target: SnapshotCaptureTarget, systemSurface: SystemSurfaceHost? ) throws -> Response { let options = Self.presentationOptions(from: command) do { var payload: DataPayload if options.raw { - payload = try snapshotRaw(app: activeApp, options: options) + payload = try snapshotRaw(target: target, options: options) } else { - payload = try snapshotFast(app: activeApp, options: options) + payload = try snapshotFast(target: target, options: options) } if let systemSurface { payload.systemSurface = SystemSurfaceProvenancePayload( @@ -127,25 +135,8 @@ extension RunnerTests { } func invalidateCachedTargetAfterSnapshotFailure() { - // Abandoned work ahead of this hop cannot be cancelled: queue the drop behind it without - // waiting, so the failed capture answers now and the next command still finds the target gone. - guard !hasAbandonedMainThreadWork() else { - NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_INVALIDATION_DEFERRED_XCTEST_OCCUPIED") - DispatchQueue.main.async { - self.invalidateCachedTarget(reason: "ax_snapshot_failure") - } - return - } - do { - try runMainThreadWork( - "target_invalidation", - timeout: 1, - timeoutError: mainThreadExecutionTimeoutError - ) { - self.invalidateCachedTarget(reason: "ax_snapshot_failure") - } - } catch { - NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_INVALIDATION_FAILED=%@", String(describing: error)) + applyMainOwnedSnapshotState("target_invalidation") { + self.invalidateCachedTarget(reason: "ax_snapshot_failure") } } } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotTiming.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotTiming.swift index 288e4cca5a..853abfa45a 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotTiming.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotTiming.swift @@ -91,6 +91,7 @@ extension RunnerTests { func recordXCTestSnapshotBackendAttemptIfNeeded( _ kind: SnapshotBackendKind, attempt: SnapshotBackendAttempt, + bundleId: String?, penaltySuppressed: Bool ) { guard !penaltySuppressed else { return } @@ -102,7 +103,7 @@ extension RunnerTests { ) else { return } penalizeSnapshotXCTestChannel( - bundleId: currentBundleId, + bundleId: bundleId, reason: reason ) } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift index 0785df2b96..ffda22b982 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift @@ -52,6 +52,9 @@ final class RunnerTests: XCTestCase { let commandExecutionQueue = DispatchQueue(label: "agent-device.runner.commands") let app = XCUIApplication() lazy var springboard = XCUIApplication(bundleIdentifier: Self.springboardBundleId) + // Main-thread owned, like `runnerAccessibilityHealth` and the warm-up exemption below: an off-main + // capture plan reads them only through its `SnapshotCaptureTarget` and writes them only through + // `applyMainOwnedSnapshotState`. var currentApp: XCUIApplication? var currentBundleId: String? var currentAppProcessIdentifier: Int? diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AXRecoveryConformanceTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AXRecoveryConformanceTests.swift index 6e6364c310..cde158367b 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AXRecoveryConformanceTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AXRecoveryConformanceTests.swift @@ -426,8 +426,6 @@ extension RunnerTests { XCTAssertFalse(fixture.hintCases.isEmpty) defer { clearPrivateAXAcceptedDepth(reason: "test-cleanup") - currentBundleId = nil - currentAppProcessIdentifier = nil } for hintCase in fixture.hintCases { clearPrivateAXAcceptedDepth(reason: "fixture-case") @@ -440,8 +438,6 @@ extension RunnerTests { let target = try XCTUnwrap(step.target, name) let processIdentifier = try XCTUnwrap( Int(target.generation.filter(\.isNumber)), "\(name): generation must end in digits") - currentBundleId = target.id - currentAppProcessIdentifier = processIdentifier let explicit = step.explicitDepth ?? false let remembered = explicit @@ -459,6 +455,8 @@ extension RunnerTests { XCTAssertEqual(capture.truncated, outcome.complete == false, "\(name): bounded") // The production path records only after a successful ladder, exactly like this. recordPrivateAXAcceptedDepth( + bundleId: target.id, + processIdentifier: processIdentifier, exactDepthRequested: explicit, effectiveDepth: capture.effectiveDepth, attemptDepths: capture.attemptDepths) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AXSnapshotFallbackTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AXSnapshotFallbackTests.swift index cbcad722a9..82f6bec10b 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AXSnapshotFallbackTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AXSnapshotFallbackTests.swift @@ -125,23 +125,22 @@ extension RunnerTests { func testViewportReadSkippedWhileXCTestChannelPenalized() { // Pins the viewport fast path (#1587 review): every penalized private AX capture used to burn // the full 1s main-thread timeout on a doomed viewport read before falling back. - currentBundleId = "xyz.blueskyweb.app" + let bundleId = "xyz.blueskyweb.app" defer { - currentBundleId = nil clearSnapshotXCTestChannelPenalty(reason: "test-cleanup") abandonedMainThreadWorkCount = 0 } - XCTAssertTrue(shouldReadPrivateAXViewportViaXCTest()) + XCTAssertTrue(shouldReadPrivateAXViewportViaXCTest(bundleId: bundleId)) - penalizeSnapshotXCTestChannel(bundleId: "xyz.blueskyweb.app", reason: "test") - XCTAssertFalse(shouldReadPrivateAXViewportViaXCTest()) + penalizeSnapshotXCTestChannel(bundleId: bundleId, reason: "test") + XCTAssertFalse(shouldReadPrivateAXViewportViaXCTest(bundleId: bundleId)) clearSnapshotXCTestChannelPenalty(reason: "test") - XCTAssertTrue(shouldReadPrivateAXViewportViaXCTest()) + XCTAssertTrue(shouldReadPrivateAXViewportViaXCTest(bundleId: bundleId)) abandonedMainThreadWorkCount = 1 - XCTAssertFalse(shouldReadPrivateAXViewportViaXCTest()) + XCTAssertFalse(shouldReadPrivateAXViewportViaXCTest(bundleId: bundleId)) } /// The wire field must reach both capture options AND the backend pin: custom diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanOccupancyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanOccupancyTests.swift index c4c92d80c7..33eb57bfbe 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanOccupancyTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanOccupancyTests.swift @@ -61,6 +61,7 @@ extension RunnerTests { currentApp = app currentBundleId = "com.callstack.agentdevice.runner.tree-capture-test" snapshotXCTestPenaltyWarmupExemptionPending = true + let captureTarget = takeSnapshotCaptureTarget(app: app) RunnerBlockingSnapshotGate.release = DispatchSemaphore(value: 0) let originalImplementation = method_getImplementation(snapshotMethod) method_setImplementation(snapshotMethod, method_getImplementation(stubMethod)) @@ -85,7 +86,7 @@ extension RunnerTests { do { box.payload = try self.runSnapshotCapturePlan( Self.regularVisiblePlan, - app: self.app, + target: captureTarget, options: PresentationOptions(interactiveOnly: false, depth: nil, scope: nil, raw: false), terminal: .sparseWithFatalOnAXFailure, deadline: Date().addingTimeInterval(12) @@ -96,7 +97,7 @@ extension RunnerTests { self.mainThreadWorkLock.lock() box.abandonedAfterPlan = self.abandonedMainThreadWorkCount self.mainThreadWorkLock.unlock() - box.penalizedAfterPlan = self.isSnapshotXCTestChannelPenalized(bundleId: self.currentBundleId) + box.penalizedAfterPlan = self.isSnapshotXCTestChannelPenalized(bundleId: captureTarget.bundleId) RunnerBlockingSnapshotGate.release.signal() planned.fulfill() } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCaptureTargetTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCaptureTargetTests.swift new file mode 100644 index 0000000000..958c6f48d0 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCaptureTargetTests.swift @@ -0,0 +1,49 @@ +import XCTest + +extension RunnerTests { +#if AGENT_DEVICE_RUNNER_UNIT_TESTS + func testSnapshotCaptureTargetKeepsPreparedIdentityAndConsumesWarmupExemptionOnce() { + currentApp = app + currentBundleId = "com.example.prepared" + currentAppProcessIdentifier = 42 + snapshotXCTestPenaltyWarmupExemptionPending = true + defer { invalidateCachedTarget(reason: "unit_test_cleanup") } + + let target = takeSnapshotCaptureTarget(app: app) + currentBundleId = "com.example.replaced" + currentAppProcessIdentifier = 43 + + XCTAssertTrue(target.app === app) + XCTAssertEqual(target.bundleId, "com.example.prepared") + XCTAssertEqual(target.processIdentifier, 42) + XCTAssertTrue(target.xCTestPenaltyWarmupExempt) + XCTAssertFalse( + snapshotXCTestPenaltyWarmupExemptionPending, + "taking the target on main consumes the exemption, so the plan never touches the flag" + ) + XCTAssertFalse(takeSnapshotCaptureTarget(app: app).xCTestPenaltyWarmupExempt) + } + + func testMainOwnedSnapshotStateWriteRunsOnMainBeforeReturningWhenMainIsFree() { + final class ResultBox { + var ranOnMainThread: Bool? + var appliedBeforeReturn: Bool? + } + let box = ResultBox() + let finished = expectation(description: "off-main write returned") + + DispatchQueue(label: "agent-device.runner.tests.main-owned-snapshot-state").async { + self.applyMainOwnedSnapshotState("unit_test") { + box.ranOnMainThread = Thread.isMainThread + } + box.appliedBeforeReturn = box.ranOnMainThread != nil + finished.fulfill() + } + + wait(for: [finished], timeout: 3) + XCTAssertEqual(box.ranOnMainThread, true) + XCTAssertEqual(box.appliedBeforeReturn, true) + XCTAssertFalse(hasAbandonedMainThreadWork()) + } +#endif +} From fe3c29268b2a90e178f8394a10c42fb22192623c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 23 Sep 2026 15:57:51 +0200 Subject: [PATCH 2/8] fix(ios): bound the query sweep by its tier slice deadline A non-interactive query sweep used Date.distantFuture as its own deadline, so only the up-to-20 s plan deadline bounded it while the caller abandoned the tier after its 1 s slice. XCTest queries cannot be cancelled, so the sweep kept the main thread busy and later commands got RUNNER_BUSY. The caller now derives one slice deadline (querySweepSliceDeadline) for both its wait and the sweep, for interactive and non-interactive requests alike. Neither the element queries nor the per-element reads start with less than flatInteractiveQueryBudget left before that deadline (querySweepCanStartQuery). Fixes #2783 --- .../RunnerTests+Snapshot.swift | 24 ++++++++++----- .../RunnerTests+SnapshotAcquisition.swift | 29 +++++++++++++------ .../RunnerTests+SnapshotCapturePlan.swift | 5 ++-- 3 files changed, 39 insertions(+), 19 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift index ba412066aa..ff538bb5bd 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift @@ -132,6 +132,20 @@ extension RunnerTests { ] static let flatInteractiveFallbackBudget: TimeInterval = 1.0 + /// The least slice time a sweep query may start with. XCTest cannot cancel a query, so one that + /// starts later outlives the slice its caller waits for and holds the main thread (#2783). + static let flatInteractiveQueryBudget: TimeInterval = 0.1 + + /// The deadline the query-sweep tier's caller waits for: one slice, clamped to the plan deadline. + /// Interactive and non-interactive requests share it, since the caller discards a later result. + static func querySweepSliceDeadline(startedAt: Date, planDeadline: Date) -> Date { + min(startedAt.addingTimeInterval(flatInteractiveFallbackBudget), planDeadline) + } + + /// Whether a sweep query started at `now` can still finish before the slice `deadline`. + static func querySweepCanStartQuery(deadline: Date, now: Date) -> Bool { + deadline.timeIntervalSince(now) >= flatInteractiveQueryBudget + } /// What one capture may spend reading the keyboard band before it gives up on the fact and lets the /// tap guard fall back to the tree rule. The scroll path pays this query per gesture and stays well @@ -424,7 +438,7 @@ extension RunnerTests { func querySweepSnapshotAcquisition( app: XCUIApplication, hint: CaptureHint, - planDeadline: Date = .distantFuture + sliceDeadline deadline: Date ) -> SnapshotAcquisition { var nodes: [RawAXNode] = [ interactiveRootNode(rect: .zero) @@ -440,19 +454,13 @@ extension RunnerTests { ) } - // Bounded by both its own sweep budget and the umbrella capture-plan deadline, so a - // chained recovery tier can never push the plan past the main-thread watchdog (#1105). - let sweepDeadline = hint.interactiveOnly - ? Date().addingTimeInterval(Self.flatInteractiveFallbackBudget) - : Date.distantFuture - let deadline = min(sweepDeadline, planDeadline) let viewport = safeSnapshotViewport(app: app) var seen = Set() var candidates: [RawAXNode] = [] let flatElements = flatInteractiveElements(app: app, deadline: deadline) var truncated = flatElements.truncated for element in flatElements.elements { - if Date() >= deadline { + if !Self.querySweepCanStartQuery(deadline: deadline, now: Date()) { NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_FLAT_FALLBACK_DEADLINE") truncated = true break diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift index 2c1d834786..102fbc1727 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift @@ -487,23 +487,34 @@ extension RunnerTests { app.images ] - var elements: [XCUIElement] = [] - var truncated = false + return Self.runFlatInteractiveQueries(queries, deadline: deadline) { query in + self.snapshotElementsQuery { + query.allElementsBoundByIndex + } + } + } + + /// Runs sweep queries in order until one reports AX unavailable, or until the next one could not + /// finish before `deadline` (`querySweepCanStartQuery`). + static func runFlatInteractiveQueries( + _ queries: [Query], + deadline: Date, + now: () -> Date = { Date() }, + run: (Query) -> (elements: [Element], axUnavailable: Bool) + ) -> (elements: [Element], truncated: Bool) { + var elements: [Element] = [] for query in queries { - if Date() >= deadline { + if !querySweepCanStartQuery(deadline: deadline, now: now()) { NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_FLAT_FALLBACK_DEADLINE") - truncated = true - break - } - let result = snapshotElementsQuery { - query.allElementsBoundByIndex + return (elements, true) } + let result = run(query) elements.append(contentsOf: result.elements) if result.axUnavailable { break } } - return (elements, truncated) + return (elements, false) } func snapshotElementsQuery( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift index 27e73a3908..e254e66396 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift @@ -450,15 +450,16 @@ extension RunnerTests { : try self.recursiveTreeSnapshotAcquisition(context: context, hint: hint) } case .querySweep: + let sliceDeadline = Self.querySweepSliceDeadline(startedAt: Date(), planDeadline: deadline) return try self.runMainThreadWork( "query_sweep", - timeout: min(Self.flatInteractiveFallbackBudget, max(0.1, deadline.timeIntervalSinceNow)), + timeout: max(0.1, sliceDeadline.timeIntervalSinceNow), timeoutError: self.snapshotMainThreadTimeoutError("running query-sweep snapshot") ) { self.querySweepSnapshotAcquisition( app: app, hint: hint, - planDeadline: deadline + sliceDeadline: sliceDeadline ) } case .privateAX: From 4ecc934bfe373f26228ff032577a5497a7055583 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 23 Sep 2026 16:15:55 +0200 Subject: [PATCH 3/8] fix(ios): spend the penalty warm-up exemption only in a capture plan that runs Taking the snapshot target on main consumed the exemption during command preparation, so a snapshot answered by the blocking system-modal probe, or an abandoned preparation that ran late, spent it without running a plan. The exemption now lives in a lock-owned type that lifecycle code arms on main and runSnapshotCapturePlan consumes at its start. --- .github/workflows/ios.yml | 1 + .../RunnerTests+Lifecycle.swift | 4 +- .../RunnerTests+SnapshotCapturePlan.swift | 10 +--- .../RunnerTests+SnapshotCaptureTarget.swift | 18 +++---- .../RunnerTests+SnapshotTiming.swift | 29 +++++++++++ .../RunnerTests.swift | 7 ++- .../RunnerTests+CommandDispatchTests.swift | 4 +- .../RunnerTests+LifecycleCacheTests.swift | 14 +++--- ...ts+SnapshotCapturePlanOccupancyTests.swift | 2 +- ...nnerTests+SnapshotCaptureTargetTests.swift | 49 ++++++++++++++++--- 10 files changed, 97 insertions(+), 41 deletions(-) diff --git a/.github/workflows/ios.yml b/.github/workflows/ios.yml index 4bc494a504..667641262d 100644 --- a/.github/workflows/ios.yml +++ b/.github/workflows/ios.yml @@ -256,6 +256,7 @@ jobs: -only-testing:AgentDeviceRunnerUITests/RunnerTests/testActionNamesAreCappedPerElementAndReported \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testHungCustomActionReadIsContainedAndRecovers \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAbandonedTreeCaptureSkipsQuerySweepAndHonorsWarmupExemption \ + -only-testing:AgentDeviceRunnerUITests/RunnerTests/testBlockingModalSnapshotLeavesWarmupExemptionForTheFirstCapturePlan \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testPrivateAXPinnedRegularDepthReachesAcquisitionAndPresentation 2>&1 | tee /tmp/agent-device-runner-regressions.log node --input-type=module -e ' import { readFileSync } from "node:fs"; diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift index 0314cbab9b..8d378a59e5 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift @@ -202,7 +202,7 @@ extension RunnerTests { /// markers. Every site that binds, rebinds, or drops the target runs this. func resetTargetBoundState() { clearRememberedTextEntryTap() - snapshotXCTestPenaltyWarmupExemptionPending = false + snapshotXCTestPenaltyWarmupExemption.isPending = false lastLoggedFastAppGuardLine = nil lastLoggedGesturePolicyLines.removeAll() } @@ -247,7 +247,7 @@ extension RunnerTests { resetTargetBoundState() clearSnapshotXCTestChannelPenalty(reason: "target_process_changed") clearPrivateAXAcceptedDepth(reason: "target_process_changed") - snapshotXCTestPenaltyWarmupExemptionPending = true + snapshotXCTestPenaltyWarmupExemption.isPending = true beginFirstInteractionStabilization() } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift index e254e66396..df91c3a58e 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift @@ -142,13 +142,6 @@ extension RunnerTests { return penalized == bundleId } - /// Main thread only, through `takeSnapshotCaptureTarget`. - func consumeSnapshotXCTestPenaltyWarmupExemption() -> Bool { - let pending = snapshotXCTestPenaltyWarmupExemptionPending - snapshotXCTestPenaltyWarmupExemptionPending = false - return pending - } - /// The pre-seeded first-failure a penalized plan stamps into its verdict. The deferred case /// uses the dedicated 'deferred' code: the breaker pre-selected the backend, nothing new /// degraded on this capture, and the daemon keys warning suppression and the settle budget @@ -271,6 +264,7 @@ extension RunnerTests { var axFailure: SnapshotCaptureFailure? // A caller may share the pre-plan system-modal probe's deadline; otherwise own the full budget (#1244). let deadline = deadline ?? Date().addingTimeInterval(Self.snapshotPlanBudget) + let suppressXCTestPenalty = snapshotXCTestPenaltyWarmupExemption.consume() // Reorder is iOS-only because hostile screens can make XCTest tree/query work grind while // the app remains visually responsive. Simulators can avoid that channel through private AX; @@ -341,7 +335,7 @@ extension RunnerTests { kind, attempt: attempt, bundleId: target.bundleId, - penaltySuppressed: target.xCTestPenaltyWarmupExempt + penaltySuppressed: suppressXCTestPenalty ) if case let .failed(failure, phase: _) = attempt.outcome { if Self.isAxSnapshotFailure(failure) { axFailure = failure } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCaptureTarget.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCaptureTarget.swift index 88a176e41d..ea8e06ecde 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCaptureTarget.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCaptureTarget.swift @@ -2,20 +2,17 @@ import XCTest // MARK: - Snapshot capture target (#2781) // -// Target identity (`currentApp`, `currentBundleId`, `currentAppProcessIdentifier`), the penalty -// warm-up exemption, and `runnerAccessibilityHealth` are owned by main-thread lifecycle code. A -// capture plan runs on the command queue, so it reads the identity from a `SnapshotCaptureTarget` -// taken on main while the command is prepared, and writes health or invalidates the target only -// through `applyMainOwnedSnapshotState`. +// Target identity (`currentApp`, `currentBundleId`, `currentAppProcessIdentifier`) and +// `runnerAccessibilityHealth` are owned by main-thread lifecycle code. A capture plan runs on the +// command queue, so it reads the identity from a `SnapshotCaptureTarget` taken on main while the +// command is prepared, and writes health or invalidates the target only through +// `applyMainOwnedSnapshotState`. /// The target one capture plan reads, taken once on the main thread. struct SnapshotCaptureTarget { let app: XCUIApplication let bundleId: String? let processIdentifier: Int? - /// The first capture of a fresh target process does not penalize the XCTest channel for a slow - /// tier; the pending exemption is consumed when the target is taken. - let xCTestPenaltyWarmupExempt: Bool } /// What snapshot command preparation hands the off-main capture. @@ -25,13 +22,12 @@ enum SnapshotCommandPreparation { } extension RunnerTests { - /// Main thread only: reads the lifecycle-owned target identity and consumes the warm-up exemption. + /// Main thread only: reads the lifecycle-owned target identity. func takeSnapshotCaptureTarget(app: XCUIApplication) -> SnapshotCaptureTarget { SnapshotCaptureTarget( app: app, bundleId: currentBundleId, - processIdentifier: currentAppProcessIdentifier, - xCTestPenaltyWarmupExempt: consumeSnapshotXCTestPenaltyWarmupExemption() + processIdentifier: currentAppProcessIdentifier ) } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotTiming.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotTiming.swift index 853abfa45a..ae2dedc21d 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotTiming.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotTiming.swift @@ -57,6 +57,35 @@ struct SnapshotPhaseTimer { } } +/// Keeps the first capture plan that runs against a fresh target process from penalizing the XCTest +/// channel for a slow tier. Lifecycle code arms and disarms it on main; the capture plan consumes it +/// on the command queue, so a snapshot that returns before running a plan leaves it pending. +final class SnapshotXCTestPenaltyWarmupExemption { + private let lock = NSLock() + private var pending = false + + var isPending: Bool { + get { + lock.lock() + defer { lock.unlock() } + return pending + } + set { + lock.lock() + pending = newValue + lock.unlock() + } + } + + func consume() -> Bool { + lock.lock() + defer { lock.unlock() } + let wasPending = pending + pending = false + return wasPending + } +} + extension RunnerTests { struct SnapshotBackendAttempt { enum Outcome { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift index ffda22b982..5574a47797 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift @@ -52,9 +52,8 @@ final class RunnerTests: XCTestCase { let commandExecutionQueue = DispatchQueue(label: "agent-device.runner.commands") let app = XCUIApplication() lazy var springboard = XCUIApplication(bundleIdentifier: Self.springboardBundleId) - // Main-thread owned, like `runnerAccessibilityHealth` and the warm-up exemption below: an off-main - // capture plan reads them only through its `SnapshotCaptureTarget` and writes them only through - // `applyMainOwnedSnapshotState`. + // Main-thread owned, like `runnerAccessibilityHealth`: an off-main capture plan reads them only + // through its `SnapshotCaptureTarget` and writes them only through `applyMainOwnedSnapshotState`. var currentApp: XCUIApplication? var currentBundleId: String? var currentAppProcessIdentifier: Int? @@ -117,7 +116,7 @@ final class RunnerTests: XCTestCase { var snapshotXCTestChannelPenaltyBundleId: String? var snapshotXCTestChannelPenaltyUntil = Date.distantPast let snapshotXCTestChannelPenaltyDuration: TimeInterval = 120 - var snapshotXCTestPenaltyWarmupExemptionPending = false + let snapshotXCTestPenaltyWarmupExemption = SnapshotXCTestPenaltyWarmupExemption() // Sticky per-bundle hint for the private AX depth ladder: deep RN screens reject the default // depth with kAXErrorIllegalArgument on EVERY capture, so once a shallower rung is accepted // later captures start there instead of re-paying the rejected deep request (~300ms per diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandDispatchTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandDispatchTests.swift index dee860248e..6e75fcec2d 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandDispatchTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandDispatchTests.swift @@ -71,7 +71,7 @@ extension RunnerTests { currentApp = app currentBundleId = "com.example.stale-target" currentAppProcessIdentifier = 42 - snapshotXCTestPenaltyWarmupExemptionPending = true + snapshotXCTestPenaltyWarmupExemption.isPending = true defer { invalidateCachedTarget(reason: "unit_test_cleanup") app.terminate() @@ -85,7 +85,7 @@ extension RunnerTests { XCTAssertNil(currentApp) XCTAssertNil(currentBundleId) XCTAssertNil(currentAppProcessIdentifier) - XCTAssertFalse(snapshotXCTestPenaltyWarmupExemptionPending) + XCTAssertFalse(snapshotXCTestPenaltyWarmupExemption.isPending) } func testSkipAppActivationPreflightIncludesForegroundCachedCoordinateOnlyTaps() throws { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LifecycleCacheTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LifecycleCacheTests.swift index 1cf54a6b29..3c1aa5a21c 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LifecycleCacheTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LifecycleCacheTests.swift @@ -101,10 +101,10 @@ extension RunnerTests { } func testSnapshotPenaltyWarmupExemptionIsConsumedOnce() { - snapshotXCTestPenaltyWarmupExemptionPending = true + snapshotXCTestPenaltyWarmupExemption.isPending = true - XCTAssertTrue(consumeSnapshotXCTestPenaltyWarmupExemption()) - XCTAssertFalse(consumeSnapshotXCTestPenaltyWarmupExemption()) + XCTAssertTrue(snapshotXCTestPenaltyWarmupExemption.consume()) + XCTAssertFalse(snapshotXCTestPenaltyWarmupExemption.consume()) } func testSnapshotPenaltyCanBeClearedAcrossTargetProcessReplacement() { @@ -120,14 +120,14 @@ extension RunnerTests { currentApp = app currentBundleId = "com.example.app" currentAppProcessIdentifier = 42 - snapshotXCTestPenaltyWarmupExemptionPending = true + snapshotXCTestPenaltyWarmupExemption.isPending = true invalidateCachedTarget(reason: "unit_test") XCTAssertNil(currentApp) XCTAssertNil(currentBundleId) XCTAssertNil(currentAppProcessIdentifier) - XCTAssertFalse(snapshotXCTestPenaltyWarmupExemptionPending) + XCTAssertFalse(snapshotXCTestPenaltyWarmupExemption.isPending) } func testTextEntryTapWitnessIsBoundToTargetIdentity() { @@ -146,7 +146,7 @@ extension RunnerTests { currentApp = app currentBundleId = "com.example.app" currentAppProcessIdentifier = 42 - snapshotXCTestPenaltyWarmupExemptionPending = true + snapshotXCTestPenaltyWarmupExemption.isPending = true firstInteractionReadyUptime = nil penalizeSnapshotXCTestChannel(bundleId: "com.example.app", reason: "test") XCTAssertTrue(isSnapshotXCTestChannelPenalized(bundleId: "com.example.app")) @@ -157,7 +157,7 @@ extension RunnerTests { XCTAssertNil(currentApp) XCTAssertNil(currentBundleId) XCTAssertNil(currentAppProcessIdentifier) - XCTAssertFalse(snapshotXCTestPenaltyWarmupExemptionPending) + XCTAssertFalse(snapshotXCTestPenaltyWarmupExemption.isPending) XCTAssertFalse(isSnapshotXCTestChannelPenalized(bundleId: "com.example.app")) XCTAssertNotNil(firstInteractionReadyUptime) } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanOccupancyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanOccupancyTests.swift index 33eb57bfbe..fc05b0f8c3 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanOccupancyTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanOccupancyTests.swift @@ -60,7 +60,7 @@ extension RunnerTests { _ = capturedInterfaceOrientation(app: app) currentApp = app currentBundleId = "com.callstack.agentdevice.runner.tree-capture-test" - snapshotXCTestPenaltyWarmupExemptionPending = true + snapshotXCTestPenaltyWarmupExemption.isPending = true let captureTarget = takeSnapshotCaptureTarget(app: app) RunnerBlockingSnapshotGate.release = DispatchSemaphore(value: 0) let originalImplementation = method_getImplementation(snapshotMethod) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCaptureTargetTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCaptureTargetTests.swift index 958c6f48d0..3d4a0a7233 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCaptureTargetTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCaptureTargetTests.swift @@ -1,12 +1,13 @@ import XCTest +import AgentDeviceSnapshotPresentation extension RunnerTests { #if AGENT_DEVICE_RUNNER_UNIT_TESTS - func testSnapshotCaptureTargetKeepsPreparedIdentityAndConsumesWarmupExemptionOnce() { + func testSnapshotCaptureTargetKeepsPreparedIdentityAndLeavesWarmupExemptionPending() { currentApp = app currentBundleId = "com.example.prepared" currentAppProcessIdentifier = 42 - snapshotXCTestPenaltyWarmupExemptionPending = true + snapshotXCTestPenaltyWarmupExemption.isPending = true defer { invalidateCachedTarget(reason: "unit_test_cleanup") } let target = takeSnapshotCaptureTarget(app: app) @@ -16,13 +17,49 @@ extension RunnerTests { XCTAssertTrue(target.app === app) XCTAssertEqual(target.bundleId, "com.example.prepared") XCTAssertEqual(target.processIdentifier, 42) - XCTAssertTrue(target.xCTestPenaltyWarmupExempt) + XCTAssertTrue( + snapshotXCTestPenaltyWarmupExemption.isPending, + "only a capture plan that runs may spend the exemption" + ) + } + +#if os(iOS) + func testBlockingModalSnapshotLeavesWarmupExemptionForTheFirstCapturePlan() throws { + currentApp = app + currentBundleId = "com.example.fresh-process" + currentAppProcessIdentifier = 42 + snapshotXCTestPenaltyWarmupExemption.isPending = true + systemModalProbeOverrideForTesting = { _ in DataPayload(message: "blocking system modal") } + defer { + systemModalProbeOverrideForTesting = nil + runnerAccessibilityHealth = .unknown + invalidateCachedTarget(reason: "unit_test_cleanup") + } + let options = PresentationOptions(interactiveOnly: false, depth: nil, scope: nil, raw: false) + + let fast = try snapshotFast(target: takeSnapshotCaptureTarget(app: app), options: options) + let raw = try snapshotRaw(target: takeSnapshotCaptureTarget(app: app), options: options) + + XCTAssertEqual(fast.message, "blocking system modal") + XCTAssertEqual(raw.message, "blocking system modal") + XCTAssertTrue( + snapshotXCTestPenaltyWarmupExemption.isPending, + "a snapshot answered by the modal probe runs no capture plan, so the exemption stays pending" + ) + + _ = try runSnapshotCapturePlan( + [], + target: takeSnapshotCaptureTarget(app: app), + options: options, + terminal: .sparseWithFatalOnAXFailure + ) + XCTAssertFalse( - snapshotXCTestPenaltyWarmupExemptionPending, - "taking the target on main consumes the exemption, so the plan never touches the flag" + snapshotXCTestPenaltyWarmupExemption.isPending, + "the first capture plan that runs spends the exemption" ) - XCTAssertFalse(takeSnapshotCaptureTarget(app: app).xCTestPenaltyWarmupExempt) } +#endif func testMainOwnedSnapshotStateWriteRunsOnMainBeforeReturningWhenMainIsFree() { final class ResultBox { From 86e0572ff8cfb6da1fd2a595d50d49a7aba82a49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 23 Sep 2026 16:16:03 +0200 Subject: [PATCH 4/8] test(ios): pin the non-interactive query sweep to its tier slice through the plan Swizzles XCUIElementQuery.allElementsBoundByIndex with a slow query and runs a non-interactive [.querySweep] plan with a 20 s plan deadline. The sweep must not be abandoned, and no query may start after the plan answers, so passing the plan deadline to the sweep in either loop turns it red. --- .github/workflows/ios.yml | 1 + ...ts+SnapshotCapturePlanOccupancyTests.swift | 126 ++++++++++++++++++ 2 files changed, 127 insertions(+) diff --git a/.github/workflows/ios.yml b/.github/workflows/ios.yml index 667641262d..10dab9e1d2 100644 --- a/.github/workflows/ios.yml +++ b/.github/workflows/ios.yml @@ -256,6 +256,7 @@ jobs: -only-testing:AgentDeviceRunnerUITests/RunnerTests/testActionNamesAreCappedPerElementAndReported \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testHungCustomActionReadIsContainedAndRecovers \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAbandonedTreeCaptureSkipsQuerySweepAndHonorsWarmupExemption \ + -only-testing:AgentDeviceRunnerUITests/RunnerTests/testNonInteractiveQuerySweepStopsAtTheSliceItsCallerWaitsFor \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testBlockingModalSnapshotLeavesWarmupExemptionForTheFirstCapturePlan \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testPrivateAXPinnedRegularDepthReachesAcquisitionAndPresentation 2>&1 | tee /tmp/agent-device-runner-regressions.log node --input-type=module -e ' diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanOccupancyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanOccupancyTests.swift index fc05b0f8c3..b6797e2079 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanOccupancyTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanOccupancyTests.swift @@ -30,6 +30,44 @@ private final class RunnerBlockingSnapshotStub: NSObject { } } +/// Records when each `RunnerSlowSweepQueryStub` query started. The swizzled IMP cannot capture +/// test-local state, so each run resets it. +private enum RunnerSlowSweepQueryGate { + /// Shorter than `flatInteractiveQueryBudget`, so a sweep that stops at its slice deadline never + /// outlives the slice; a full sweep of these outlasts the slice, so one that ignores it does. + static let queryDuration: TimeInterval = 0.06 + private static let lock = NSLock() + private static var starts: [Date] = [] + + static func reset() { + lock.lock() + starts = [] + lock.unlock() + } + + static func recordStart() { + lock.lock() + starts.append(Date()) + lock.unlock() + } + + static func recordedStarts() -> [Date] { + lock.lock() + defer { lock.unlock() } + return starts + } +} + +/// Stands in for `-[XCUIElementQuery allElementsBoundByIndex]` so every query the sweep runs holds +/// the main thread for a fixed time and finds nothing. +private final class RunnerSlowSweepQueryStub: NSObject { + @objc var allElementsBoundByIndex: [XCUIElement] { + RunnerSlowSweepQueryGate.recordStart() + Thread.sleep(forTimeInterval: RunnerSlowSweepQueryGate.queryDuration) + return [] + } +} + extension RunnerTests { /// The Bluesky feed shape: the tree XPC grinds past its slice. The plan must recover through /// private AX without queueing the query sweep behind the abandoned XPC, and a fresh process's @@ -132,5 +170,93 @@ extension RunnerTests { return XCTFail("expected the runner idle once the tree XPC drained") } } + + /// A non-interactive query sweep runs its queries on the main thread one at a time. It must stop + /// starting them at the slice deadline its caller waits for, not at the plan deadline, or it holds + /// the main thread after the plan has answered (#2783). + func testNonInteractiveQuerySweepStopsAtTheSliceItsCallerWaitsFor() throws { + let sweepQueryCount = 19 + XCTAssertLessThan(RunnerSlowSweepQueryGate.queryDuration, Self.flatInteractiveQueryBudget) + XCTAssertGreaterThan( + RunnerSlowSweepQueryGate.queryDuration * Double(sweepQueryCount), + Self.flatInteractiveFallbackBudget + ) + guard + let queryMethod = class_getInstanceMethod( + XCUIElementQuery.self, + #selector(getter: XCUIElementQuery.allElementsBoundByIndex) + ), + let stubMethod = class_getInstanceMethod( + RunnerSlowSweepQueryStub.self, + #selector(getter: RunnerSlowSweepQueryStub.allElementsBoundByIndex) + ) + else { + XCTFail("unable to install the slow sweep query stub") + return + } + app.launchArguments = ["--agent-device-selector-read-regression"] + app.launch() + XCTAssertTrue(app.wait(for: .runningForeground, timeout: 10)) + XCTAssertFalse(app.frame.isEmpty) + currentApp = app + currentBundleId = "com.callstack.agentdevice.runner.query-sweep-slice-test" + let captureTarget = takeSnapshotCaptureTarget(app: app) + RunnerSlowSweepQueryGate.reset() + let originalImplementation = method_getImplementation(queryMethod) + method_setImplementation(queryMethod, method_getImplementation(stubMethod)) + defer { + method_setImplementation(queryMethod, originalImplementation) + clearSnapshotXCTestChannelPenalty(reason: "test-cleanup") + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } + + final class ResultBox { + var payload: DataPayload? + var error: Error? + var returnedAt: Date? + var abandonedAtReturn: Bool? + } + let box = ResultBox() + let planned = expectation(description: "query-sweep plan answered") + DispatchQueue(label: "agent-device.runner.tests.query-sweep-slice").async { + do { + box.payload = try self.runSnapshotCapturePlan( + [.querySweep], + target: captureTarget, + options: PresentationOptions(interactiveOnly: false, depth: nil, scope: nil, raw: false), + terminal: .sparseWithFatalOnAXFailure, + deadline: Date().addingTimeInterval(20) + ) + } catch { + box.error = error + } + box.returnedAt = Date() + box.abandonedAtReturn = self.hasAbandonedMainThreadWork() + planned.fulfill() + } + + wait(for: [planned], timeout: 30) + let drainDeadline = Date().addingTimeInterval(3) + while hasAbandonedMainThreadWork(), Date() < drainDeadline { + sleepFor(0.005) + } + let starts = RunnerSlowSweepQueryGate.recordedStarts() + + XCTAssertNil(box.error) + XCTAssertEqual(box.payload?.snapshotQuality?.backend, SnapshotBackendKind.querySweep.rawValue) + XCTAssertEqual( + box.abandonedAtReturn, + false, + "the sweep must finish inside the slice its caller waits for" + ) + let returnedAt = try XCTUnwrap(box.returnedAt) + XCTAssertTrue( + starts.allSatisfy { $0 <= returnedAt }, + "no sweep query may start after the plan answered" + ) + XCTAssertLessThan(starts.count, sweepQueryCount, "the slice must cut the sweep short") + XCTAssertFalse(hasAbandonedMainThreadWork()) + } } #endif From ecf1dd8f6ae0945fb832656878922f819dc8e95b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 23 Sep 2026 21:45:16 +0200 Subject: [PATCH 5/8] test(ios): pin the query-sweep tier timeout and the tap routing identity to main Two regressions #2781 still leaves open, both red on this head: A query sweep that stops at its own slice deadline is a tier timeout, but the plan classified its payload by node count, so a partial sweep that beat the sparse threshold armed no XCTest-channel penalty and no later capture of the same screen was deferred. The coordinate tap's system-modal routing probe armed its penalty with currentBundleId read on the command queue, while main owned that state and could still be clearing or rebinding it. --- .../RunnerTests+CommandDispatchTests.swift | 73 ++++++++++++ ...ts+SnapshotCapturePlanOccupancyTests.swift | 108 ++++++++++++++++++ 2 files changed, 181 insertions(+) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandDispatchTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandDispatchTests.swift index 6e75fcec2d..e9b6c6a1b2 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandDispatchTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandDispatchTests.swift @@ -248,6 +248,79 @@ extension RunnerTests { XCTAssertFalse(hasAbandonedMainThreadWork()) } + /// A coordinate tap resolves its system-modal routing on the command queue while main may still be + /// clearing or rebinding the cached target. Target identity belongs to main, so an abandoned + /// routing probe must arm its penalty with the identity main settled on — never with the identity + /// the command queue read while that write was still pending (#2781). + func testCoordinateTapRoutingPenalizesTheIdentityMainSettledOnWhileTheWriteWasPending() throws { + app.launch() + XCTAssertTrue(app.wait(for: .runningForeground, timeout: 10)) + let pendingBundleId = "com.example.routing-pending-stale" + let settledBundleId = "com.example.routing-pending-settled" + currentApp = app + currentBundleId = pendingBundleId + snapshotXCTestPenaltyWarmupExemption.isPending = false + clearSnapshotXCTestChannelPenalty(reason: "test-setup") + defer { + systemModalProbeOverrideForTesting = nil + clearSnapshotXCTestChannelPenalty(reason: "test-cleanup") + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } + + // Occupy main and rebind the target inside that block: every identity read that arrives while it + // is queued sees a target that main is on its way to replacing. + let mainRelease = DispatchSemaphore(value: 0) + DispatchQueue.main.async { + _ = mainRelease.wait(timeout: .now() + 0.5) + self.currentBundleId = settledBundleId + } + + let probeStarted = expectation(description: "system-modal routing probe started") + let probeReleaseGate = DispatchSemaphore(value: 0) + systemModalProbeOverrideForTesting = { _ in + probeStarted.fulfill() + // Outlives the probe's own slice, so the abandonment hook is what arms the penalty. + _ = probeReleaseGate.wait(timeout: .now() + 15) + return nil + } + let tap = try runnerCommandFixture( + #"{"command":"tap","commandId":"tap-pending-target-write","x":10,"y":20}"# + ) + + final class ResultBox { + var response: Response? + var penalizedPendingIdentity = false + var penalizedSettledIdentity = false + } + let box = ResultBox() + let tapFinished = expectation(description: "off-main tap returned") + DispatchQueue(label: "agent-device.runner.tests.tap-pending-target-write").async { + box.response = try? self.executeDispatched(command: tap) + box.penalizedPendingIdentity = self.isSnapshotXCTestChannelPenalized(bundleId: pendingBundleId) + box.penalizedSettledIdentity = self.isSnapshotXCTestChannelPenalized(bundleId: settledBundleId) + probeReleaseGate.signal() + tapFinished.fulfill() + } + + wait(for: [probeStarted, tapFinished], timeout: 40) + mainRelease.signal() + let drainDeadline = Date().addingTimeInterval(5) + while hasAbandonedMainThreadWork(), Date() < drainDeadline { + sleepFor(0.002) + } + + XCTAssertTrue( + box.penalizedSettledIdentity, + "the abandoned routing probe must penalize the target main settled on" + ) + XCTAssertFalse( + box.penalizedPendingIdentity, + "the command queue may not key a penalty with an identity whose write was still pending on main" + ) + XCTAssertFalse(hasAbandonedMainThreadWork()) + } + func testSkipAppActivationPreflightRejectsSelectorAndMixedSequenceGestures() throws { app.launch() currentApp = app diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanOccupancyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanOccupancyTests.swift index b6797e2079..edec6e8333 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanOccupancyTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanOccupancyTests.swift @@ -258,5 +258,113 @@ extension RunnerTests { XCTAssertLessThan(starts.count, sweepQueryCount, "the slice must cut the sweep short") XCTAssertFalse(hasAbandonedMainThreadWork()) } + + /// A query sweep that stops at its own slice deadline collected a partial tree, not an answer, so + /// it is a tier timeout: the plan must arm the XCTest-channel penalty and let private AX answer, + /// keeping the partial sweep only as the fallback. Accepting it because it carries more nodes than + /// the sparse threshold ships a hierarchy-free capture and leaves every later capture of the same + /// screen to pay for the full sweep again (#2781). + func testQuerySweepThatEndsOnItsSliceDeadlinePenalizesChannelAndReachesPrivateAX() throws { + guard + let queryMethod = class_getInstanceMethod( + XCUIElementQuery.self, + #selector(getter: XCUIElementQuery.allElementsBoundByIndex) + ), + let stubMethod = class_getInstanceMethod( + RunnerSlowSweepQueryStub.self, + #selector(getter: RunnerSlowSweepQueryStub.allElementsBoundByIndex) + ) + else { + XCTFail("unable to install the slow sweep query stub") + return + } + app.launchArguments = ["--agent-device-selector-read-regression"] + app.launch() + XCTAssertTrue(app.wait(for: .runningForeground, timeout: 10)) + currentApp = app + currentBundleId = "com.callstack.agentdevice.runner.query-sweep-timeout-test" + snapshotXCTestPenaltyWarmupExemption.isPending = false + clearSnapshotXCTestChannelPenalty(reason: "test-setup") + RunnerSlowSweepQueryGate.reset() + let captureTarget = takeSnapshotCaptureTarget(app: app) + let originalImplementation = method_getImplementation(queryMethod) + method_setImplementation(queryMethod, method_getImplementation(stubMethod)) + defer { + method_setImplementation(queryMethod, originalImplementation) + clearSnapshotXCTestChannelPenalty(reason: "test-cleanup") + clearPrivateAXAcceptedDepth(reason: "test-cleanup") + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } + + final class ResultBox { + var payload: DataPayload? + var error: Error? + var abandonedAtReturn = false + var penalizedAtReturn = false + var sweepStartsAfterFirstPlan = 0 + var secondPayload: DataPayload? + var secondError: Error? + } + let box = ResultBox() + let planned = expectation(description: "slow-sweep plan answered") + DispatchQueue(label: "agent-device.runner.tests.query-sweep-timeout").async { + do { + box.payload = try self.runSnapshotCapturePlan( + [.querySweep, .privateAX], + target: captureTarget, + options: PresentationOptions(interactiveOnly: false, depth: nil, scope: nil, raw: false), + terminal: .sparseWithFatalOnAXFailure, + deadline: Date().addingTimeInterval(20) + ) + } catch { + box.error = error + } + box.abandonedAtReturn = self.hasAbandonedMainThreadWork() + box.penalizedAtReturn = self.isSnapshotXCTestChannelPenalized(bundleId: captureTarget.bundleId) + box.sweepStartsAfterFirstPlan = RunnerSlowSweepQueryGate.recordedStarts().count + do { + box.secondPayload = try self.runSnapshotCapturePlan( + Self.regularVisiblePlan, + target: self.takeSnapshotCaptureTarget(app: self.app), + options: PresentationOptions(interactiveOnly: false, depth: nil, scope: nil, raw: false), + terminal: .sparseWithFatalOnAXFailure, + deadline: Date().addingTimeInterval(20) + ) + } catch { + box.secondError = error + } + planned.fulfill() + } + + wait(for: [planned], timeout: 60) + XCTAssertNil(box.error) + let quality = box.payload?.snapshotQuality + XCTAssertEqual( + quality?.backend, + SnapshotBackendKind.privateAX.rawValue, + "a sweep that ended on its slice deadline is a tier timeout, not an accepted capture" + ) + XCTAssertEqual(quality?.state, "recovered") + XCTAssertGreaterThan(box.payload?.nodes?.count ?? 0, 1, "private AX answers with a real tree") + XCTAssertFalse(box.abandonedAtReturn, "the sweep must answer inside its own main-thread hop") + XCTAssertTrue( + box.penalizedAtReturn, + "the tier timeout must arm the XCTest-channel penalty for the captured target" + ) + + XCTAssertNil(box.secondError) + XCTAssertEqual( + box.secondPayload?.snapshotQuality?.backend, + SnapshotBackendKind.privateAX.rawValue, + "the armed penalty must defer the next plan's XCTest tiers, not re-run the sweep" + ) + XCTAssertEqual( + RunnerSlowSweepQueryGate.recordedStarts().count, + box.sweepStartsAfterFirstPlan, + "the deferred plan may start no sweep query at all" + ) + XCTAssertFalse(hasAbandonedMainThreadWork()) + } } #endif From f1ad73724d47a094258608e97e8a81365e1fe0cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 23 Sep 2026 21:50:10 +0200 Subject: [PATCH 6/8] fix(ios): count a query sweep that ends on its slice deadline as a tier timeout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The sweep tier stopped starting queries at its slice deadline, but the plan classified only what it collected. sparsePayloadReason accepts anything above the sparse node threshold, so a partial flat sweep became a `recovered` capture: no XCTest-channel penalty was armed, private AX never ran, and every later capture of that screen paid for the full sweep again. `runFlatInteractiveQueries` now returns a typed `SnapshotTierOutcome`, the sweep tier carries it into the attempt, and the plan rejects a `.deadlineExhausted` tier through `snapshotTierRejectionReason` — keeping its payload only as the fallback, arming the penalty through the existing tier timeout reason, and letting the next backend answer. --- .../RunnerTests+Snapshot.swift | 40 ++++++---- .../RunnerTests+SnapshotAcquisition.swift | 12 +-- .../RunnerTests+SnapshotCapturePlan.swift | 79 +++++++++++++++---- .../RunnerTests+SnapshotTiming.swift | 17 ++++ 4 files changed, 112 insertions(+), 36 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift index ff538bb5bd..bc08a4b55e 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift @@ -439,18 +439,21 @@ extension RunnerTests { app: XCUIApplication, hint: CaptureHint, sliceDeadline deadline: Date - ) -> SnapshotAcquisition { + ) -> (acquisition: SnapshotAcquisition, outcome: SnapshotTierOutcome) { var nodes: [RawAXNode] = [ interactiveRootNode(rect: .zero) ] if hint.rawTraversalDepth == 0 || hint.regularPresentedDepth == 0 { - return SnapshotAcquisition( - hint: hint, - nodes: nodes, - truncated: false, - effectiveDepth: nil, - viewport: .infinite, - interfaceOrientation: RunnerInterfaceOrientation.unknown + return ( + SnapshotAcquisition( + hint: hint, + nodes: nodes, + truncated: false, + effectiveDepth: nil, + viewport: .infinite, + interfaceOrientation: RunnerInterfaceOrientation.unknown + ), + .completed ) } @@ -458,11 +461,11 @@ extension RunnerTests { var seen = Set() var candidates: [RawAXNode] = [] let flatElements = flatInteractiveElements(app: app, deadline: deadline) - var truncated = flatElements.truncated + var outcome = flatElements.outcome for element in flatElements.elements { if !Self.querySweepCanStartQuery(deadline: deadline, now: Date()) { NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_FLAT_FALLBACK_DEADLINE") - truncated = true + outcome = .deadlineExhausted break } guard let node = flatSnapshotNode(element: element, index: 0, parentIndex: 0) else { @@ -510,13 +513,16 @@ extension RunnerTests { ) ) } - return SnapshotAcquisition( - hint: hint, - nodes: nodes, - truncated: truncated, - effectiveDepth: nil, - viewport: viewport, - interfaceOrientation: RunnerInterfaceOrientation.unknown + return ( + SnapshotAcquisition( + hint: hint, + nodes: nodes, + truncated: outcome == .deadlineExhausted, + effectiveDepth: nil, + viewport: viewport, + interfaceOrientation: RunnerInterfaceOrientation.unknown + ), + outcome ) } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift index 102fbc1727..26572f56fa 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift @@ -464,7 +464,7 @@ extension RunnerTests { func flatInteractiveElements( app: XCUIApplication, deadline: Date - ) -> (elements: [XCUIElement], truncated: Bool) { + ) -> (elements: [XCUIElement], outcome: SnapshotTierOutcome) { let queries: [XCUIElementQuery] = [ app.buttons, app.links, @@ -495,18 +495,20 @@ extension RunnerTests { } /// Runs sweep queries in order until one reports AX unavailable, or until the next one could not - /// finish before `deadline` (`querySweepCanStartQuery`). + /// finish before `deadline` (`querySweepCanStartQuery`). A sweep stopped by the deadline reports + /// `.deadlineExhausted` rather than a truncation flag: what it collected is a partial tree, and + /// only the caller that owns the tier decides whether that counts as an answer (#2781). static func runFlatInteractiveQueries( _ queries: [Query], deadline: Date, now: () -> Date = { Date() }, run: (Query) -> (elements: [Element], axUnavailable: Bool) - ) -> (elements: [Element], truncated: Bool) { + ) -> (elements: [Element], outcome: SnapshotTierOutcome) { var elements: [Element] = [] for query in queries { if !querySweepCanStartQuery(deadline: deadline, now: now()) { NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_FLAT_FALLBACK_DEADLINE") - return (elements, true) + return (elements, .deadlineExhausted) } let result = run(query) elements.append(contentsOf: result.elements) @@ -514,7 +516,7 @@ extension RunnerTests { break } } - return (elements, false) + return (elements, .completed) } func snapshotElementsQuery( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift index df91c3a58e..e7f3898652 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift @@ -37,6 +37,16 @@ enum SnapshotXCTestChannelPlanState: Equatable { case boundedXCTestProbe } +/// How one tier's bounded work ended. `deadlineExhausted` is a tier timeout: the tier stopped +/// starting work it could not finish inside its own slice, so what it returns is a partial result +/// it never completed collecting. The plan keeps that result only as the fallback and lets the next +/// backend answer; a node count cannot tell the two apart, because a short sweep still collects +/// more than the sparse threshold (#2781). +enum SnapshotTierOutcome: Equatable { + case completed + case deadlineExhausted +} + struct EffectiveSnapshotCapturePlan { let plan: [SnapshotBackendKind] let xCTestChannelState: SnapshotXCTestChannelPlanState @@ -351,8 +361,19 @@ extension RunnerTests { } guard case let .captured(capture) = attempt.outcome else { continue } - if let sparseReason = Self.sparsePayloadReason(capture.qualityPayload ?? capture.payload) { - if firstFailure == nil { firstFailure = sparseReason } + if attempt.tierOutcome == .deadlineExhausted { + NSLog( + "AGENT_DEVICE_RUNNER_SNAPSHOT_TIER_DEADLINE_EXHAUSTED backend=%@ nodes=%d", + kind.rawValue, + Self.payloadNodeCount(capture.payload) + ) + } + if let rejection = Self.snapshotTierRejectionReason( + outcome: attempt.tierOutcome, + kind: kind, + payload: capture.qualityPayload ?? capture.payload + ) { + if firstFailure == nil { firstFailure = rejection } if Self.payloadNodeCount(capture.payload) > Self.payloadNodeCount(best?.capture.payload) { best = (kind, capture) } @@ -416,11 +437,12 @@ extension RunnerTests { let hint = SnapshotPresentation.captureHint(for: options) var timer = SnapshotPhaseTimer() let acquisition: SnapshotAcquisition? + let tierOutcome: SnapshotTierOutcome // The band is read inside the tree tier's own bounded work, so it has to be lifted out of the // acquisition phase and carried to the stamping step, where the payload is assembled (#2660). var keyboardBand: RunnerKeyboardBandFact? do { - acquisition = try timer.measure(.acquisition) { + let measured = try timer.measure(.acquisition) { () -> (SnapshotAcquisition?, SnapshotTierOutcome) in switch kind { case .recursiveTree: guard @@ -431,10 +453,10 @@ extension RunnerTests { treeCaptureSliceBudgetOverride: treeCaptureSliceBudgetOverride ) else { - return nil + return (nil, .completed) } keyboardBand = context.keyboardBand - return try self.runMainThreadWork( + let tree = try self.runMainThreadWork( "tree_processing", timeout: min(self.treeCaptureSliceBudget, max(0.5, deadline.timeIntervalSinceNow)), timeoutError: self.snapshotMainThreadTimeoutError("processing tree snapshot") @@ -443,9 +465,10 @@ extension RunnerTests { ? try self.rawTreeSnapshotAcquisition(context: context, hint: hint) : try self.recursiveTreeSnapshotAcquisition(context: context, hint: hint) } + return (tree, .completed) case .querySweep: let sliceDeadline = Self.querySweepSliceDeadline(startedAt: Date(), planDeadline: deadline) - return try self.runMainThreadWork( + let sweep = try self.runMainThreadWork( "query_sweep", timeout: max(0.1, sliceDeadline.timeIntervalSinceNow), timeoutError: self.snapshotMainThreadTimeoutError("running query-sweep snapshot") @@ -456,14 +479,20 @@ extension RunnerTests { sliceDeadline: sliceDeadline ) } + return (sweep.acquisition, sweep.outcome) case .privateAX: - return self.privateAXSnapshotAcquisition( - target: target, - hint: hint, - deadline: deadline + return ( + self.privateAXSnapshotAcquisition( + target: target, + hint: hint, + deadline: deadline + ), + .completed ) } } + acquisition = measured.0 + tierOutcome = measured.1 } catch let failure as SnapshotCaptureFailure { return SnapshotBackendAttempt( outcome: .failed(failure, phase: .acquisition), @@ -473,7 +502,8 @@ extension RunnerTests { guard let acquisition else { return SnapshotBackendAttempt( outcome: .noCapture, - timing: timer.timing + timing: timer.timing, + tierOutcome: tierOutcome ) } @@ -508,12 +538,14 @@ extension RunnerTests { } catch let failure as SnapshotPresentationFailure { return SnapshotBackendAttempt( outcome: .failed(Self.snapshotCaptureFailure(for: failure), phase: .presentation), - timing: timer.timing + timing: timer.timing, + tierOutcome: tierOutcome ) } catch let failure as SnapshotCaptureFailure { return SnapshotBackendAttempt( outcome: .failed(failure, phase: .presentation), - timing: timer.timing + timing: timer.timing, + tierOutcome: tierOutcome ) } @@ -522,7 +554,8 @@ extension RunnerTests { capture.keyboardBand = keyboardBand?.payload return SnapshotBackendAttempt( outcome: .captured(capture), - timing: timer.timing + timing: timer.timing, + tierOutcome: tierOutcome ) } @@ -543,6 +576,24 @@ extension RunnerTests { // MARK: Quality classifier (the single source of "is this snapshot degraded") + /// Why a captured tier may not end the plan, or nil when it may. A tier that stopped starting work + /// at its own deadline is rejected as a timeout whatever it collected: keeping its payload as the + /// fallback is right, accepting it is not, and a node count cannot tell a finished capture from a + /// sweep the slice cut short (#2781). + static func snapshotTierRejectionReason( + outcome: SnapshotTierOutcome, + kind: SnapshotBackendKind, + payload: DataPayload + ) -> (reason: String, code: String)? { + if outcome == .deadlineExhausted { + return ( + "the \(kind.rawValue) backend spent its capture slice with the collection unfinished", + "budget" + ) + } + return sparsePayloadReason(payload) + } + /// Returns a degradation reason + machine code when the payload is too degraded to accept. static func sparsePayloadReason(_ payload: DataPayload) -> (reason: String, code: String)? { guard let nodes = payload.nodes, !nodes.isEmpty else { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotTiming.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotTiming.swift index ae2dedc21d..fd11051e0f 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotTiming.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotTiming.swift @@ -98,6 +98,20 @@ extension RunnerTests { /// or error text. let outcome: Outcome let timing: SnapshotCaptureTiming + /// Whether the tier finished collecting or stopped at its own deadline. A tier that stopped at + /// its deadline timed out even when it handed back a payload, so penalty and recovery policy + /// read this instead of classifying the payload (#2781). + let tierOutcome: SnapshotTierOutcome + + init( + outcome: Outcome, + timing: SnapshotCaptureTiming, + tierOutcome: SnapshotTierOutcome = .completed + ) { + self.outcome = outcome + self.timing = timing + self.tierOutcome = tierOutcome + } } /// The penalty breaker observes only acquisition facts. Presentation is a separate phase and @@ -113,6 +127,9 @@ extension RunnerTests { { return "\(kind.rawValue)_backend_timeout" } + if attempt.tierOutcome == .deadlineExhausted { + return "\(kind.rawValue)_backend_timeout" + } guard attempt.timing.acquisitionMs > slowThresholdMs else { return nil } return "slow_\(kind.rawValue)_capture_\(Int(attempt.timing.acquisitionMs))ms" } From 8272c8a9c7f2706959220db71cc996c8f50ecdfd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 23 Sep 2026 21:50:12 +0200 Subject: [PATCH 7/8] fix(ios): take the system-modal probe's target identity on main The coordinate tap resolved its system-modal routing on the command queue and armed an abandoned probe's penalty with `currentBundleId` read there, while `applyMainOwnedSnapshotState` and the lifecycle code write that identity through `DispatchQueue.main.async`. A tap could penalize a target main was still clearing or rebinding. The probe now takes a `SnapshotProbePenaltyTarget`: a capture hands over the identity it already took on main, and the tap route hands over nothing and lets the probe's own main-side block capture the identity main holds once its work starts. No off-main path reads target identity any more. --- .github/workflows/ios.yml | 2 + .../RunnerTests+CommandDispatch.swift | 6 ++- .../RunnerTests+Snapshot.swift | 21 +++++---- .../RunnerTests+SnapshotCaptureTarget.swift | 44 +++++++++++++++++++ ...nnerTests+SnapshotCaptureTargetTests.swift | 18 ++++++++ 5 files changed, 82 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ios.yml b/.github/workflows/ios.yml index 10dab9e1d2..00dc9df934 100644 --- a/.github/workflows/ios.yml +++ b/.github/workflows/ios.yml @@ -257,6 +257,8 @@ jobs: -only-testing:AgentDeviceRunnerUITests/RunnerTests/testHungCustomActionReadIsContainedAndRecovers \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAbandonedTreeCaptureSkipsQuerySweepAndHonorsWarmupExemption \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testNonInteractiveQuerySweepStopsAtTheSliceItsCallerWaitsFor \ + -only-testing:AgentDeviceRunnerUITests/RunnerTests/testQuerySweepThatEndsOnItsSliceDeadlinePenalizesChannelAndReachesPrivateAX \ + -only-testing:AgentDeviceRunnerUITests/RunnerTests/testCoordinateTapRoutingPenalizesTheIdentityMainSettledOnWhileTheWriteWasPending \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testBlockingModalSnapshotLeavesWarmupExemptionForTheFirstCapturePlan \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testPrivateAXPinnedRegularDepthReachesAcquisitionAndPresentation 2>&1 | tee /tmp/agent-device-runner-regressions.log node --input-type=module -e ' diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandDispatch.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandDispatch.swift index da4792455d..16b2dfccd2 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandDispatch.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandDispatch.swift @@ -579,9 +579,13 @@ extension RunnerTests { } #endif let probeDeadline = Date().addingTimeInterval(systemModalProbeBudget) + // Routing runs on the command queue, so this hands the probe a target rather than a bundle id it + // read across the main boundary: the penalty an abandoned probe arms carries the identity main + // holds once the probe starts, not one whose write was still queued behind the block that + // occupied main (#2781). return boundedBlockingSystemAlertSnapshot( deadline: probeDeadline, - penaltyBundleId: currentBundleId + penaltyTarget: .mainOwnedTarget ) != nil #else return false diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift index bc08a4b55e..3d469bf868 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift @@ -163,7 +163,7 @@ extension RunnerTests { let deadline = Date().addingTimeInterval(Self.snapshotPlanBudget) if let blocking = boundedBlockingSystemAlertSnapshot( deadline: deadline, - penaltyBundleId: target.bundleId + penaltyTarget: .prepared(bundleId: target.bundleId) ) { return blocking } @@ -288,7 +288,7 @@ extension RunnerTests { let deadline = Date().addingTimeInterval(Self.snapshotPlanBudget) if let blocking = boundedBlockingSystemAlertSnapshot( deadline: deadline, - penaltyBundleId: target.bundleId + penaltyTarget: .prepared(bundleId: target.bundleId) ) { return blocking } @@ -303,11 +303,14 @@ extension RunnerTests { /// Runs the pre-plan SpringBoard system-modal probe as a bounded capture tier sharing the plan /// deadline, so a slow alert enumeration cannot bypass the snapshot timeout and stall (#1244). - /// An abandoned probe penalizes the XCTest channel for `penaltyBundleId`. - func boundedBlockingSystemAlertSnapshot(deadline: Date, penaltyBundleId: String?) -> DataPayload? { + /// An abandoned probe penalizes the XCTest channel for `penaltyTarget`. + func boundedBlockingSystemAlertSnapshot( + deadline: Date, + penaltyTarget: SnapshotProbePenaltyTarget + ) -> DataPayload? { boundedBlockingSystemAlertSnapshotBody( deadline: deadline, - penaltyBundleId: penaltyBundleId + penaltyTarget: penaltyTarget ) { probeDeadline in #if AGENT_DEVICE_RUNNER_UNIT_TESTS if let override = self.systemModalProbeOverrideForTesting { @@ -325,7 +328,7 @@ extension RunnerTests { /// production runs and what the unit tests exercise. private func boundedBlockingSystemAlertSnapshotBody( deadline: Date, - penaltyBundleId: String?, + penaltyTarget: SnapshotProbePenaltyTarget, probe: @escaping (Date) -> DataPayload? ) -> DataPayload? { #if os(macOS) @@ -341,6 +344,7 @@ extension RunnerTests { } let probeDeadline = Date().addingTimeInterval(slice) let startedAt = Date() + let penaltyIdentity = SnapshotProbePenaltyIdentity(penaltyTarget) do { return try runMainThreadWork( "system_modal_probe", @@ -354,12 +358,13 @@ extension RunnerTests { }, onAbandoned: { self.penalizeSnapshotXCTestChannel( - bundleId: penaltyBundleId, + bundleId: penaltyIdentity.penalizedBundleId, reason: "system_modal_probe_timeout" ) } ) { - probe(probeDeadline) + penaltyIdentity.captureFromMain(bundleId: self.currentBundleId) + return probe(probeDeadline) } } catch { NSLog( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCaptureTarget.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCaptureTarget.swift index ea8e06ecde..446db190f2 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCaptureTarget.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCaptureTarget.swift @@ -21,6 +21,50 @@ enum SnapshotCommandPreparation { case capture(SnapshotCaptureTarget, systemSurface: SystemSurfaceHost?) } +/// The target a bounded XCTest probe arms its abandonment penalty with. +/// +/// The hook that arms the penalty fires on the command queue the moment the probe's slice is spent, +/// while the probe's own work block may still be running on main. `currentBundleId` belongs to main, +/// so it is never read across that boundary: a caller that already took the identity on main hands it +/// over, and a caller that is on the command queue lets the probe's main-side block capture the +/// identity main holds once the work actually starts (#2781). +enum SnapshotProbePenaltyTarget: Equatable { + /// Identity a capture took on main when it prepared its target. + case prepared(bundleId: String?) + /// Read `currentBundleId` inside the probe's main-side block. + case mainOwnedTarget +} + +/// One probe's penalty identity: written by the probe's main-side block, read by the command queue's +/// abandonment hook through this lock. +final class SnapshotProbePenaltyIdentity { + private let lock = NSLock() + private var bundleId: String? + private let readsMainOwnedTarget: Bool + + init(_ target: SnapshotProbePenaltyTarget) { + readsMainOwnedTarget = target == .mainOwnedTarget + if case .prepared(let bundleId) = target { + self.bundleId = bundleId + } + } + + /// Called on the main thread inside the probe's work block, before it enumerates anything, so the + /// identity is the one main had settled on rather than one a queued write is about to replace. + func captureFromMain(bundleId: String?) { + guard readsMainOwnedTarget else { return } + lock.lock() + self.bundleId = bundleId + lock.unlock() + } + + var penalizedBundleId: String? { + lock.lock() + defer { lock.unlock() } + return bundleId + } +} + extension RunnerTests { /// Main thread only: reads the lifecycle-owned target identity. func takeSnapshotCaptureTarget(app: XCUIApplication) -> SnapshotCaptureTarget { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCaptureTargetTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCaptureTargetTests.swift index 3d4a0a7233..2139a30ccb 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCaptureTargetTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCaptureTargetTests.swift @@ -61,6 +61,24 @@ extension RunnerTests { } #endif + func testProbePenaltyIdentityTakesTheIdentityItsCallerMayLegallyKnow() { + let prepared = SnapshotProbePenaltyIdentity(.prepared(bundleId: "com.example.prepared")) + prepared.captureFromMain(bundleId: "com.example.settled") + XCTAssertEqual( + prepared.penalizedBundleId, + "com.example.prepared", + "a capture penalizes the target it was prepared for, whatever lifecycle binds meanwhile" + ) + + let mainOwned = SnapshotProbePenaltyIdentity(.mainOwnedTarget) + XCTAssertNil( + mainOwned.penalizedBundleId, + "a command-queue caller knows no identity until the probe's main-side block runs" + ) + mainOwned.captureFromMain(bundleId: "com.example.settled") + XCTAssertEqual(mainOwned.penalizedBundleId, "com.example.settled") + } + func testMainOwnedSnapshotStateWriteRunsOnMainBeforeReturningWhenMainIsFree() { final class ResultBox { var ranOnMainThread: Bool? From 7414149edb71e11de02f1685db96111976e3ede5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 24 Sep 2026 08:01:29 +0200 Subject: [PATCH 8/8] test(ios): place this PR's runner tests in main's split UnitTests files main (#2854, #2855) moved the runner's inline unit tests into UnitTests/ and split RunnerTests+CommandExecution.swift along command families. Rebase conflicts dropped this PR's edits from the inline blocks the rebase kept on main; relocate them into the file main now owns for each family: - RunnerTests+SnapshotTests.swift: the query-sweep slice-deadline and fail-closed-invalidation tests, and the bounded-modal-probe helper now takes a SnapshotCaptureTarget. - RunnerTests+SnapshotCapturePlanTests.swift: the tier-timeout rejection pair and the private-AX depth helper now builds a SnapshotCaptureTarget. - RunnerTests+SnapshotTimingTests.swift: the deadline-exhausted tier penalty test. --- ...RunnerTests+SnapshotCapturePlanTests.swift | 33 +++- .../UnitTests/RunnerTests+SnapshotTests.swift | 164 +++++++++++++++++- .../RunnerTests+SnapshotTimingTests.swift | 45 +++++ 3 files changed, 236 insertions(+), 6 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanTests.swift index f0eae25e74..a2c709eeb9 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanTests.swift @@ -76,6 +76,37 @@ extension RunnerTests { XCTAssertNotNil(Self.sparsePayloadReason(DataPayload(nodes: [], truncated: false))) } + /// A sweep the slice cut short is rejected as a tier timeout even when what it collected clears + /// every bar the quality classifier has, while the identical payload from a sweep that ran all its + /// queries is accepted. The pair is the point: node count cannot see the deadline (#2781). + func testDeadlineExhaustedTierIsRejectedWhileItsIdenticalCompletedPayloadIsAccepted() { + let root = planTestNode(index: 0, type: "Application", label: "Example App", hittable: true) + let nodes: [PresentedNode] = [root] + (1..<13).map { index in + planTestNode(index: index, type: "Button", label: "Row \(index)", hittable: true, parentIndex: 0) + } + let payload = DataPayload(nodes: nodes, truncated: true) + XCTAssertNil( + Self.sparsePayloadReason(payload), + "this payload clears the classifier on its own, so only the tier outcome can reject it" + ) + XCTAssertEqual( + Self.snapshotTierRejectionReason( + outcome: .deadlineExhausted, + kind: .querySweep, + payload: payload + )?.code, + "budget", + "a tier that spent its capture slice is a timeout whatever it collected" + ) + XCTAssertNil( + Self.snapshotTierRejectionReason( + outcome: .completed, + kind: .querySweep, + payload: payload + ) + ) + } + func testCollapsedLeafIndexesFlagsMergedContainersOnly() { let root = planTestNode(index: 0, type: "Application", label: "App") let merged = planTestNode( @@ -460,7 +491,7 @@ extension RunnerTests { func capture(depth: Int?) throws -> DataPayload { try runSnapshotCapturePlan( Self.regularVisiblePlan, - app: app, + target: takeSnapshotCaptureTarget(app: app), options: PresentationOptions( interactiveOnly: false, depth: depth, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotTests.swift index 83ac2f5a00..c679bbcbde 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotTests.swift @@ -61,6 +61,159 @@ extension RunnerTests { XCTAssertEqual(Self.systemModalProbeSlice(budget: 4, deadlineRemaining: -5), 0) } + func testSnapshotAccessibilityUnavailableQueuesInvalidationBehindAbandonedMainThreadWork() { + currentBundleId = "com.example.stale-target" + runnerAccessibilityHealth = .healthy + defer { + currentBundleId = nil + runnerAccessibilityHealth = .unknown + } + + final class ResultBox { + var elapsed: TimeInterval? + var payload: DataPayload? + var bundleStillCachedWhileBlocked: Bool? + var healthWhileBlocked: RunnerAccessibilityHealth? + var abandonedWhileBlocked: Int? + } + let box = ResultBox() + let mainBlocked = DispatchSemaphore(value: 0) + let releaseMain = DispatchSemaphore(value: 0) + let finished = expectation(description: "fail-closed capture returned while main was blocked") + + DispatchQueue(label: "agent-device.runner.tests.ax-unavailable-invalidation").async { + _ = try? self.runMainThreadWork( + "command_execution", + timeout: 0, + timeoutError: self.mainThreadExecutionTimeoutError + ) { + mainBlocked.signal() + _ = releaseMain.wait(timeout: .now() + 5) + return true + } + _ = mainBlocked.wait(timeout: .now() + 2) + let startedAt = Date() + box.payload = self.snapshotAccessibilityUnavailable( + failure: SnapshotCaptureFailure( + code: Self.axSnapshotErrorCode, + message: Self.axSnapshotFailureMessage, + hint: Self.axSnapshotHint + ) + ) + box.elapsed = Date().timeIntervalSince(startedAt) + box.bundleStillCachedWhileBlocked = self.currentBundleId != nil + box.healthWhileBlocked = self.runnerAccessibilityHealth + self.mainThreadWorkLock.lock() + box.abandonedWhileBlocked = self.abandonedMainThreadWorkCount + self.mainThreadWorkLock.unlock() + releaseMain.signal() + finished.fulfill() + } + + wait(for: [finished], timeout: 8) + let drainDeadline = Date().addingTimeInterval(2) + while hasAbandonedMainThreadWork() || currentBundleId != nil, Date() < drainDeadline { + sleepFor(0.005) + } + + XCTAssertEqual(box.payload?.runnerFatal, true) + XCTAssertLessThan( + box.elapsed ?? .infinity, + 0.5, + "the fail-closed capture must not wait behind abandoned main-thread work" + ) + XCTAssertEqual( + box.bundleStillCachedWhileBlocked, + true, + "the invalidation must queue behind the blocked main thread, not run on the command queue" + ) + XCTAssertEqual( + box.healthWhileBlocked, + .healthy, + "the health write must queue behind the blocked main thread, not run on the command queue" + ) + XCTAssertEqual(box.abandonedWhileBlocked, 1, "the deferred write must not add an abandoned unit") + XCTAssertFalse(hasAbandonedMainThreadWork()) + XCTAssertNil(currentBundleId, "the invalidation must run once the main thread frees") + XCTAssertEqual(runnerAccessibilityHealth, .unavailable) + } + + func testQuerySweepSliceDeadlineIsTheTierSliceNotThePlanDeadline() { + let startedAt = Date(timeIntervalSinceReferenceDate: 1_000) + let slice = startedAt.addingTimeInterval(Self.flatInteractiveFallbackBudget) + + XCTAssertEqual( + Self.querySweepSliceDeadline( + startedAt: startedAt, + planDeadline: startedAt.addingTimeInterval(Self.snapshotPlanBudget) + ), + slice + ) + let nearPlanDeadline = startedAt.addingTimeInterval(Self.flatInteractiveFallbackBudget / 2) + XCTAssertEqual( + Self.querySweepSliceDeadline(startedAt: startedAt, planDeadline: nearPlanDeadline), + nearPlanDeadline + ) + } + + func testQuerySweepStopsAtTheSliceDeadlineBeforeAQueryThatCannotFinish() { + let startedAt = Date(timeIntervalSinceReferenceDate: 1_000) + let sliceDeadline = Self.querySweepSliceDeadline( + startedAt: startedAt, + planDeadline: startedAt.addingTimeInterval(Self.snapshotPlanBudget) + ) + var clock = startedAt + var startedQueries: [Int] = [] + + let sweep = Self.runFlatInteractiveQueries( + Array(0..<19), + deadline: sliceDeadline, + now: { clock } + ) { query -> (elements: [Int], axUnavailable: Bool) in + startedQueries.append(query) + clock = clock.addingTimeInterval(0.475) + return ([query], false) + } + + XCTAssertEqual(startedQueries, [0, 1], "a query must not start with less than one query budget left") + XCTAssertEqual(sweep.elements, [0, 1]) + XCTAssertEqual(sweep.outcome, .deadlineExhausted) + XCTAssertLessThanOrEqual(clock, sliceDeadline, "no started query may outlive the tier slice") + + XCTAssertTrue( + Self.querySweepCanStartQuery(deadline: sliceDeadline, now: sliceDeadline.addingTimeInterval(-0.2)) + ) + XCTAssertFalse( + Self.querySweepCanStartQuery(deadline: sliceDeadline, now: sliceDeadline.addingTimeInterval(-0.05)) + ) + XCTAssertFalse(Self.querySweepCanStartQuery(deadline: sliceDeadline, now: sliceDeadline)) + } + + func testQuerySweepRunsEveryQueryThatFitsAndStopsOnAXUnavailable() { + let startedAt = Date(timeIntervalSinceReferenceDate: 1_000) + let deadline = startedAt.addingTimeInterval(Self.flatInteractiveFallbackBudget) + + let complete = Self.runFlatInteractiveQueries( + [0, 1, 2], + deadline: deadline, + now: { startedAt } + ) { query -> (elements: [Int], axUnavailable: Bool) in ([query], false) } + XCTAssertEqual(complete.elements, [0, 1, 2]) + XCTAssertEqual(complete.outcome, .completed) + + let rejected = Self.runFlatInteractiveQueries( + [0, 1, 2], + deadline: deadline, + now: { startedAt } + ) { query -> (elements: [Int], axUnavailable: Bool) in ([query], query == 1) } + XCTAssertEqual(rejected.elements, [0, 1]) + XCTAssertEqual( + rejected.outcome, + .completed, + "an AX refusal ends the sweep on its own terms, not on the deadline" + ) + } + // Simulator-only: the bounded probe body returns nil on macOS (no SpringBoard host), so the // timeout/penalty/drain machinery below only exists on the iOS branch. #if os(iOS) @@ -83,13 +236,14 @@ extension RunnerTests { /// assertion instead of racing a fixed-timing guess. private func assertBoundedSystemModalProbeTimeoutRecoversThenReleasesOnDrain( entryPointName: String, - callEntryPoint: @escaping (XCUIApplication, PresentationOptions) throws -> DataPayload + callEntryPoint: @escaping (SnapshotCaptureTarget, PresentationOptions) throws -> DataPayload ) { let targetBundleId = "com.callstack.agentdevice.runner.missing.snapshot-timeout-test" let snapshotTarget = XCUIApplication(bundleIdentifier: targetBundleId) let probeReleaseGate = DispatchSemaphore(value: 0) currentApp = snapshotTarget currentBundleId = targetBundleId + let captureTarget = takeSnapshotCaptureTarget(app: snapshotTarget) defer { probeReleaseGate.signal() currentApp = nil @@ -120,7 +274,7 @@ extension RunnerTests { let drained = expectation(description: "\(entryPointName) modal probe drained") DispatchQueue(label: "agent-device.runner.tests.modal-probe-timeout").async { box.payload = try? callEntryPoint( - snapshotTarget, + captureTarget, PresentationOptions(interactiveOnly: false, depth: nil, scope: nil, raw: false) ) @@ -130,7 +284,7 @@ extension RunnerTests { box.wasBusyBeforeDrain = true } box.hadAbandonedCaptureBeforeDrain = self.hasAbandonedMainThreadWork() - box.wasPenalizedBeforeDrain = self.isSnapshotXCTestChannelPenalized(bundleId: self.currentBundleId) + box.wasPenalizedBeforeDrain = self.isSnapshotXCTestChannelPenalized(bundleId: targetBundleId) // 2) `box.payload` above was already produced -- through the capture plan's recovery // tiers -- while the probe is still blocked on `probeReleaseGate`, i.e. recovered before @@ -185,14 +339,14 @@ extension RunnerTests { func testBoundedSystemModalProbeTimeoutRecoversThenReleasesOnDrain() { assertBoundedSystemModalProbeTimeoutRecoversThenReleasesOnDrain(entryPointName: "snapshotFast") { target, options in - try self.snapshotFast(app: target, options: options) + try self.snapshotFast(target: target, options: options) } } func testBoundedSystemModalProbeTimeoutRecoversThenReleasesOnDrainForSnapshotRaw() { assertBoundedSystemModalProbeTimeoutRecoversThenReleasesOnDrain(entryPointName: "snapshotRaw") { target, options in - try self.snapshotRaw(app: target, options: options) + try self.snapshotRaw(target: target, options: options) } } #endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotTimingTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotTimingTests.swift index 24fdcfaa46..b815c96725 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotTimingTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotTimingTests.swift @@ -60,6 +60,51 @@ extension RunnerTests { ) } + /// A tier that stopped starting work at its own deadline is a timeout for the breaker even when it + /// answered fast and with a payload: the partial sweep must arm the penalty exactly as a slow tier + /// does, and a tier that finished fast must not (#2781). + func testDeadlineExhaustedTierAttemptArmsTheChannelPenaltyWithoutBeingSlow() { + let capture = SnapshotBackendCapture( + payload: DataPayload(nodes: [], truncated: true), + effectiveDepth: nil + ) + let exhausted = SnapshotBackendAttempt( + outcome: .captured(capture), + timing: SnapshotCaptureTiming(acquisitionMs: 1_000, presentationMs: 10), + tierOutcome: .deadlineExhausted + ) + XCTAssertEqual( + Self.snapshotXCTestPenaltyReason( + kind: .querySweep, + attempt: exhausted, + slowThresholdMs: 3_000 + ), + "queries_backend_timeout" + ) + + let completed = SnapshotBackendAttempt( + outcome: .captured(capture), + timing: SnapshotCaptureTiming(acquisitionMs: 1_000, presentationMs: 10), + tierOutcome: .completed + ) + XCTAssertNil( + Self.snapshotXCTestPenaltyReason( + kind: .querySweep, + attempt: completed, + slowThresholdMs: 3_000 + ) + ) + + XCTAssertNil( + Self.snapshotXCTestPenaltyReason( + kind: .privateAX, + attempt: exhausted, + slowThresholdMs: 3_000 + ), + "a tier that owes nothing to the XCTest channel cannot penalize it" + ) + } + func testSnapshotPhaseTimerReportsAcquisitionAndPresentationSeparately() { var now = Date(timeIntervalSinceReferenceDate: 100) var timer = SnapshotPhaseTimer(now: { now })