diff --git a/.github/workflows/ios.yml b/.github/workflows/ios.yml index 4bc494a504..00dc9df934 100644 --- a/.github/workflows/ios.yml +++ b/.github/workflows/ios.yml @@ -256,6 +256,10 @@ jobs: -only-testing:AgentDeviceRunnerUITests/RunnerTests/testActionNamesAreCappedPerElementAndReported \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testHungCustomActionReadIsContainedAndRecovers \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAbandonedTreeCaptureSkipsQuerySweepAndHonorsWarmupExemption \ + -only-testing:AgentDeviceRunnerUITests/RunnerTests/testNonInteractiveQuerySweepStopsAtTheSliceItsCallerWaitsFor \ + -only-testing:AgentDeviceRunnerUITests/RunnerTests/testQuerySweepThatEndsOnItsSliceDeadlinePenalizesChannelAndReachesPrivateAX \ + -only-testing:AgentDeviceRunnerUITests/RunnerTests/testCoordinateTapRoutingPenalizesTheIdentityMainSettledOnWhileTheWriteWasPending \ + -only-testing:AgentDeviceRunnerUITests/RunnerTests/testBlockingModalSnapshotLeavesWarmupExemptionForTheFirstCapturePlan \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testPrivateAXPinnedRegularDepthReachesAcquisitionAndPresentation 2>&1 | tee /tmp/agent-device-runner-regressions.log node --input-type=module -e ' import { readFileSync } from "node:fs"; diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift index ed79919b17..13244aeaf2 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift @@ -110,14 +110,16 @@ extension RunnerTests { /// remembered depth deliberately does NOT refresh the TTL, so expiry re-probes the full /// requested depth once per window instead of capping this screen class forever. func recordPrivateAXAcceptedDepth( + bundleId: String?, + processIdentifier: Int?, exactDepthRequested: Bool, effectiveDepth: Int, attemptDepths: [Int] ) { guard !exactDepthRequested, effectiveDepth != attemptDepths.first else { return } rememberPrivateAXAcceptedDepth( - bundleId: currentBundleId, - processIdentifier: currentAppProcessIdentifier, + bundleId: bundleId, + processIdentifier: processIdentifier, depth: effectiveDepth ) } @@ -160,11 +162,12 @@ extension RunnerTests { } func privateAXSnapshotAcquisition( - app: XCUIApplication, + target: SnapshotCaptureTarget, hint: CaptureHint, deadline: Date = .distantFuture ) -> SnapshotAcquisition? { #if os(iOS) && targetEnvironment(simulator) + let app = target.app let requestedDepth = hint.rawTraversalDepth ?? 64 // An explicit --depth request is honored as asked: no accepted-depth // memory, no frontier extension past it. @@ -173,8 +176,8 @@ extension RunnerTests { exactDepthRequested ? nil : rememberedPrivateAXAcceptedDepth( - bundleId: currentBundleId, - processIdentifier: currentAppProcessIdentifier + bundleId: target.bundleId, + processIdentifier: target.processIdentifier ) let attemptDepths = Self.privateAXAttemptDepths( requestedDepth: requestedDepth, @@ -202,6 +205,8 @@ extension RunnerTests { return nil } recordPrivateAXAcceptedDepth( + bundleId: target.bundleId, + processIdentifier: target.processIdentifier, exactDepthRequested: exactDepthRequested, effectiveDepth: effectiveDepth, attemptDepths: attemptDepths @@ -212,7 +217,11 @@ extension RunnerTests { } let rootFrame = privateAXRect(root["frame"]) - let geometry = privateAXSnapshotGeometry(app: app, rootFrame: rootFrame) + let geometry = privateAXSnapshotGeometry( + app: app, + bundleId: target.bundleId, + rootFrame: rootFrame + ) let viewport = geometry.viewport let nodes = privateAXAcquisition( rawRoot: root, @@ -262,8 +271,8 @@ extension RunnerTests { /// grinding on this screen class. Under penalty it reliably burns its full timeout and /// falls back anyway (~1s added to every private AX capture on the Bluesky bench feed), /// so honor the penalty here the same way capture plans do. - func shouldReadPrivateAXViewportViaXCTest() -> Bool { - !hasAbandonedMainThreadWork() && !isSnapshotXCTestChannelPenalized(bundleId: currentBundleId) + func shouldReadPrivateAXViewportViaXCTest(bundleId: String?) -> Bool { + !hasAbandonedMainThreadWork() && !isSnapshotXCTestChannelPenalized(bundleId: bundleId) } /// The geometry this tier may anchor a rotation on. The bridge's own root frame is one more @@ -272,10 +281,11 @@ extension RunnerTests { /// consumers already treat as geometry they cannot measure (#2612). private func privateAXSnapshotGeometry( app: XCUIApplication, + bundleId: String?, rootFrame: CGRect ) -> (viewport: CGRect, interfaceOrientation: Int) { let fallback = rootFrame.isEmpty ? CGRect.infinite : rootFrame - guard shouldReadPrivateAXViewportViaXCTest() else { + guard shouldReadPrivateAXViewportViaXCTest(bundleId: bundleId) else { return (fallback, RunnerInterfaceOrientation.unknown) } do { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandDispatch.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandDispatch.swift index a791984dc1..16b2dfccd2 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandDispatch.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandDispatch.swift @@ -579,8 +579,13 @@ extension RunnerTests { } #endif let probeDeadline = Date().addingTimeInterval(systemModalProbeBudget) + // Routing runs on the command queue, so this hands the probe a target rather than a bundle id it + // read across the main boundary: the penalty an abandoned probe arms carries the identity main + // holds once the probe starts, not one whose write was still queued behind the block that + // occupied main (#2781). return boundedBlockingSystemAlertSnapshot( - deadline: probeDeadline + deadline: probeDeadline, + penaltyTarget: .mainOwnedTarget ) != nil #else return false diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift index 0314cbab9b..8d378a59e5 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift @@ -202,7 +202,7 @@ extension RunnerTests { /// markers. Every site that binds, rebinds, or drops the target runs this. func resetTargetBoundState() { clearRememberedTextEntryTap() - snapshotXCTestPenaltyWarmupExemptionPending = false + snapshotXCTestPenaltyWarmupExemption.isPending = false lastLoggedFastAppGuardLine = nil lastLoggedGesturePolicyLines.removeAll() } @@ -247,7 +247,7 @@ extension RunnerTests { resetTargetBoundState() clearSnapshotXCTestChannelPenalty(reason: "target_process_changed") clearPrivateAXAcceptedDepth(reason: "target_process_changed") - snapshotXCTestPenaltyWarmupExemptionPending = true + snapshotXCTestPenaltyWarmupExemption.isPending = true beginFirstInteractionStabilization() } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift index 14fe138387..3d469bf868 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift @@ -132,6 +132,20 @@ extension RunnerTests { ] static let flatInteractiveFallbackBudget: TimeInterval = 1.0 + /// The least slice time a sweep query may start with. XCTest cannot cancel a query, so one that + /// starts later outlives the slice its caller waits for and holds the main thread (#2783). + static let flatInteractiveQueryBudget: TimeInterval = 0.1 + + /// The deadline the query-sweep tier's caller waits for: one slice, clamped to the plan deadline. + /// Interactive and non-interactive requests share it, since the caller discards a later result. + static func querySweepSliceDeadline(startedAt: Date, planDeadline: Date) -> Date { + min(startedAt.addingTimeInterval(flatInteractiveFallbackBudget), planDeadline) + } + + /// Whether a sweep query started at `now` can still finish before the slice `deadline`. + static func querySweepCanStartQuery(deadline: Date, now: Date) -> Bool { + deadline.timeIntervalSince(now) >= flatInteractiveQueryBudget + } /// What one capture may spend reading the keyboard band before it gives up on the fact and lets the /// tap guard fall back to the tree rule. The scroll path pays this query per gesture and stays well @@ -145,14 +159,17 @@ extension RunnerTests { // `boundedBlockingSystemAlertSnapshot`'s probe closure (see `systemModalProbeOverrideForTesting` // in RunnerTests.swift), so reverting this entry point to bypass the bounded probe fails the // regression test. - func snapshotFast(app: XCUIApplication, options: PresentationOptions) throws -> DataPayload { + func snapshotFast(target: SnapshotCaptureTarget, options: PresentationOptions) throws -> DataPayload { let deadline = Date().addingTimeInterval(Self.snapshotPlanBudget) - if let blocking = boundedBlockingSystemAlertSnapshot(deadline: deadline) { + if let blocking = boundedBlockingSystemAlertSnapshot( + deadline: deadline, + penaltyTarget: .prepared(bundleId: target.bundleId) + ) { return blocking } return try runSnapshotCapturePlan( Self.regularVisiblePlan, - app: app, + target: target, options: options, terminal: .sparseWithFatalOnAXFailure, deadline: deadline @@ -267,14 +284,17 @@ extension RunnerTests { } // See `snapshotFast` above: the single production entry point, no unit-test overload. - func snapshotRaw(app: XCUIApplication, options: PresentationOptions) throws -> DataPayload { + func snapshotRaw(target: SnapshotCaptureTarget, options: PresentationOptions) throws -> DataPayload { let deadline = Date().addingTimeInterval(Self.snapshotPlanBudget) - if let blocking = boundedBlockingSystemAlertSnapshot(deadline: deadline) { + if let blocking = boundedBlockingSystemAlertSnapshot( + deadline: deadline, + penaltyTarget: .prepared(bundleId: target.bundleId) + ) { return blocking } return try runSnapshotCapturePlan( Self.rawDiagnosticPlan, - app: app, + target: target, options: options, terminal: .throwOnAXFailure, deadline: deadline @@ -283,8 +303,15 @@ extension RunnerTests { /// Runs the pre-plan SpringBoard system-modal probe as a bounded capture tier sharing the plan /// deadline, so a slow alert enumeration cannot bypass the snapshot timeout and stall (#1244). - func boundedBlockingSystemAlertSnapshot(deadline: Date) -> DataPayload? { - boundedBlockingSystemAlertSnapshotBody(deadline: deadline) { probeDeadline in + /// An abandoned probe penalizes the XCTest channel for `penaltyTarget`. + func boundedBlockingSystemAlertSnapshot( + deadline: Date, + penaltyTarget: SnapshotProbePenaltyTarget + ) -> DataPayload? { + boundedBlockingSystemAlertSnapshotBody( + deadline: deadline, + penaltyTarget: penaltyTarget + ) { probeDeadline in #if AGENT_DEVICE_RUNNER_UNIT_TESTS if let override = self.systemModalProbeOverrideForTesting { return override(probeDeadline) @@ -301,6 +328,7 @@ extension RunnerTests { /// production runs and what the unit tests exercise. private func boundedBlockingSystemAlertSnapshotBody( deadline: Date, + penaltyTarget: SnapshotProbePenaltyTarget, probe: @escaping (Date) -> DataPayload? ) -> DataPayload? { #if os(macOS) @@ -316,6 +344,7 @@ extension RunnerTests { } let probeDeadline = Date().addingTimeInterval(slice) let startedAt = Date() + let penaltyIdentity = SnapshotProbePenaltyIdentity(penaltyTarget) do { return try runMainThreadWork( "system_modal_probe", @@ -329,12 +358,13 @@ extension RunnerTests { }, onAbandoned: { self.penalizeSnapshotXCTestChannel( - bundleId: self.currentBundleId, + bundleId: penaltyIdentity.penalizedBundleId, reason: "system_modal_probe_timeout" ) } ) { - probe(probeDeadline) + penaltyIdentity.captureFromMain(bundleId: self.currentBundleId) + return probe(probeDeadline) } } catch { NSLog( @@ -413,37 +443,34 @@ extension RunnerTests { func querySweepSnapshotAcquisition( app: XCUIApplication, hint: CaptureHint, - planDeadline: Date = .distantFuture - ) -> SnapshotAcquisition { + sliceDeadline deadline: Date + ) -> (acquisition: SnapshotAcquisition, outcome: SnapshotTierOutcome) { var nodes: [RawAXNode] = [ interactiveRootNode(rect: .zero) ] if hint.rawTraversalDepth == 0 || hint.regularPresentedDepth == 0 { - return SnapshotAcquisition( - hint: hint, - nodes: nodes, - truncated: false, - effectiveDepth: nil, - viewport: .infinite, - interfaceOrientation: RunnerInterfaceOrientation.unknown + return ( + SnapshotAcquisition( + hint: hint, + nodes: nodes, + truncated: false, + effectiveDepth: nil, + viewport: .infinite, + interfaceOrientation: RunnerInterfaceOrientation.unknown + ), + .completed ) } - // Bounded by both its own sweep budget and the umbrella capture-plan deadline, so a - // chained recovery tier can never push the plan past the main-thread watchdog (#1105). - let sweepDeadline = hint.interactiveOnly - ? Date().addingTimeInterval(Self.flatInteractiveFallbackBudget) - : Date.distantFuture - let deadline = min(sweepDeadline, planDeadline) let viewport = safeSnapshotViewport(app: app) var seen = Set() var candidates: [RawAXNode] = [] let flatElements = flatInteractiveElements(app: app, deadline: deadline) - var truncated = flatElements.truncated + var outcome = flatElements.outcome for element in flatElements.elements { - if Date() >= deadline { + if !Self.querySweepCanStartQuery(deadline: deadline, now: Date()) { NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_FLAT_FALLBACK_DEADLINE") - truncated = true + outcome = .deadlineExhausted break } guard let node = flatSnapshotNode(element: element, index: 0, parentIndex: 0) else { @@ -491,20 +518,25 @@ extension RunnerTests { ) ) } - return SnapshotAcquisition( - hint: hint, - nodes: nodes, - truncated: truncated, - effectiveDepth: nil, - viewport: viewport, - interfaceOrientation: RunnerInterfaceOrientation.unknown + return ( + SnapshotAcquisition( + hint: hint, + nodes: nodes, + truncated: outcome == .deadlineExhausted, + effectiveDepth: nil, + viewport: viewport, + interfaceOrientation: RunnerInterfaceOrientation.unknown + ), + outcome ) } func snapshotAccessibilityUnavailable(failure: SnapshotCaptureFailure) -> DataPayload { NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_AX_UNAVAILABLE=%@", failure.message) - runnerAccessibilityHealth = .unavailable - invalidateCachedTarget(reason: Self.axSnapshotUnavailableReason) + applyMainOwnedSnapshotState("ax_unavailable_invalidation") { + self.runnerAccessibilityHealth = .unavailable + self.invalidateCachedTarget(reason: Self.axSnapshotUnavailableReason) + } // This is a planned terminal result, so it carries the structured verdict like every other // planned snapshot — downstream sparse handling keys off the verdict, not node shapes. return sparseTruncatedSnapshotPayload( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift index 2c1d834786..26572f56fa 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift @@ -464,7 +464,7 @@ extension RunnerTests { func flatInteractiveElements( app: XCUIApplication, deadline: Date - ) -> (elements: [XCUIElement], truncated: Bool) { + ) -> (elements: [XCUIElement], outcome: SnapshotTierOutcome) { let queries: [XCUIElementQuery] = [ app.buttons, app.links, @@ -487,23 +487,36 @@ extension RunnerTests { app.images ] - var elements: [XCUIElement] = [] - var truncated = false + return Self.runFlatInteractiveQueries(queries, deadline: deadline) { query in + self.snapshotElementsQuery { + query.allElementsBoundByIndex + } + } + } + + /// Runs sweep queries in order until one reports AX unavailable, or until the next one could not + /// finish before `deadline` (`querySweepCanStartQuery`). A sweep stopped by the deadline reports + /// `.deadlineExhausted` rather than a truncation flag: what it collected is a partial tree, and + /// only the caller that owns the tier decides whether that counts as an answer (#2781). + static func runFlatInteractiveQueries( + _ queries: [Query], + deadline: Date, + now: () -> Date = { Date() }, + run: (Query) -> (elements: [Element], axUnavailable: Bool) + ) -> (elements: [Element], outcome: SnapshotTierOutcome) { + var elements: [Element] = [] for query in queries { - if Date() >= deadline { + if !querySweepCanStartQuery(deadline: deadline, now: now()) { NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_FLAT_FALLBACK_DEADLINE") - truncated = true - break - } - let result = snapshotElementsQuery { - query.allElementsBoundByIndex + return (elements, .deadlineExhausted) } + let result = run(query) elements.append(contentsOf: result.elements) if result.axUnavailable { break } } - return (elements, truncated) + return (elements, .completed) } func snapshotElementsQuery( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift index b6e9946957..e7f3898652 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift @@ -37,6 +37,16 @@ enum SnapshotXCTestChannelPlanState: Equatable { case boundedXCTestProbe } +/// How one tier's bounded work ended. `deadlineExhausted` is a tier timeout: the tier stopped +/// starting work it could not finish inside its own slice, so what it returns is a partial result +/// it never completed collecting. The plan keeps that result only as the fallback and lets the next +/// backend answer; a node count cannot tell the two apart, because a short sweep still collects +/// more than the sparse threshold (#2781). +enum SnapshotTierOutcome: Equatable { + case completed + case deadlineExhausted +} + struct EffectiveSnapshotCapturePlan { let plan: [SnapshotBackendKind] let xCTestChannelState: SnapshotXCTestChannelPlanState @@ -142,12 +152,6 @@ extension RunnerTests { return penalized == bundleId } - func consumeSnapshotXCTestPenaltyWarmupExemption() -> Bool { - let pending = snapshotXCTestPenaltyWarmupExemptionPending - snapshotXCTestPenaltyWarmupExemptionPending = false - return pending - } - /// The pre-seeded first-failure a penalized plan stamps into its verdict. The deferred case /// uses the dedicated 'deferred' code: the breaker pre-selected the backend, nothing new /// degraded on this capture, and the daemon keys warning suppression and the settle budget @@ -260,7 +264,7 @@ extension RunnerTests { func runSnapshotCapturePlan( _ plan: [SnapshotBackendKind], - app: XCUIApplication, + target: SnapshotCaptureTarget, options: PresentationOptions, terminal: SnapshotCaptureTerminalPolicy, deadline: Date? = nil @@ -270,7 +274,7 @@ extension RunnerTests { var axFailure: SnapshotCaptureFailure? // A caller may share the pre-plan system-modal probe's deadline; otherwise own the full budget (#1244). let deadline = deadline ?? Date().addingTimeInterval(Self.snapshotPlanBudget) - let suppressXCTestPenalty = consumeSnapshotXCTestPenaltyWarmupExemption() + let suppressXCTestPenalty = snapshotXCTestPenaltyWarmupExemption.consume() // Reorder is iOS-only because hostile screens can make XCTest tree/query work grind while // the app remains visually responsive. Simulators can avoid that channel through private AX; @@ -281,7 +285,7 @@ extension RunnerTests { var xCTestChannelPenalized = false var xCTestChannelPenalizedByBreaker = false #if os(iOS) - xCTestChannelPenalizedByBreaker = isSnapshotXCTestChannelPenalized(bundleId: currentBundleId) + xCTestChannelPenalizedByBreaker = isSnapshotXCTestChannelPenalized(bundleId: target.bundleId) xCTestChannelPenalized = Self.snapshotXCTestChannelTreatedAsPenalized( penalized: xCTestChannelPenalizedByBreaker, preferredBackend: options.preferredBackend @@ -305,9 +309,9 @@ extension RunnerTests { case .normal: break case .deferredToIndependentBackend: - NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_XCTEST_CHANNEL_DEFERRED bundle=%@", currentBundleId ?? "") + NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_XCTEST_CHANNEL_DEFERRED bundle=%@", target.bundleId ?? "") case .boundedXCTestProbe: - NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_XCTEST_CHANNEL_PROBE_BOUNDED bundle=%@", currentBundleId ?? "") + NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_XCTEST_CHANNEL_PROBE_BOUNDED bundle=%@", target.bundleId ?? "") } for kind in effectivePlan { @@ -332,7 +336,7 @@ extension RunnerTests { } let attempt = try captureWithBackend( kind, - app: app, + target: target, options: options, deadline: deadline, treeCaptureSliceBudgetOverride: effective.treeCaptureSliceBudgetOverride @@ -340,6 +344,7 @@ extension RunnerTests { recordXCTestSnapshotBackendAttemptIfNeeded( kind, attempt: attempt, + bundleId: target.bundleId, penaltySuppressed: suppressXCTestPenalty ) if case let .failed(failure, phase: _) = attempt.outcome { @@ -356,8 +361,19 @@ extension RunnerTests { } guard case let .captured(capture) = attempt.outcome else { continue } - if let sparseReason = Self.sparsePayloadReason(capture.qualityPayload ?? capture.payload) { - if firstFailure == nil { firstFailure = sparseReason } + if attempt.tierOutcome == .deadlineExhausted { + NSLog( + "AGENT_DEVICE_RUNNER_SNAPSHOT_TIER_DEADLINE_EXHAUSTED backend=%@ nodes=%d", + kind.rawValue, + Self.payloadNodeCount(capture.payload) + ) + } + if let rejection = Self.snapshotTierRejectionReason( + outcome: attempt.tierOutcome, + kind: kind, + payload: capture.qualityPayload ?? capture.payload + ) { + if firstFailure == nil { firstFailure = rejection } if Self.payloadNodeCount(capture.payload) > Self.payloadNodeCount(best?.capture.payload) { best = (kind, capture) } @@ -412,19 +428,21 @@ extension RunnerTests { private func captureWithBackend( _ kind: SnapshotBackendKind, - app: XCUIApplication, + target: SnapshotCaptureTarget, options: PresentationOptions, deadline: Date, treeCaptureSliceBudgetOverride: TimeInterval? ) throws -> SnapshotBackendAttempt { + let app = target.app let hint = SnapshotPresentation.captureHint(for: options) var timer = SnapshotPhaseTimer() let acquisition: SnapshotAcquisition? + let tierOutcome: SnapshotTierOutcome // The band is read inside the tree tier's own bounded work, so it has to be lifted out of the // acquisition phase and carried to the stamping step, where the payload is assembled (#2660). var keyboardBand: RunnerKeyboardBandFact? do { - acquisition = try timer.measure(.acquisition) { + let measured = try timer.measure(.acquisition) { () -> (SnapshotAcquisition?, SnapshotTierOutcome) in switch kind { case .recursiveTree: guard @@ -435,10 +453,10 @@ extension RunnerTests { treeCaptureSliceBudgetOverride: treeCaptureSliceBudgetOverride ) else { - return nil + return (nil, .completed) } keyboardBand = context.keyboardBand - return try self.runMainThreadWork( + let tree = try self.runMainThreadWork( "tree_processing", timeout: min(self.treeCaptureSliceBudget, max(0.5, deadline.timeIntervalSinceNow)), timeoutError: self.snapshotMainThreadTimeoutError("processing tree snapshot") @@ -447,26 +465,34 @@ extension RunnerTests { ? try self.rawTreeSnapshotAcquisition(context: context, hint: hint) : try self.recursiveTreeSnapshotAcquisition(context: context, hint: hint) } + return (tree, .completed) case .querySweep: - return try self.runMainThreadWork( + let sliceDeadline = Self.querySweepSliceDeadline(startedAt: Date(), planDeadline: deadline) + let sweep = try self.runMainThreadWork( "query_sweep", - timeout: min(Self.flatInteractiveFallbackBudget, max(0.1, deadline.timeIntervalSinceNow)), + timeout: max(0.1, sliceDeadline.timeIntervalSinceNow), timeoutError: self.snapshotMainThreadTimeoutError("running query-sweep snapshot") ) { self.querySweepSnapshotAcquisition( app: app, hint: hint, - planDeadline: deadline + sliceDeadline: sliceDeadline ) } + return (sweep.acquisition, sweep.outcome) case .privateAX: - return self.privateAXSnapshotAcquisition( - app: app, - hint: hint, - deadline: deadline + return ( + self.privateAXSnapshotAcquisition( + target: target, + hint: hint, + deadline: deadline + ), + .completed ) } } + acquisition = measured.0 + tierOutcome = measured.1 } catch let failure as SnapshotCaptureFailure { return SnapshotBackendAttempt( outcome: .failed(failure, phase: .acquisition), @@ -476,7 +502,8 @@ extension RunnerTests { guard let acquisition else { return SnapshotBackendAttempt( outcome: .noCapture, - timing: timer.timing + timing: timer.timing, + tierOutcome: tierOutcome ) } @@ -511,12 +538,14 @@ extension RunnerTests { } catch let failure as SnapshotPresentationFailure { return SnapshotBackendAttempt( outcome: .failed(Self.snapshotCaptureFailure(for: failure), phase: .presentation), - timing: timer.timing + timing: timer.timing, + tierOutcome: tierOutcome ) } catch let failure as SnapshotCaptureFailure { return SnapshotBackendAttempt( outcome: .failed(failure, phase: .presentation), - timing: timer.timing + timing: timer.timing, + tierOutcome: tierOutcome ) } @@ -525,7 +554,8 @@ extension RunnerTests { capture.keyboardBand = keyboardBand?.payload return SnapshotBackendAttempt( outcome: .captured(capture), - timing: timer.timing + timing: timer.timing, + tierOutcome: tierOutcome ) } @@ -546,6 +576,24 @@ extension RunnerTests { // MARK: Quality classifier (the single source of "is this snapshot degraded") + /// Why a captured tier may not end the plan, or nil when it may. A tier that stopped starting work + /// at its own deadline is rejected as a timeout whatever it collected: keeping its payload as the + /// fallback is right, accepting it is not, and a node count cannot tell a finished capture from a + /// sweep the slice cut short (#2781). + static func snapshotTierRejectionReason( + outcome: SnapshotTierOutcome, + kind: SnapshotBackendKind, + payload: DataPayload + ) -> (reason: String, code: String)? { + if outcome == .deadlineExhausted { + return ( + "the \(kind.rawValue) backend spent its capture slice with the collection unfinished", + "budget" + ) + } + return sparsePayloadReason(payload) + } + /// Returns a degradation reason + machine code when the payload is too degraded to accept. static func sparsePayloadReason(_ payload: DataPayload) -> (reason: String, code: String)? { guard let nodes = payload.nodes, !nodes.isEmpty else { @@ -635,7 +683,10 @@ extension RunnerTests { state: String, reason: (reason: String, code: String)? ) -> DataPayload { - runnerAccessibilityHealth = reason?.code == "ax-rejected" ? .unavailable : .healthy + let health: RunnerAccessibilityHealth = reason?.code == "ax-rejected" ? .unavailable : .healthy + applyMainOwnedSnapshotState("accessibility_health") { + self.runnerAccessibilityHealth = health + } let payload = capture.payload let quality = SnapshotQuality( state: state, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCaptureTarget.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCaptureTarget.swift new file mode 100644 index 0000000000..446db190f2 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCaptureTarget.swift @@ -0,0 +1,106 @@ +import XCTest + +// MARK: - Snapshot capture target (#2781) +// +// Target identity (`currentApp`, `currentBundleId`, `currentAppProcessIdentifier`) and +// `runnerAccessibilityHealth` are owned by main-thread lifecycle code. A capture plan runs on the +// command queue, so it reads the identity from a `SnapshotCaptureTarget` taken on main while the +// command is prepared, and writes health or invalidates the target only through +// `applyMainOwnedSnapshotState`. + +/// The target one capture plan reads, taken once on the main thread. +struct SnapshotCaptureTarget { + let app: XCUIApplication + let bundleId: String? + let processIdentifier: Int? +} + +/// What snapshot command preparation hands the off-main capture. +enum SnapshotCommandPreparation { + case response(Response) + case capture(SnapshotCaptureTarget, systemSurface: SystemSurfaceHost?) +} + +/// The target a bounded XCTest probe arms its abandonment penalty with. +/// +/// The hook that arms the penalty fires on the command queue the moment the probe's slice is spent, +/// while the probe's own work block may still be running on main. `currentBundleId` belongs to main, +/// so it is never read across that boundary: a caller that already took the identity on main hands it +/// over, and a caller that is on the command queue lets the probe's main-side block capture the +/// identity main holds once the work actually starts (#2781). +enum SnapshotProbePenaltyTarget: Equatable { + /// Identity a capture took on main when it prepared its target. + case prepared(bundleId: String?) + /// Read `currentBundleId` inside the probe's main-side block. + case mainOwnedTarget +} + +/// One probe's penalty identity: written by the probe's main-side block, read by the command queue's +/// abandonment hook through this lock. +final class SnapshotProbePenaltyIdentity { + private let lock = NSLock() + private var bundleId: String? + private let readsMainOwnedTarget: Bool + + init(_ target: SnapshotProbePenaltyTarget) { + readsMainOwnedTarget = target == .mainOwnedTarget + if case .prepared(let bundleId) = target { + self.bundleId = bundleId + } + } + + /// Called on the main thread inside the probe's work block, before it enumerates anything, so the + /// identity is the one main had settled on rather than one a queued write is about to replace. + func captureFromMain(bundleId: String?) { + guard readsMainOwnedTarget else { return } + lock.lock() + self.bundleId = bundleId + lock.unlock() + } + + var penalizedBundleId: String? { + lock.lock() + defer { lock.unlock() } + return bundleId + } +} + +extension RunnerTests { + /// Main thread only: reads the lifecycle-owned target identity. + func takeSnapshotCaptureTarget(app: XCUIApplication) -> SnapshotCaptureTarget { + SnapshotCaptureTarget( + app: app, + bundleId: currentBundleId, + processIdentifier: currentAppProcessIdentifier + ) + } + + /// Runs `write` against main-owned runner state for a capture that may be on the command queue. + /// Abandoned work ahead of the hop cannot be cancelled, so behind it the write queues without + /// waiting: the capture answers now and the next command still observes the write. + func applyMainOwnedSnapshotState(_ operation: String, _ write: @escaping () -> Void) { + if Thread.isMainThread { + write() + return + } + guard !hasAbandonedMainThreadWork() else { + NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_STATE_DEFERRED_XCTEST_OCCUPIED operation=%@", operation) + DispatchQueue.main.async(execute: write) + return + } + do { + try runMainThreadWork( + operation, + timeout: 1, + timeoutError: mainThreadExecutionTimeoutError, + write + ) + } catch { + NSLog( + "AGENT_DEVICE_RUNNER_SNAPSHOT_STATE_FAILED operation=%@ error=%@", + operation, + String(describing: error) + ) + } + } +} diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotExecution.swift index d17d79eeba..ec0b2c8faa 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotExecution.swift @@ -9,21 +9,29 @@ extension RunnerTests { } private func executeSnapshotDispatchedOnce(command: Command) throws -> Response { - let preparation = try runMainThreadWork( + let preparation: SnapshotCommandPreparation = try runMainThreadWork( "command_preparation", timeout: mainThreadExecutionTimeout, timeoutError: mainThreadExecutionTimeoutError - ) { - try self.prepareActiveCommandContextSafely(command: command, routeToSpringboard: false) + ) { () -> SnapshotCommandPreparation in + switch try self.prepareActiveCommandContextSafely(command: command, routeToSpringboard: false) { + case .response(let response): + return .response(response) + case .context(let context): + return .capture( + self.takeSnapshotCaptureTarget(app: context.app), + systemSurface: context.systemSurface + ) + } } switch preparation { case .response(let response): return response - case .context(let context): + case .capture(let target, let systemSurface): return try executeSnapshotPrepared( command: command, - activeApp: context.app, - systemSurface: context.systemSurface + target: target, + systemSurface: systemSurface ) } } @@ -76,16 +84,16 @@ extension RunnerTests { private func executeSnapshotPrepared( command: Command, - activeApp: XCUIApplication, + target: SnapshotCaptureTarget, systemSurface: SystemSurfaceHost? ) throws -> Response { let options = Self.presentationOptions(from: command) do { var payload: DataPayload if options.raw { - payload = try snapshotRaw(app: activeApp, options: options) + payload = try snapshotRaw(target: target, options: options) } else { - payload = try snapshotFast(app: activeApp, options: options) + payload = try snapshotFast(target: target, options: options) } if let systemSurface { payload.systemSurface = SystemSurfaceProvenancePayload( @@ -127,25 +135,8 @@ extension RunnerTests { } func invalidateCachedTargetAfterSnapshotFailure() { - // Abandoned work ahead of this hop cannot be cancelled: queue the drop behind it without - // waiting, so the failed capture answers now and the next command still finds the target gone. - guard !hasAbandonedMainThreadWork() else { - NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_INVALIDATION_DEFERRED_XCTEST_OCCUPIED") - DispatchQueue.main.async { - self.invalidateCachedTarget(reason: "ax_snapshot_failure") - } - return - } - do { - try runMainThreadWork( - "target_invalidation", - timeout: 1, - timeoutError: mainThreadExecutionTimeoutError - ) { - self.invalidateCachedTarget(reason: "ax_snapshot_failure") - } - } catch { - NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_INVALIDATION_FAILED=%@", String(describing: error)) + applyMainOwnedSnapshotState("target_invalidation") { + self.invalidateCachedTarget(reason: "ax_snapshot_failure") } } } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotTiming.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotTiming.swift index 288e4cca5a..fd11051e0f 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotTiming.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotTiming.swift @@ -57,6 +57,35 @@ struct SnapshotPhaseTimer { } } +/// Keeps the first capture plan that runs against a fresh target process from penalizing the XCTest +/// channel for a slow tier. Lifecycle code arms and disarms it on main; the capture plan consumes it +/// on the command queue, so a snapshot that returns before running a plan leaves it pending. +final class SnapshotXCTestPenaltyWarmupExemption { + private let lock = NSLock() + private var pending = false + + var isPending: Bool { + get { + lock.lock() + defer { lock.unlock() } + return pending + } + set { + lock.lock() + pending = newValue + lock.unlock() + } + } + + func consume() -> Bool { + lock.lock() + defer { lock.unlock() } + let wasPending = pending + pending = false + return wasPending + } +} + extension RunnerTests { struct SnapshotBackendAttempt { enum Outcome { @@ -69,6 +98,20 @@ extension RunnerTests { /// or error text. let outcome: Outcome let timing: SnapshotCaptureTiming + /// Whether the tier finished collecting or stopped at its own deadline. A tier that stopped at + /// its deadline timed out even when it handed back a payload, so penalty and recovery policy + /// read this instead of classifying the payload (#2781). + let tierOutcome: SnapshotTierOutcome + + init( + outcome: Outcome, + timing: SnapshotCaptureTiming, + tierOutcome: SnapshotTierOutcome = .completed + ) { + self.outcome = outcome + self.timing = timing + self.tierOutcome = tierOutcome + } } /// The penalty breaker observes only acquisition facts. Presentation is a separate phase and @@ -84,6 +127,9 @@ extension RunnerTests { { return "\(kind.rawValue)_backend_timeout" } + if attempt.tierOutcome == .deadlineExhausted { + return "\(kind.rawValue)_backend_timeout" + } guard attempt.timing.acquisitionMs > slowThresholdMs else { return nil } return "slow_\(kind.rawValue)_capture_\(Int(attempt.timing.acquisitionMs))ms" } @@ -91,6 +137,7 @@ extension RunnerTests { func recordXCTestSnapshotBackendAttemptIfNeeded( _ kind: SnapshotBackendKind, attempt: SnapshotBackendAttempt, + bundleId: String?, penaltySuppressed: Bool ) { guard !penaltySuppressed else { return } @@ -102,7 +149,7 @@ extension RunnerTests { ) else { return } penalizeSnapshotXCTestChannel( - bundleId: currentBundleId, + bundleId: bundleId, reason: reason ) } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift index 0785df2b96..5574a47797 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift @@ -52,6 +52,8 @@ final class RunnerTests: XCTestCase { let commandExecutionQueue = DispatchQueue(label: "agent-device.runner.commands") let app = XCUIApplication() lazy var springboard = XCUIApplication(bundleIdentifier: Self.springboardBundleId) + // Main-thread owned, like `runnerAccessibilityHealth`: an off-main capture plan reads them only + // through its `SnapshotCaptureTarget` and writes them only through `applyMainOwnedSnapshotState`. var currentApp: XCUIApplication? var currentBundleId: String? var currentAppProcessIdentifier: Int? @@ -114,7 +116,7 @@ final class RunnerTests: XCTestCase { var snapshotXCTestChannelPenaltyBundleId: String? var snapshotXCTestChannelPenaltyUntil = Date.distantPast let snapshotXCTestChannelPenaltyDuration: TimeInterval = 120 - var snapshotXCTestPenaltyWarmupExemptionPending = false + let snapshotXCTestPenaltyWarmupExemption = SnapshotXCTestPenaltyWarmupExemption() // Sticky per-bundle hint for the private AX depth ladder: deep RN screens reject the default // depth with kAXErrorIllegalArgument on EVERY capture, so once a shallower rung is accepted // later captures start there instead of re-paying the rejected deep request (~300ms per diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AXRecoveryConformanceTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AXRecoveryConformanceTests.swift index 6e6364c310..cde158367b 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AXRecoveryConformanceTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AXRecoveryConformanceTests.swift @@ -426,8 +426,6 @@ extension RunnerTests { XCTAssertFalse(fixture.hintCases.isEmpty) defer { clearPrivateAXAcceptedDepth(reason: "test-cleanup") - currentBundleId = nil - currentAppProcessIdentifier = nil } for hintCase in fixture.hintCases { clearPrivateAXAcceptedDepth(reason: "fixture-case") @@ -440,8 +438,6 @@ extension RunnerTests { let target = try XCTUnwrap(step.target, name) let processIdentifier = try XCTUnwrap( Int(target.generation.filter(\.isNumber)), "\(name): generation must end in digits") - currentBundleId = target.id - currentAppProcessIdentifier = processIdentifier let explicit = step.explicitDepth ?? false let remembered = explicit @@ -459,6 +455,8 @@ extension RunnerTests { XCTAssertEqual(capture.truncated, outcome.complete == false, "\(name): bounded") // The production path records only after a successful ladder, exactly like this. recordPrivateAXAcceptedDepth( + bundleId: target.id, + processIdentifier: processIdentifier, exactDepthRequested: explicit, effectiveDepth: capture.effectiveDepth, attemptDepths: capture.attemptDepths) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AXSnapshotFallbackTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AXSnapshotFallbackTests.swift index cbcad722a9..82f6bec10b 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AXSnapshotFallbackTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AXSnapshotFallbackTests.swift @@ -125,23 +125,22 @@ extension RunnerTests { func testViewportReadSkippedWhileXCTestChannelPenalized() { // Pins the viewport fast path (#1587 review): every penalized private AX capture used to burn // the full 1s main-thread timeout on a doomed viewport read before falling back. - currentBundleId = "xyz.blueskyweb.app" + let bundleId = "xyz.blueskyweb.app" defer { - currentBundleId = nil clearSnapshotXCTestChannelPenalty(reason: "test-cleanup") abandonedMainThreadWorkCount = 0 } - XCTAssertTrue(shouldReadPrivateAXViewportViaXCTest()) + XCTAssertTrue(shouldReadPrivateAXViewportViaXCTest(bundleId: bundleId)) - penalizeSnapshotXCTestChannel(bundleId: "xyz.blueskyweb.app", reason: "test") - XCTAssertFalse(shouldReadPrivateAXViewportViaXCTest()) + penalizeSnapshotXCTestChannel(bundleId: bundleId, reason: "test") + XCTAssertFalse(shouldReadPrivateAXViewportViaXCTest(bundleId: bundleId)) clearSnapshotXCTestChannelPenalty(reason: "test") - XCTAssertTrue(shouldReadPrivateAXViewportViaXCTest()) + XCTAssertTrue(shouldReadPrivateAXViewportViaXCTest(bundleId: bundleId)) abandonedMainThreadWorkCount = 1 - XCTAssertFalse(shouldReadPrivateAXViewportViaXCTest()) + XCTAssertFalse(shouldReadPrivateAXViewportViaXCTest(bundleId: bundleId)) } /// The wire field must reach both capture options AND the backend pin: custom diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandDispatchTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandDispatchTests.swift index dee860248e..e9b6c6a1b2 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandDispatchTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandDispatchTests.swift @@ -71,7 +71,7 @@ extension RunnerTests { currentApp = app currentBundleId = "com.example.stale-target" currentAppProcessIdentifier = 42 - snapshotXCTestPenaltyWarmupExemptionPending = true + snapshotXCTestPenaltyWarmupExemption.isPending = true defer { invalidateCachedTarget(reason: "unit_test_cleanup") app.terminate() @@ -85,7 +85,7 @@ extension RunnerTests { XCTAssertNil(currentApp) XCTAssertNil(currentBundleId) XCTAssertNil(currentAppProcessIdentifier) - XCTAssertFalse(snapshotXCTestPenaltyWarmupExemptionPending) + XCTAssertFalse(snapshotXCTestPenaltyWarmupExemption.isPending) } func testSkipAppActivationPreflightIncludesForegroundCachedCoordinateOnlyTaps() throws { @@ -248,6 +248,79 @@ extension RunnerTests { XCTAssertFalse(hasAbandonedMainThreadWork()) } + /// A coordinate tap resolves its system-modal routing on the command queue while main may still be + /// clearing or rebinding the cached target. Target identity belongs to main, so an abandoned + /// routing probe must arm its penalty with the identity main settled on — never with the identity + /// the command queue read while that write was still pending (#2781). + func testCoordinateTapRoutingPenalizesTheIdentityMainSettledOnWhileTheWriteWasPending() throws { + app.launch() + XCTAssertTrue(app.wait(for: .runningForeground, timeout: 10)) + let pendingBundleId = "com.example.routing-pending-stale" + let settledBundleId = "com.example.routing-pending-settled" + currentApp = app + currentBundleId = pendingBundleId + snapshotXCTestPenaltyWarmupExemption.isPending = false + clearSnapshotXCTestChannelPenalty(reason: "test-setup") + defer { + systemModalProbeOverrideForTesting = nil + clearSnapshotXCTestChannelPenalty(reason: "test-cleanup") + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } + + // Occupy main and rebind the target inside that block: every identity read that arrives while it + // is queued sees a target that main is on its way to replacing. + let mainRelease = DispatchSemaphore(value: 0) + DispatchQueue.main.async { + _ = mainRelease.wait(timeout: .now() + 0.5) + self.currentBundleId = settledBundleId + } + + let probeStarted = expectation(description: "system-modal routing probe started") + let probeReleaseGate = DispatchSemaphore(value: 0) + systemModalProbeOverrideForTesting = { _ in + probeStarted.fulfill() + // Outlives the probe's own slice, so the abandonment hook is what arms the penalty. + _ = probeReleaseGate.wait(timeout: .now() + 15) + return nil + } + let tap = try runnerCommandFixture( + #"{"command":"tap","commandId":"tap-pending-target-write","x":10,"y":20}"# + ) + + final class ResultBox { + var response: Response? + var penalizedPendingIdentity = false + var penalizedSettledIdentity = false + } + let box = ResultBox() + let tapFinished = expectation(description: "off-main tap returned") + DispatchQueue(label: "agent-device.runner.tests.tap-pending-target-write").async { + box.response = try? self.executeDispatched(command: tap) + box.penalizedPendingIdentity = self.isSnapshotXCTestChannelPenalized(bundleId: pendingBundleId) + box.penalizedSettledIdentity = self.isSnapshotXCTestChannelPenalized(bundleId: settledBundleId) + probeReleaseGate.signal() + tapFinished.fulfill() + } + + wait(for: [probeStarted, tapFinished], timeout: 40) + mainRelease.signal() + let drainDeadline = Date().addingTimeInterval(5) + while hasAbandonedMainThreadWork(), Date() < drainDeadline { + sleepFor(0.002) + } + + XCTAssertTrue( + box.penalizedSettledIdentity, + "the abandoned routing probe must penalize the target main settled on" + ) + XCTAssertFalse( + box.penalizedPendingIdentity, + "the command queue may not key a penalty with an identity whose write was still pending on main" + ) + XCTAssertFalse(hasAbandonedMainThreadWork()) + } + func testSkipAppActivationPreflightRejectsSelectorAndMixedSequenceGestures() throws { app.launch() currentApp = app diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LifecycleCacheTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LifecycleCacheTests.swift index 1cf54a6b29..3c1aa5a21c 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LifecycleCacheTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LifecycleCacheTests.swift @@ -101,10 +101,10 @@ extension RunnerTests { } func testSnapshotPenaltyWarmupExemptionIsConsumedOnce() { - snapshotXCTestPenaltyWarmupExemptionPending = true + snapshotXCTestPenaltyWarmupExemption.isPending = true - XCTAssertTrue(consumeSnapshotXCTestPenaltyWarmupExemption()) - XCTAssertFalse(consumeSnapshotXCTestPenaltyWarmupExemption()) + XCTAssertTrue(snapshotXCTestPenaltyWarmupExemption.consume()) + XCTAssertFalse(snapshotXCTestPenaltyWarmupExemption.consume()) } func testSnapshotPenaltyCanBeClearedAcrossTargetProcessReplacement() { @@ -120,14 +120,14 @@ extension RunnerTests { currentApp = app currentBundleId = "com.example.app" currentAppProcessIdentifier = 42 - snapshotXCTestPenaltyWarmupExemptionPending = true + snapshotXCTestPenaltyWarmupExemption.isPending = true invalidateCachedTarget(reason: "unit_test") XCTAssertNil(currentApp) XCTAssertNil(currentBundleId) XCTAssertNil(currentAppProcessIdentifier) - XCTAssertFalse(snapshotXCTestPenaltyWarmupExemptionPending) + XCTAssertFalse(snapshotXCTestPenaltyWarmupExemption.isPending) } func testTextEntryTapWitnessIsBoundToTargetIdentity() { @@ -146,7 +146,7 @@ extension RunnerTests { currentApp = app currentBundleId = "com.example.app" currentAppProcessIdentifier = 42 - snapshotXCTestPenaltyWarmupExemptionPending = true + snapshotXCTestPenaltyWarmupExemption.isPending = true firstInteractionReadyUptime = nil penalizeSnapshotXCTestChannel(bundleId: "com.example.app", reason: "test") XCTAssertTrue(isSnapshotXCTestChannelPenalized(bundleId: "com.example.app")) @@ -157,7 +157,7 @@ extension RunnerTests { XCTAssertNil(currentApp) XCTAssertNil(currentBundleId) XCTAssertNil(currentAppProcessIdentifier) - XCTAssertFalse(snapshotXCTestPenaltyWarmupExemptionPending) + XCTAssertFalse(snapshotXCTestPenaltyWarmupExemption.isPending) XCTAssertFalse(isSnapshotXCTestChannelPenalized(bundleId: "com.example.app")) XCTAssertNotNil(firstInteractionReadyUptime) } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanOccupancyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanOccupancyTests.swift index c4c92d80c7..edec6e8333 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanOccupancyTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanOccupancyTests.swift @@ -30,6 +30,44 @@ private final class RunnerBlockingSnapshotStub: NSObject { } } +/// Records when each `RunnerSlowSweepQueryStub` query started. The swizzled IMP cannot capture +/// test-local state, so each run resets it. +private enum RunnerSlowSweepQueryGate { + /// Shorter than `flatInteractiveQueryBudget`, so a sweep that stops at its slice deadline never + /// outlives the slice; a full sweep of these outlasts the slice, so one that ignores it does. + static let queryDuration: TimeInterval = 0.06 + private static let lock = NSLock() + private static var starts: [Date] = [] + + static func reset() { + lock.lock() + starts = [] + lock.unlock() + } + + static func recordStart() { + lock.lock() + starts.append(Date()) + lock.unlock() + } + + static func recordedStarts() -> [Date] { + lock.lock() + defer { lock.unlock() } + return starts + } +} + +/// Stands in for `-[XCUIElementQuery allElementsBoundByIndex]` so every query the sweep runs holds +/// the main thread for a fixed time and finds nothing. +private final class RunnerSlowSweepQueryStub: NSObject { + @objc var allElementsBoundByIndex: [XCUIElement] { + RunnerSlowSweepQueryGate.recordStart() + Thread.sleep(forTimeInterval: RunnerSlowSweepQueryGate.queryDuration) + return [] + } +} + extension RunnerTests { /// The Bluesky feed shape: the tree XPC grinds past its slice. The plan must recover through /// private AX without queueing the query sweep behind the abandoned XPC, and a fresh process's @@ -60,7 +98,8 @@ extension RunnerTests { _ = capturedInterfaceOrientation(app: app) currentApp = app currentBundleId = "com.callstack.agentdevice.runner.tree-capture-test" - snapshotXCTestPenaltyWarmupExemptionPending = true + snapshotXCTestPenaltyWarmupExemption.isPending = true + let captureTarget = takeSnapshotCaptureTarget(app: app) RunnerBlockingSnapshotGate.release = DispatchSemaphore(value: 0) let originalImplementation = method_getImplementation(snapshotMethod) method_setImplementation(snapshotMethod, method_getImplementation(stubMethod)) @@ -85,7 +124,7 @@ extension RunnerTests { do { box.payload = try self.runSnapshotCapturePlan( Self.regularVisiblePlan, - app: self.app, + target: captureTarget, options: PresentationOptions(interactiveOnly: false, depth: nil, scope: nil, raw: false), terminal: .sparseWithFatalOnAXFailure, deadline: Date().addingTimeInterval(12) @@ -96,7 +135,7 @@ extension RunnerTests { self.mainThreadWorkLock.lock() box.abandonedAfterPlan = self.abandonedMainThreadWorkCount self.mainThreadWorkLock.unlock() - box.penalizedAfterPlan = self.isSnapshotXCTestChannelPenalized(bundleId: self.currentBundleId) + box.penalizedAfterPlan = self.isSnapshotXCTestChannelPenalized(bundleId: captureTarget.bundleId) RunnerBlockingSnapshotGate.release.signal() planned.fulfill() } @@ -131,5 +170,201 @@ extension RunnerTests { return XCTFail("expected the runner idle once the tree XPC drained") } } + + /// A non-interactive query sweep runs its queries on the main thread one at a time. It must stop + /// starting them at the slice deadline its caller waits for, not at the plan deadline, or it holds + /// the main thread after the plan has answered (#2783). + func testNonInteractiveQuerySweepStopsAtTheSliceItsCallerWaitsFor() throws { + let sweepQueryCount = 19 + XCTAssertLessThan(RunnerSlowSweepQueryGate.queryDuration, Self.flatInteractiveQueryBudget) + XCTAssertGreaterThan( + RunnerSlowSweepQueryGate.queryDuration * Double(sweepQueryCount), + Self.flatInteractiveFallbackBudget + ) + guard + let queryMethod = class_getInstanceMethod( + XCUIElementQuery.self, + #selector(getter: XCUIElementQuery.allElementsBoundByIndex) + ), + let stubMethod = class_getInstanceMethod( + RunnerSlowSweepQueryStub.self, + #selector(getter: RunnerSlowSweepQueryStub.allElementsBoundByIndex) + ) + else { + XCTFail("unable to install the slow sweep query stub") + return + } + app.launchArguments = ["--agent-device-selector-read-regression"] + app.launch() + XCTAssertTrue(app.wait(for: .runningForeground, timeout: 10)) + XCTAssertFalse(app.frame.isEmpty) + currentApp = app + currentBundleId = "com.callstack.agentdevice.runner.query-sweep-slice-test" + let captureTarget = takeSnapshotCaptureTarget(app: app) + RunnerSlowSweepQueryGate.reset() + let originalImplementation = method_getImplementation(queryMethod) + method_setImplementation(queryMethod, method_getImplementation(stubMethod)) + defer { + method_setImplementation(queryMethod, originalImplementation) + clearSnapshotXCTestChannelPenalty(reason: "test-cleanup") + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } + + final class ResultBox { + var payload: DataPayload? + var error: Error? + var returnedAt: Date? + var abandonedAtReturn: Bool? + } + let box = ResultBox() + let planned = expectation(description: "query-sweep plan answered") + DispatchQueue(label: "agent-device.runner.tests.query-sweep-slice").async { + do { + box.payload = try self.runSnapshotCapturePlan( + [.querySweep], + target: captureTarget, + options: PresentationOptions(interactiveOnly: false, depth: nil, scope: nil, raw: false), + terminal: .sparseWithFatalOnAXFailure, + deadline: Date().addingTimeInterval(20) + ) + } catch { + box.error = error + } + box.returnedAt = Date() + box.abandonedAtReturn = self.hasAbandonedMainThreadWork() + planned.fulfill() + } + + wait(for: [planned], timeout: 30) + let drainDeadline = Date().addingTimeInterval(3) + while hasAbandonedMainThreadWork(), Date() < drainDeadline { + sleepFor(0.005) + } + let starts = RunnerSlowSweepQueryGate.recordedStarts() + + XCTAssertNil(box.error) + XCTAssertEqual(box.payload?.snapshotQuality?.backend, SnapshotBackendKind.querySweep.rawValue) + XCTAssertEqual( + box.abandonedAtReturn, + false, + "the sweep must finish inside the slice its caller waits for" + ) + let returnedAt = try XCTUnwrap(box.returnedAt) + XCTAssertTrue( + starts.allSatisfy { $0 <= returnedAt }, + "no sweep query may start after the plan answered" + ) + XCTAssertLessThan(starts.count, sweepQueryCount, "the slice must cut the sweep short") + XCTAssertFalse(hasAbandonedMainThreadWork()) + } + + /// A query sweep that stops at its own slice deadline collected a partial tree, not an answer, so + /// it is a tier timeout: the plan must arm the XCTest-channel penalty and let private AX answer, + /// keeping the partial sweep only as the fallback. Accepting it because it carries more nodes than + /// the sparse threshold ships a hierarchy-free capture and leaves every later capture of the same + /// screen to pay for the full sweep again (#2781). + func testQuerySweepThatEndsOnItsSliceDeadlinePenalizesChannelAndReachesPrivateAX() throws { + guard + let queryMethod = class_getInstanceMethod( + XCUIElementQuery.self, + #selector(getter: XCUIElementQuery.allElementsBoundByIndex) + ), + let stubMethod = class_getInstanceMethod( + RunnerSlowSweepQueryStub.self, + #selector(getter: RunnerSlowSweepQueryStub.allElementsBoundByIndex) + ) + else { + XCTFail("unable to install the slow sweep query stub") + return + } + app.launchArguments = ["--agent-device-selector-read-regression"] + app.launch() + XCTAssertTrue(app.wait(for: .runningForeground, timeout: 10)) + currentApp = app + currentBundleId = "com.callstack.agentdevice.runner.query-sweep-timeout-test" + snapshotXCTestPenaltyWarmupExemption.isPending = false + clearSnapshotXCTestChannelPenalty(reason: "test-setup") + RunnerSlowSweepQueryGate.reset() + let captureTarget = takeSnapshotCaptureTarget(app: app) + let originalImplementation = method_getImplementation(queryMethod) + method_setImplementation(queryMethod, method_getImplementation(stubMethod)) + defer { + method_setImplementation(queryMethod, originalImplementation) + clearSnapshotXCTestChannelPenalty(reason: "test-cleanup") + clearPrivateAXAcceptedDepth(reason: "test-cleanup") + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } + + final class ResultBox { + var payload: DataPayload? + var error: Error? + var abandonedAtReturn = false + var penalizedAtReturn = false + var sweepStartsAfterFirstPlan = 0 + var secondPayload: DataPayload? + var secondError: Error? + } + let box = ResultBox() + let planned = expectation(description: "slow-sweep plan answered") + DispatchQueue(label: "agent-device.runner.tests.query-sweep-timeout").async { + do { + box.payload = try self.runSnapshotCapturePlan( + [.querySweep, .privateAX], + target: captureTarget, + options: PresentationOptions(interactiveOnly: false, depth: nil, scope: nil, raw: false), + terminal: .sparseWithFatalOnAXFailure, + deadline: Date().addingTimeInterval(20) + ) + } catch { + box.error = error + } + box.abandonedAtReturn = self.hasAbandonedMainThreadWork() + box.penalizedAtReturn = self.isSnapshotXCTestChannelPenalized(bundleId: captureTarget.bundleId) + box.sweepStartsAfterFirstPlan = RunnerSlowSweepQueryGate.recordedStarts().count + do { + box.secondPayload = try self.runSnapshotCapturePlan( + Self.regularVisiblePlan, + target: self.takeSnapshotCaptureTarget(app: self.app), + options: PresentationOptions(interactiveOnly: false, depth: nil, scope: nil, raw: false), + terminal: .sparseWithFatalOnAXFailure, + deadline: Date().addingTimeInterval(20) + ) + } catch { + box.secondError = error + } + planned.fulfill() + } + + wait(for: [planned], timeout: 60) + XCTAssertNil(box.error) + let quality = box.payload?.snapshotQuality + XCTAssertEqual( + quality?.backend, + SnapshotBackendKind.privateAX.rawValue, + "a sweep that ended on its slice deadline is a tier timeout, not an accepted capture" + ) + XCTAssertEqual(quality?.state, "recovered") + XCTAssertGreaterThan(box.payload?.nodes?.count ?? 0, 1, "private AX answers with a real tree") + XCTAssertFalse(box.abandonedAtReturn, "the sweep must answer inside its own main-thread hop") + XCTAssertTrue( + box.penalizedAtReturn, + "the tier timeout must arm the XCTest-channel penalty for the captured target" + ) + + XCTAssertNil(box.secondError) + XCTAssertEqual( + box.secondPayload?.snapshotQuality?.backend, + SnapshotBackendKind.privateAX.rawValue, + "the armed penalty must defer the next plan's XCTest tiers, not re-run the sweep" + ) + XCTAssertEqual( + RunnerSlowSweepQueryGate.recordedStarts().count, + box.sweepStartsAfterFirstPlan, + "the deferred plan may start no sweep query at all" + ) + XCTAssertFalse(hasAbandonedMainThreadWork()) + } } #endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanTests.swift index f0eae25e74..a2c709eeb9 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanTests.swift @@ -76,6 +76,37 @@ extension RunnerTests { XCTAssertNotNil(Self.sparsePayloadReason(DataPayload(nodes: [], truncated: false))) } + /// A sweep the slice cut short is rejected as a tier timeout even when what it collected clears + /// every bar the quality classifier has, while the identical payload from a sweep that ran all its + /// queries is accepted. The pair is the point: node count cannot see the deadline (#2781). + func testDeadlineExhaustedTierIsRejectedWhileItsIdenticalCompletedPayloadIsAccepted() { + let root = planTestNode(index: 0, type: "Application", label: "Example App", hittable: true) + let nodes: [PresentedNode] = [root] + (1..<13).map { index in + planTestNode(index: index, type: "Button", label: "Row \(index)", hittable: true, parentIndex: 0) + } + let payload = DataPayload(nodes: nodes, truncated: true) + XCTAssertNil( + Self.sparsePayloadReason(payload), + "this payload clears the classifier on its own, so only the tier outcome can reject it" + ) + XCTAssertEqual( + Self.snapshotTierRejectionReason( + outcome: .deadlineExhausted, + kind: .querySweep, + payload: payload + )?.code, + "budget", + "a tier that spent its capture slice is a timeout whatever it collected" + ) + XCTAssertNil( + Self.snapshotTierRejectionReason( + outcome: .completed, + kind: .querySweep, + payload: payload + ) + ) + } + func testCollapsedLeafIndexesFlagsMergedContainersOnly() { let root = planTestNode(index: 0, type: "Application", label: "App") let merged = planTestNode( @@ -460,7 +491,7 @@ extension RunnerTests { func capture(depth: Int?) throws -> DataPayload { try runSnapshotCapturePlan( Self.regularVisiblePlan, - app: app, + target: takeSnapshotCaptureTarget(app: app), options: PresentationOptions( interactiveOnly: false, depth: depth, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCaptureTargetTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCaptureTargetTests.swift new file mode 100644 index 0000000000..2139a30ccb --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCaptureTargetTests.swift @@ -0,0 +1,104 @@ +import XCTest +import AgentDeviceSnapshotPresentation + +extension RunnerTests { +#if AGENT_DEVICE_RUNNER_UNIT_TESTS + func testSnapshotCaptureTargetKeepsPreparedIdentityAndLeavesWarmupExemptionPending() { + currentApp = app + currentBundleId = "com.example.prepared" + currentAppProcessIdentifier = 42 + snapshotXCTestPenaltyWarmupExemption.isPending = true + defer { invalidateCachedTarget(reason: "unit_test_cleanup") } + + let target = takeSnapshotCaptureTarget(app: app) + currentBundleId = "com.example.replaced" + currentAppProcessIdentifier = 43 + + XCTAssertTrue(target.app === app) + XCTAssertEqual(target.bundleId, "com.example.prepared") + XCTAssertEqual(target.processIdentifier, 42) + XCTAssertTrue( + snapshotXCTestPenaltyWarmupExemption.isPending, + "only a capture plan that runs may spend the exemption" + ) + } + +#if os(iOS) + func testBlockingModalSnapshotLeavesWarmupExemptionForTheFirstCapturePlan() throws { + currentApp = app + currentBundleId = "com.example.fresh-process" + currentAppProcessIdentifier = 42 + snapshotXCTestPenaltyWarmupExemption.isPending = true + systemModalProbeOverrideForTesting = { _ in DataPayload(message: "blocking system modal") } + defer { + systemModalProbeOverrideForTesting = nil + runnerAccessibilityHealth = .unknown + invalidateCachedTarget(reason: "unit_test_cleanup") + } + let options = PresentationOptions(interactiveOnly: false, depth: nil, scope: nil, raw: false) + + let fast = try snapshotFast(target: takeSnapshotCaptureTarget(app: app), options: options) + let raw = try snapshotRaw(target: takeSnapshotCaptureTarget(app: app), options: options) + + XCTAssertEqual(fast.message, "blocking system modal") + XCTAssertEqual(raw.message, "blocking system modal") + XCTAssertTrue( + snapshotXCTestPenaltyWarmupExemption.isPending, + "a snapshot answered by the modal probe runs no capture plan, so the exemption stays pending" + ) + + _ = try runSnapshotCapturePlan( + [], + target: takeSnapshotCaptureTarget(app: app), + options: options, + terminal: .sparseWithFatalOnAXFailure + ) + + XCTAssertFalse( + snapshotXCTestPenaltyWarmupExemption.isPending, + "the first capture plan that runs spends the exemption" + ) + } +#endif + + func testProbePenaltyIdentityTakesTheIdentityItsCallerMayLegallyKnow() { + let prepared = SnapshotProbePenaltyIdentity(.prepared(bundleId: "com.example.prepared")) + prepared.captureFromMain(bundleId: "com.example.settled") + XCTAssertEqual( + prepared.penalizedBundleId, + "com.example.prepared", + "a capture penalizes the target it was prepared for, whatever lifecycle binds meanwhile" + ) + + let mainOwned = SnapshotProbePenaltyIdentity(.mainOwnedTarget) + XCTAssertNil( + mainOwned.penalizedBundleId, + "a command-queue caller knows no identity until the probe's main-side block runs" + ) + mainOwned.captureFromMain(bundleId: "com.example.settled") + XCTAssertEqual(mainOwned.penalizedBundleId, "com.example.settled") + } + + func testMainOwnedSnapshotStateWriteRunsOnMainBeforeReturningWhenMainIsFree() { + final class ResultBox { + var ranOnMainThread: Bool? + var appliedBeforeReturn: Bool? + } + let box = ResultBox() + let finished = expectation(description: "off-main write returned") + + DispatchQueue(label: "agent-device.runner.tests.main-owned-snapshot-state").async { + self.applyMainOwnedSnapshotState("unit_test") { + box.ranOnMainThread = Thread.isMainThread + } + box.appliedBeforeReturn = box.ranOnMainThread != nil + finished.fulfill() + } + + wait(for: [finished], timeout: 3) + XCTAssertEqual(box.ranOnMainThread, true) + XCTAssertEqual(box.appliedBeforeReturn, true) + XCTAssertFalse(hasAbandonedMainThreadWork()) + } +#endif +} diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotTests.swift index 83ac2f5a00..c679bbcbde 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotTests.swift @@ -61,6 +61,159 @@ extension RunnerTests { XCTAssertEqual(Self.systemModalProbeSlice(budget: 4, deadlineRemaining: -5), 0) } + func testSnapshotAccessibilityUnavailableQueuesInvalidationBehindAbandonedMainThreadWork() { + currentBundleId = "com.example.stale-target" + runnerAccessibilityHealth = .healthy + defer { + currentBundleId = nil + runnerAccessibilityHealth = .unknown + } + + final class ResultBox { + var elapsed: TimeInterval? + var payload: DataPayload? + var bundleStillCachedWhileBlocked: Bool? + var healthWhileBlocked: RunnerAccessibilityHealth? + var abandonedWhileBlocked: Int? + } + let box = ResultBox() + let mainBlocked = DispatchSemaphore(value: 0) + let releaseMain = DispatchSemaphore(value: 0) + let finished = expectation(description: "fail-closed capture returned while main was blocked") + + DispatchQueue(label: "agent-device.runner.tests.ax-unavailable-invalidation").async { + _ = try? self.runMainThreadWork( + "command_execution", + timeout: 0, + timeoutError: self.mainThreadExecutionTimeoutError + ) { + mainBlocked.signal() + _ = releaseMain.wait(timeout: .now() + 5) + return true + } + _ = mainBlocked.wait(timeout: .now() + 2) + let startedAt = Date() + box.payload = self.snapshotAccessibilityUnavailable( + failure: SnapshotCaptureFailure( + code: Self.axSnapshotErrorCode, + message: Self.axSnapshotFailureMessage, + hint: Self.axSnapshotHint + ) + ) + box.elapsed = Date().timeIntervalSince(startedAt) + box.bundleStillCachedWhileBlocked = self.currentBundleId != nil + box.healthWhileBlocked = self.runnerAccessibilityHealth + self.mainThreadWorkLock.lock() + box.abandonedWhileBlocked = self.abandonedMainThreadWorkCount + self.mainThreadWorkLock.unlock() + releaseMain.signal() + finished.fulfill() + } + + wait(for: [finished], timeout: 8) + let drainDeadline = Date().addingTimeInterval(2) + while hasAbandonedMainThreadWork() || currentBundleId != nil, Date() < drainDeadline { + sleepFor(0.005) + } + + XCTAssertEqual(box.payload?.runnerFatal, true) + XCTAssertLessThan( + box.elapsed ?? .infinity, + 0.5, + "the fail-closed capture must not wait behind abandoned main-thread work" + ) + XCTAssertEqual( + box.bundleStillCachedWhileBlocked, + true, + "the invalidation must queue behind the blocked main thread, not run on the command queue" + ) + XCTAssertEqual( + box.healthWhileBlocked, + .healthy, + "the health write must queue behind the blocked main thread, not run on the command queue" + ) + XCTAssertEqual(box.abandonedWhileBlocked, 1, "the deferred write must not add an abandoned unit") + XCTAssertFalse(hasAbandonedMainThreadWork()) + XCTAssertNil(currentBundleId, "the invalidation must run once the main thread frees") + XCTAssertEqual(runnerAccessibilityHealth, .unavailable) + } + + func testQuerySweepSliceDeadlineIsTheTierSliceNotThePlanDeadline() { + let startedAt = Date(timeIntervalSinceReferenceDate: 1_000) + let slice = startedAt.addingTimeInterval(Self.flatInteractiveFallbackBudget) + + XCTAssertEqual( + Self.querySweepSliceDeadline( + startedAt: startedAt, + planDeadline: startedAt.addingTimeInterval(Self.snapshotPlanBudget) + ), + slice + ) + let nearPlanDeadline = startedAt.addingTimeInterval(Self.flatInteractiveFallbackBudget / 2) + XCTAssertEqual( + Self.querySweepSliceDeadline(startedAt: startedAt, planDeadline: nearPlanDeadline), + nearPlanDeadline + ) + } + + func testQuerySweepStopsAtTheSliceDeadlineBeforeAQueryThatCannotFinish() { + let startedAt = Date(timeIntervalSinceReferenceDate: 1_000) + let sliceDeadline = Self.querySweepSliceDeadline( + startedAt: startedAt, + planDeadline: startedAt.addingTimeInterval(Self.snapshotPlanBudget) + ) + var clock = startedAt + var startedQueries: [Int] = [] + + let sweep = Self.runFlatInteractiveQueries( + Array(0..<19), + deadline: sliceDeadline, + now: { clock } + ) { query -> (elements: [Int], axUnavailable: Bool) in + startedQueries.append(query) + clock = clock.addingTimeInterval(0.475) + return ([query], false) + } + + XCTAssertEqual(startedQueries, [0, 1], "a query must not start with less than one query budget left") + XCTAssertEqual(sweep.elements, [0, 1]) + XCTAssertEqual(sweep.outcome, .deadlineExhausted) + XCTAssertLessThanOrEqual(clock, sliceDeadline, "no started query may outlive the tier slice") + + XCTAssertTrue( + Self.querySweepCanStartQuery(deadline: sliceDeadline, now: sliceDeadline.addingTimeInterval(-0.2)) + ) + XCTAssertFalse( + Self.querySweepCanStartQuery(deadline: sliceDeadline, now: sliceDeadline.addingTimeInterval(-0.05)) + ) + XCTAssertFalse(Self.querySweepCanStartQuery(deadline: sliceDeadline, now: sliceDeadline)) + } + + func testQuerySweepRunsEveryQueryThatFitsAndStopsOnAXUnavailable() { + let startedAt = Date(timeIntervalSinceReferenceDate: 1_000) + let deadline = startedAt.addingTimeInterval(Self.flatInteractiveFallbackBudget) + + let complete = Self.runFlatInteractiveQueries( + [0, 1, 2], + deadline: deadline, + now: { startedAt } + ) { query -> (elements: [Int], axUnavailable: Bool) in ([query], false) } + XCTAssertEqual(complete.elements, [0, 1, 2]) + XCTAssertEqual(complete.outcome, .completed) + + let rejected = Self.runFlatInteractiveQueries( + [0, 1, 2], + deadline: deadline, + now: { startedAt } + ) { query -> (elements: [Int], axUnavailable: Bool) in ([query], query == 1) } + XCTAssertEqual(rejected.elements, [0, 1]) + XCTAssertEqual( + rejected.outcome, + .completed, + "an AX refusal ends the sweep on its own terms, not on the deadline" + ) + } + // Simulator-only: the bounded probe body returns nil on macOS (no SpringBoard host), so the // timeout/penalty/drain machinery below only exists on the iOS branch. #if os(iOS) @@ -83,13 +236,14 @@ extension RunnerTests { /// assertion instead of racing a fixed-timing guess. private func assertBoundedSystemModalProbeTimeoutRecoversThenReleasesOnDrain( entryPointName: String, - callEntryPoint: @escaping (XCUIApplication, PresentationOptions) throws -> DataPayload + callEntryPoint: @escaping (SnapshotCaptureTarget, PresentationOptions) throws -> DataPayload ) { let targetBundleId = "com.callstack.agentdevice.runner.missing.snapshot-timeout-test" let snapshotTarget = XCUIApplication(bundleIdentifier: targetBundleId) let probeReleaseGate = DispatchSemaphore(value: 0) currentApp = snapshotTarget currentBundleId = targetBundleId + let captureTarget = takeSnapshotCaptureTarget(app: snapshotTarget) defer { probeReleaseGate.signal() currentApp = nil @@ -120,7 +274,7 @@ extension RunnerTests { let drained = expectation(description: "\(entryPointName) modal probe drained") DispatchQueue(label: "agent-device.runner.tests.modal-probe-timeout").async { box.payload = try? callEntryPoint( - snapshotTarget, + captureTarget, PresentationOptions(interactiveOnly: false, depth: nil, scope: nil, raw: false) ) @@ -130,7 +284,7 @@ extension RunnerTests { box.wasBusyBeforeDrain = true } box.hadAbandonedCaptureBeforeDrain = self.hasAbandonedMainThreadWork() - box.wasPenalizedBeforeDrain = self.isSnapshotXCTestChannelPenalized(bundleId: self.currentBundleId) + box.wasPenalizedBeforeDrain = self.isSnapshotXCTestChannelPenalized(bundleId: targetBundleId) // 2) `box.payload` above was already produced -- through the capture plan's recovery // tiers -- while the probe is still blocked on `probeReleaseGate`, i.e. recovered before @@ -185,14 +339,14 @@ extension RunnerTests { func testBoundedSystemModalProbeTimeoutRecoversThenReleasesOnDrain() { assertBoundedSystemModalProbeTimeoutRecoversThenReleasesOnDrain(entryPointName: "snapshotFast") { target, options in - try self.snapshotFast(app: target, options: options) + try self.snapshotFast(target: target, options: options) } } func testBoundedSystemModalProbeTimeoutRecoversThenReleasesOnDrainForSnapshotRaw() { assertBoundedSystemModalProbeTimeoutRecoversThenReleasesOnDrain(entryPointName: "snapshotRaw") { target, options in - try self.snapshotRaw(app: target, options: options) + try self.snapshotRaw(target: target, options: options) } } #endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotTimingTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotTimingTests.swift index 24fdcfaa46..b815c96725 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotTimingTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotTimingTests.swift @@ -60,6 +60,51 @@ extension RunnerTests { ) } + /// A tier that stopped starting work at its own deadline is a timeout for the breaker even when it + /// answered fast and with a payload: the partial sweep must arm the penalty exactly as a slow tier + /// does, and a tier that finished fast must not (#2781). + func testDeadlineExhaustedTierAttemptArmsTheChannelPenaltyWithoutBeingSlow() { + let capture = SnapshotBackendCapture( + payload: DataPayload(nodes: [], truncated: true), + effectiveDepth: nil + ) + let exhausted = SnapshotBackendAttempt( + outcome: .captured(capture), + timing: SnapshotCaptureTiming(acquisitionMs: 1_000, presentationMs: 10), + tierOutcome: .deadlineExhausted + ) + XCTAssertEqual( + Self.snapshotXCTestPenaltyReason( + kind: .querySweep, + attempt: exhausted, + slowThresholdMs: 3_000 + ), + "queries_backend_timeout" + ) + + let completed = SnapshotBackendAttempt( + outcome: .captured(capture), + timing: SnapshotCaptureTiming(acquisitionMs: 1_000, presentationMs: 10), + tierOutcome: .completed + ) + XCTAssertNil( + Self.snapshotXCTestPenaltyReason( + kind: .querySweep, + attempt: completed, + slowThresholdMs: 3_000 + ) + ) + + XCTAssertNil( + Self.snapshotXCTestPenaltyReason( + kind: .privateAX, + attempt: exhausted, + slowThresholdMs: 3_000 + ), + "a tier that owes nothing to the XCTest channel cannot penalize it" + ) + } + func testSnapshotPhaseTimerReportsAcquisitionAndPresentationSeparately() { var now = Date(timeIntervalSinceReferenceDate: 100) var timer = SnapshotPhaseTimer(now: { now })