From 5785e206b5589710935386d07598c9d464a65a50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 23 Sep 2026 20:11:23 +0200 Subject: [PATCH 1/2] refactor(move): move the runner's inline unit tests into UnitTests/ Every whole `#if AGENT_DEVICE_RUNNER_UNIT_TESTS` test block in a production runner source moves to `UnitTests/RunnerTests+Tests.swift` under the same guard, with the same test names and bodies. Production sources keep only the guarded test seams (overrides, injected failures, the recorder timestamp accessor, and RunnerTests' stored test properties). Declarations a moved test reads widen from private to internal. Fixture loaders that resolve contracts/fixtures from #filePath step up one more directory, as the existing UnitTests loaders do. Part of #2792 Co-Authored-By: Claude Opus 5.5 (1M context) --- .../RunnerScrollViewportPolicy.swift | 169 +--- .../RunnerSystemSurfaceHostPolicy.swift | 48 -- .../RunnerTapPointPolicy.swift | 50 -- .../RunnerTests+AXSnapshotFallback.swift | 529 ------------- .../RunnerTests+Alert.swift | 8 +- ...rTests+BlockingSystemModalResolution.swift | 36 - .../RunnerTests+CommandExecution.swift | 722 +----------------- .../RunnerTests+CommandJournal.swift | 337 -------- .../RunnerTests+Interaction.swift | 23 - .../RunnerTests+Keyboard.swift | 142 ---- .../RunnerTests+Lifecycle.swift | 30 - .../RunnerTests+MainThreadWork.swift | 92 --- .../RunnerTests+Navigation.swift | 166 ---- .../RunnerTests+PrivateAXPresentation.swift | 209 ----- .../RunnerTests+ScrollGesture.swift | 200 +---- .../RunnerTests+SequenceExecution.swift | 185 ----- .../RunnerTests+Snapshot.swift | 254 +----- ...nerTests+SnapshotBackendCapabilities.swift | 78 -- .../RunnerTests+SnapshotCapturePlan.swift | 492 +----------- .../RunnerTests+SnapshotTiming.swift | 78 +- ...RunnerTests+SynthesizedGesturePolicy.swift | 104 --- ...RunnerTests+TextInputCandidatePolicy.swift | 31 - .../RunnerTests+Transport.swift | 43 +- .../RunnerTests+TvRemote.swift | 22 - .../RunnerTests+AXSnapshotFallbackTests.swift | 531 +++++++++++++ .../UnitTests/RunnerTests+AlertTests.swift | 9 + ...s+BlockingSystemModalResolutionTests.swift | 37 + .../RunnerTests+CommandExecutionTests.swift | 704 +++++++++++++++++ .../RunnerTests+CommandJournalTests.swift | 340 +++++++++ .../RunnerTests+InteractionTests.swift | 27 + .../UnitTests/RunnerTests+KeyboardTests.swift | 144 ++++ .../RunnerTests+LifecycleTests.swift | 30 + .../RunnerTests+MainThreadWorkTests.swift | 93 +++ .../RunnerTests+NavigationTests.swift | 169 ++++ ...nnerTests+PrivateAXPresentationTests.swift | 211 +++++ .../RunnerTests+ScrollGestureTests.swift | 197 +++++ ...unnerTests+ScrollViewportPolicyTests.swift | 165 ++++ .../RunnerTests+SequenceExecutionTests.swift | 186 +++++ ...sts+SnapshotBackendCapabilitiesTests.swift | 80 ++ ...RunnerTests+SnapshotCapturePlanTests.swift | 492 ++++++++++++ .../UnitTests/RunnerTests+SnapshotTests.swift | 245 ++++++ .../RunnerTests+SnapshotTimingTests.swift | 78 ++ ...rTests+SynthesizedGesturePolicyTests.swift | 105 +++ ...erTests+SystemSurfaceHostPolicyTests.swift | 50 ++ .../RunnerTests+TapPointPolicyTests.swift | 52 ++ ...rTests+TextInputCandidatePolicyTests.swift | 33 + .../RunnerTests+TransportTests.swift | 43 ++ .../UnitTests/RunnerTests+TvRemoteTests.swift | 25 + apple/runner/README.md | 2 + packages/contracts/src/scroll-gesture.test.ts | 7 +- .../src/snapshot-tap-point-policy.test.ts | 7 +- 51 files changed, 4083 insertions(+), 4027 deletions(-) create mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AXSnapshotFallbackTests.swift create mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertTests.swift create mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+BlockingSystemModalResolutionTests.swift create mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandExecutionTests.swift create mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandJournalTests.swift create mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+InteractionTests.swift create mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+KeyboardTests.swift create mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+MainThreadWorkTests.swift create mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift create mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+PrivateAXPresentationTests.swift create mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScrollGestureTests.swift create mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScrollViewportPolicyTests.swift create mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SequenceExecutionTests.swift create mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotBackendCapabilitiesTests.swift create mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanTests.swift create mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotTests.swift create mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotTimingTests.swift create mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGesturePolicyTests.swift create mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SystemSurfaceHostPolicyTests.swift create mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TapPointPolicyTests.swift create mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextInputCandidatePolicyTests.swift create mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TransportTests.swift create mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TvRemoteTests.swift diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerScrollViewportPolicy.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerScrollViewportPolicy.swift index 18c8638222..a2dad024e4 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerScrollViewportPolicy.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerScrollViewportPolicy.swift @@ -12,9 +12,10 @@ import AgentDeviceSnapshotPresentation // // The pure rule below is geometry on purpose — no XCUIApplication — so its exact decision is proven // against the golden table shared with its TS twin, `clipScrollViewportAboveKeyboard` in -// packages/contracts/src/scroll-gesture.ts, asserted in that file's test beside this one. The table -// carries only frames representable in both languages: `CGRect` standardizes a negative extent into a -// positive height at a moved origin, so a negative `height` is tested on the TS side alone. +// packages/contracts/src/scroll-gesture.ts, asserted in that file's test and in +// UnitTests/RunnerTests+ScrollViewportPolicyTests.swift. The table carries only frames +// representable in both languages: `CGRect` standardizes a negative extent into a positive height +// at a moved origin, so a negative `height` is tested on the TS side alone. // // The `extension RunnerTests` below is the one impure caller: it reads the runner's own live keyboard // frame, because a frame threaded from the daemon would predate the keyboard. @@ -248,165 +249,3 @@ extension RunnerTests { } #endif } - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS -private struct ScrollViewportPolicyFixture: Decodable { - struct Frame: Decodable { - let x: Double - let y: Double - let width: Double - let height: Double - - var cgRect: CGRect { - CGRect(x: x, y: y, width: width, height: height) - } - } - - struct Constants: Decodable { - let minVisibleFraction: Double - let accessoryAllowance: Double - } - - struct Expected: Decodable { - let kind: String - let viewport: Frame? - let keyboardMinY: Double? - let visibleHeight: Double? - } - - struct TestCase: Decodable { - let name: String - let viewport: Frame - let keyboard: Frame - let expected: Expected - } - - let constants: Constants - let cases: [TestCase] -} - -extension RunnerTests { - /// Golden parity table (#2500): every case in contracts/fixtures/scroll-keyboard-policy.json must - /// agree with the vitest twin. Add cases there, never fork the rule. - func testScrollViewportKeyboardClipMatchesGoldenParityTable() throws { - let fixture = try loadScrollViewportPolicyFixture() - XCTAssertFalse(fixture.cases.isEmpty, "parity table must not be empty") - for testCase in fixture.cases { - let clip = ScrollViewportPolicy.clip( - viewport: testCase.viewport.cgRect, - keyboard: testCase.keyboard.cgRect - ) - switch testCase.expected.kind { - case "unobstructed": - XCTAssertEqual(clip, .unobstructed, testCase.name) - case "avoided": - let expectedFrame = try XCTUnwrap(testCase.expected.viewport, testCase.name).cgRect - let expectedMinY = try XCTUnwrap(testCase.expected.keyboardMinY, testCase.name) - XCTAssertEqual( - clip, - .avoided(frame: expectedFrame, keyboardMinY: expectedMinY), - testCase.name - ) - case "occluded": - let expectedMinY = try XCTUnwrap(testCase.expected.keyboardMinY, testCase.name) - let expectedVisibleHeight = try XCTUnwrap(testCase.expected.visibleHeight, testCase.name) - XCTAssertEqual( - clip, - .occluded(keyboardMinY: expectedMinY, visibleHeight: expectedVisibleHeight), - testCase.name - ) - default: - XCTFail("unknown expected kind `\(testCase.expected.kind)` in \(testCase.name)") - } - } - } - - /// The thresholds are the table's, not this file's. The refusal reason and the runner code are - /// each one side's own vocabulary: the reason is what the host publishes, the code is what this - /// runner answers with, and neither is a shared clip constant. - func testScrollViewportPolicyUsesParityTableConstants() throws { - let constants = try loadScrollViewportPolicyFixture().constants - XCTAssertEqual(constants.minVisibleFraction, ScrollViewportPolicy.minVisibleFraction) - XCTAssertEqual(constants.accessoryAllowance, ScrollViewportPolicy.accessoryAllowance) - } - - /// A clipped landscape band shortens the frame, and `CoordinateSpaceRotation.native(point:)` derives a - /// `landscapeRight` native x from the frame's HEIGHT. Rotating inside the band therefore moves the - /// dispatched path sideways by exactly what the keyboard took, off the lane the plan was built for, - /// so the plan band and the coordinate basis stay separate values through dispatch (#2500). - func testScrollViewportDispatchKeepsTheUnclippedFrameAsItsCoordinateRotationBasis() throws { - let viewport = CGRect(x: 0, y: 0, width: 1210, height: 834) - let keyboard = CGRect(x: 0, y: 588, width: 1210, height: 246) - let clip = ScrollViewportPolicy.clip(viewport: viewport, keyboard: keyboard) - guard case .avoided(let band, let keyboardMinY) = clip else { - return XCTFail("expected a landscape keyboard to be avoided, got \(clip)") - } - XCTAssertEqual(band.height, 576) - - guard case .gesture(let gesture) = ScrollViewportPolicy.frames( - referenceFrame: viewport, - clip: clip - ).gestureDispatch(direction: .up, amount: nil, pixels: nil) else { - return XCTFail("expected a gesture inside the clipped band") - } - XCTAssertEqual(gesture.planFrame, band) - XCTAssertEqual(gesture.keyboardMinY, keyboardMinY) - XCTAssertEqual(gesture.coordinateFrame, viewport, "the rotation basis must survive the clip") - XCTAssertLessThanOrEqual( - max(gesture.plan.y1, gesture.plan.y2), - keyboard.minY - ScrollViewportPolicy.accessoryAllowance, - "a landscape swipe must stay clear of the keys" - ) - - let reported = gesture.attachingEvidence( - to: Response( - ok: true, - data: DataPayload(referenceWidth: viewport.width, referenceHeight: viewport.height), - error: nil - ) - ) - XCTAssertEqual( - reported.data?.referenceHeight, - band.height, - "the payload names the band the plan ran inside, not the synthesis frame" - ) - XCTAssertEqual(reported.data?.referenceWidth, viewport.width) - XCTAssertEqual(reported.data?.keyboardMinY, keyboardMinY) - XCTAssertEqual(reported.data?.keyboardAvoided, true) - - let orientedStartY = gesture.planFrame.minY + gesture.plan.y1 - let dispatchedFromViewport = CoordinateSpaceRotation.native( - point: CGPoint(x: gesture.planFrame.minX + gesture.plan.x1, y: orientedStartY), - in: gesture.coordinateFrame, - interfaceOrientation: RunnerInterfaceOrientation.landscapeRight - ) - let dispatchedFromBand = CoordinateSpaceRotation.native( - point: CGPoint(x: gesture.planFrame.minX + gesture.plan.x1, y: orientedStartY), - in: gesture.planFrame, - interfaceOrientation: RunnerInterfaceOrientation.landscapeRight - ) - XCTAssertEqual( - dispatchedFromViewport.x - dispatchedFromBand.x, - viewport.height - band.height, - accuracy: 0.001, - "rotating inside the clipped band would shift native x by what the keyboard took" - ) - } - - private func loadScrollViewportPolicyFixture() throws -> ScrollViewportPolicyFixture { - let fixtureURL = URL(fileURLWithPath: #filePath) - .deletingLastPathComponent() // AgentDeviceRunnerUITests - .deletingLastPathComponent() // AgentDeviceRunner - .deletingLastPathComponent() // runner - .deletingLastPathComponent() // apple - .deletingLastPathComponent() // repo root - .appendingPathComponent("contracts") - .appendingPathComponent("fixtures") - .appendingPathComponent("scroll-keyboard-policy.json") - return try JSONDecoder().decode( - ScrollViewportPolicyFixture.self, - from: Data(contentsOf: fixtureURL) - ) - } -} -#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSystemSurfaceHostPolicy.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSystemSurfaceHostPolicy.swift index 97dd753e28..9db900e9e3 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSystemSurfaceHostPolicy.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSystemSurfaceHostPolicy.swift @@ -30,51 +30,3 @@ enum SystemSurfaceHostRegistry { host(forBundleId: bundleId) != nil } } - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS -import XCTest - -private struct SystemSurfaceHostFixture: Decodable { - struct Host: Decodable { - let bundleId: String - let kind: String - } - let hosts: [Host] -} - -extension RunnerTests { - func testSystemSurfaceHostRegistryMirrorsGoldenFixture() throws { - let fixtureURL = URL(fileURLWithPath: #filePath) - .deletingLastPathComponent() // AgentDeviceRunnerUITests - .deletingLastPathComponent() // AgentDeviceRunner - .deletingLastPathComponent() // runner - .deletingLastPathComponent() // apple - .deletingLastPathComponent() // repo root - .appendingPathComponent("contracts") - .appendingPathComponent("fixtures") - .appendingPathComponent("ios-system-surface-hosts.json") - let fixture = try JSONDecoder().decode( - SystemSurfaceHostFixture.self, - from: Data(contentsOf: fixtureURL) - ) - let registry = SystemSurfaceHostRegistry.hosts.map { [$0.bundleId, $0.kind.rawValue] } - let golden = fixture.hosts.map { [$0.bundleId, $0.kind] } - XCTAssertEqual(registry, golden, "SystemSurfaceHostRegistry drifted from the golden fixture") - } - - func testSystemSurfaceHostRegistryRecognizesRegisteredHosts() { - XCTAssertTrue(SystemSurfaceHostRegistry.isSystemSurfaceHost("com.apple.SafariViewService")) - XCTAssertTrue(SystemSurfaceHostRegistry.isSystemSurfaceHost("com.apple.PassbookUIService")) - XCTAssertFalse(SystemSurfaceHostRegistry.isSystemSurfaceHost("com.example.app")) - XCTAssertFalse(SystemSurfaceHostRegistry.isSystemSurfaceHost(nil)) - XCTAssertEqual( - SystemSurfaceHostRegistry.host(forBundleId: "com.apple.SafariViewService")?.kind, - .webAuth - ) - XCTAssertEqual( - SystemSurfaceHostRegistry.host(forBundleId: "com.apple.PassbookUIService")?.kind, - .payment - ) - } -} -#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTapPointPolicy.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTapPointPolicy.swift index 379f78b734..33dde24083 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTapPointPolicy.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTapPointPolicy.swift @@ -25,53 +25,3 @@ enum TapPointPolicy { && centerY >= windowFrame.minY && centerY <= windowFrame.maxY } } - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS -private struct TapPointPolicyFixture: Decodable { - struct Frame: Decodable { - let x: Double - let y: Double - let width: Double - let height: Double - - var cgRect: CGRect { - CGRect(x: x, y: y, width: width, height: height) - } - } - - let name: String - let elementFrame: Frame - let windowFrame: Frame - let allowed: Bool -} - -extension RunnerTests { - // Golden parity table (ADR 0011 Layer 2): every case in - // contracts/fixtures/tap-point-policy.json must agree with the vitest twin - // (tap-point-policy-parity.test.ts). Add cases there, never fork the rule. - func testTapPointPolicyMatchesGoldenParityTable() throws { - let fixtureURL = URL(fileURLWithPath: #filePath) - .deletingLastPathComponent() // AgentDeviceRunnerUITests - .deletingLastPathComponent() // AgentDeviceRunner - .deletingLastPathComponent() // runner - .deletingLastPathComponent() // apple - .deletingLastPathComponent() // repo root - .appendingPathComponent("contracts") - .appendingPathComponent("fixtures") - .appendingPathComponent("tap-point-policy.json") - let data = try Data(contentsOf: fixtureURL) - let cases = try JSONDecoder().decode([TapPointPolicyFixture].self, from: data) - XCTAssertFalse(cases.isEmpty, "parity table must not be empty") - for fixture in cases { - XCTAssertEqual( - TapPointPolicy.isAllowed( - elementFrame: fixture.elementFrame.cgRect, - windowFrame: fixture.windowFrame.cgRect - ), - fixture.allowed, - fixture.name - ) - } - } -} -#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift index f77e0f6035..ed79919b17 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift @@ -317,532 +317,3 @@ extension RunnerTests { return nil } } - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS -// MARK: - In-bundle unit tests - -extension RunnerTests { - func testPrivateAXAttemptDepthsAppliesRememberedDepth() { - XCTAssertEqual( - Self.privateAXAttemptDepths(requestedDepth: 64, rememberedDepth: nil), - [64, 56, 40, 24, 12] - ) - XCTAssertEqual( - Self.privateAXAttemptDepths(requestedDepth: 64, rememberedDepth: 56), - [56, 40, 24, 12] - ) - XCTAssertEqual(Self.privateAXAttemptDepths(requestedDepth: 64, rememberedDepth: 12), [12]) - // Remembered at/above the requested depth changes nothing. - XCTAssertEqual( - Self.privateAXAttemptDepths(requestedDepth: 64, rememberedDepth: 64), - [64, 56, 40, 24, 12] - ) - // A shallower explicit request keeps its own rungs; deeper stale memory is ignored. - XCTAssertEqual(Self.privateAXAttemptDepths(requestedDepth: 24, rememberedDepth: 56), [24, 12]) - } - - /// Executed producer contract for the #1627 review blocker: a frontier whose - /// live element vanished, and one whose re-rooted request fails, must BOTH - /// count as missed — an all-miss extension reporting itself drained would - /// present a capped capture as complete. Goes red if either miss-path - /// increment in extendSnapshotFrontiers is removed. - func testDeepExtensionCountsMissedFrontiers() { - // Element vanished (list churn between serialization and extension): the - // fabricated snapshot answers nil for accessibilityElement — missed, and - // no request call is consumed. (An explicit nil property: bare NSObject - // resolves the key through a UIKit category and would take the call path.) - let orphan = RunnerAXSnapshotFrontier() - orphan.snapshot = FrontierSnapshotWithoutElementForTesting() - orphan.node = NSMutableDictionary() - // Re-rooted request fails: the element resolves but the client cannot - // serve requestSnapshotForElement — one consumed call AND a miss. - let unreachable = RunnerAXSnapshotFrontier() - unreachable.snapshot = FrontierSnapshotWithElementForTesting() - unreachable.node = NSMutableDictionary() - - var nodeCount = 0 - var truncated = ObjCBool(false) - let outcome = RunnerAXSnapshotBridge.extend( - NSMutableArray(array: [orphan, unreachable]), - axClient: NSObject(), - attributes: [], - maxDepth: 56, - maxNodes: 5_000, - nodeCount: &nodeCount, - truncated: &truncated, - callsAllowed: 8, - mergedLeaves: nil, - deadline: nil - ) - - XCTAssertEqual(outcome?[RunnerAXSnapshotDeepExtensionMissedKey] as? Int, 2) - XCTAssertEqual(outcome?[RunnerAXSnapshotDeepExtensionCallsKey] as? Int, 1) - XCTAssertEqual(outcome?[RunnerAXSnapshotDeepExtensionPendingKey] as? Int, 0) - XCTAssertEqual(outcome?[RunnerAXSnapshotDeepExtensionNodesAddedKey] as? Int, 0) - XCTAssertFalse(truncated.boolValue) - // And the consumer verdict over exactly this outcome: still depth-limited. - XCTAssertTrue( - Self.privateAXDepthLimited( - effectiveDepth: 56, requestedDepth: 64, pendingFrontiers: 0, missedFrontiers: 2)) - } - - func testPrivateAXDepthLimitedRequiresEveryFrontierResolved() { - // Un-capped capture is never depth-limited, extension or not. - XCTAssertFalse( - Self.privateAXDepthLimited( - effectiveDepth: 64, requestedDepth: 64, pendingFrontiers: nil, missedFrontiers: nil)) - // Capped with no extension outcome (never ran) stays depth-limited. - XCTAssertTrue( - Self.privateAXDepthLimited( - effectiveDepth: 56, requestedDepth: 64, pendingFrontiers: nil, missedFrontiers: nil)) - // Fully drained extension clears the verdict. - XCTAssertFalse( - Self.privateAXDepthLimited( - effectiveDepth: 56, requestedDepth: 64, pendingFrontiers: 0, missedFrontiers: 0)) - // Budget exhaustion (pending frontiers) keeps it. - XCTAssertTrue( - Self.privateAXDepthLimited( - effectiveDepth: 56, requestedDepth: 64, pendingFrontiers: 2, missedFrontiers: 0)) - // The #1627 review blocker: an all-miss extension (elements vanished or - // re-rooted requests failed) resolved nothing — it must NOT present the - // capture as complete just because the queue emptied. - XCTAssertTrue( - Self.privateAXDepthLimited( - effectiveDepth: 56, requestedDepth: 64, pendingFrontiers: 0, missedFrontiers: 8)) - } - - func testPrivateAXAcceptedDepthMemoryMatchesBundleProcessAndExpires() { - defer { clearPrivateAXAcceptedDepth(reason: "test-cleanup") } - - rememberPrivateAXAcceptedDepth(bundleId: "xyz.blueskyweb.app", processIdentifier: 111, depth: 56) - XCTAssertEqual( - rememberedPrivateAXAcceptedDepth(bundleId: "xyz.blueskyweb.app", processIdentifier: 111), - 56 - ) - XCTAssertNil(rememberedPrivateAXAcceptedDepth(bundleId: "com.other.app", processIdentifier: 111)) - // A relaunch changes the PID; the new process must re-probe the full depth even inside the - // TTL, and an unknown current PID (post-invalidation) must never match. - XCTAssertNil(rememberedPrivateAXAcceptedDepth(bundleId: "xyz.blueskyweb.app", processIdentifier: 222)) - XCTAssertNil(rememberedPrivateAXAcceptedDepth(bundleId: "xyz.blueskyweb.app", processIdentifier: nil)) - - // Expired memory stops applying (the expiry re-probes the full requested depth). - privateAXAcceptedDepthUntil = Date(timeIntervalSinceNow: -1) - XCTAssertNil(rememberedPrivateAXAcceptedDepth(bundleId: "xyz.blueskyweb.app", processIdentifier: 111)) - } - - func testPrivateAXAcceptedDepthMemoryRequiresProcessIdentifierToRecord() { - defer { clearPrivateAXAcceptedDepth(reason: "test-cleanup") } - - rememberPrivateAXAcceptedDepth(bundleId: "xyz.blueskyweb.app", processIdentifier: nil, depth: 56) - XCTAssertNil( - rememberedPrivateAXAcceptedDepth(bundleId: "xyz.blueskyweb.app", processIdentifier: nil) - ) - } - - func testViewportReadSkippedWhileXCTestChannelPenalized() { - // Pins the viewport fast path (#1587 review): every penalized private AX capture used to burn - // the full 1s main-thread timeout on a doomed viewport read before falling back. - currentBundleId = "xyz.blueskyweb.app" - defer { - currentBundleId = nil - clearSnapshotXCTestChannelPenalty(reason: "test-cleanup") - abandonedMainThreadWorkCount = 0 - } - - XCTAssertTrue(shouldReadPrivateAXViewportViaXCTest()) - - penalizeSnapshotXCTestChannel(bundleId: "xyz.blueskyweb.app", reason: "test") - XCTAssertFalse(shouldReadPrivateAXViewportViaXCTest()) - - clearSnapshotXCTestChannelPenalty(reason: "test") - XCTAssertTrue(shouldReadPrivateAXViewportViaXCTest()) - - abandonedMainThreadWorkCount = 1 - XCTAssertFalse(shouldReadPrivateAXViewportViaXCTest()) - } - - /// The wire field must reach both capture options AND the backend pin: custom - /// actions are only readable through the private AX client, so a capture that - /// asked for them but planned the XCTest tree backend would return a payload - /// that structurally cannot carry them. - func testCustomActionsRequestPinsPrivateAXBackend() throws { - let asked = try JSONDecoder().decode( - Command.self, from: Data(#"{"command":"snapshot","customActions":true}"#.utf8)) - let options = Self.presentationOptions(from: asked) - XCTAssertTrue(options.customActions) - XCTAssertEqual(options.preferredBackend, SnapshotBackendKind.privateAX.rawValue) - XCTAssertTrue( - Self.snapshotXCTestChannelTreatedAsPenalized( - penalized: false, preferredBackend: options.preferredBackend)) - - // An explicit pin is never overwritten by the implied one. - let pinned = try JSONDecoder().decode( - Command.self, - from: Data(#"{"command":"snapshot","customActions":true,"preferredBackend":"tree"}"#.utf8)) - XCTAssertEqual(Self.presentationOptions(from: pinned).preferredBackend, "tree") - - // And the default capture neither asks nor pins. - let bare = try JSONDecoder().decode(Command.self, from: Data(#"{"command":"snapshot"}"#.utf8)) - XCTAssertFalse(Self.presentationOptions(from: bare).customActions) - XCTAssertNil(Self.presentationOptions(from: bare).preferredBackend) - } - - /// A request-pinned backend degraded nothing, so its verdict must not claim - /// slow accessibility work — that reason drives a user-facing warning. - func testRequestPinnedBackendReportsItsOwnReason() { - let requested = Self.xcTestChannelStateFirstFailure( - .deferredToIndependentBackend, requestPinnedBackend: true) - XCTAssertEqual(requested?.code, "requested-backend") - XCTAssertFalse(requested?.reason.contains("slow accessibility work") ?? true) - - // The circuit breaker's own deferral keeps its established code and wording. - let breaker = Self.xcTestChannelStateFirstFailure(.deferredToIndependentBackend) - XCTAssertEqual(breaker?.code, "deferred") - - // The bounded probe and the healthy plan are untouched by the new flag. - XCTAssertEqual( - Self.xcTestChannelStateFirstFailure(.boundedXCTestProbe, requestPinnedBackend: true)?.code, - "budget") - XCTAssertNil(Self.xcTestChannelStateFirstFailure(.normal, requestPinnedBackend: true)) - } - - /// The disclosure only exists if the counts survive the bridge boundary, and - /// "did not ask" must stay distinguishable from "read none". - func testCustomActionCoverageParsesOnlyCompletePairs() { - let complete: [String: Any] = [ - RunnerAXSnapshotCustomActionsReadKey: 12, - RunnerAXSnapshotCustomActionsCandidatesKey: 19, - RunnerAXSnapshotCustomActionsTruncatedKey: 2, - RunnerAXSnapshotCustomActionsBlockedKey: true, - ] - let coverage = Self.privateAXCustomActionCoverage(complete) - XCTAssertEqual(coverage?.read, 12) - XCTAssertEqual(coverage?.candidates, 19) - XCTAssertEqual(coverage?.truncated, 2) - XCTAssertEqual(coverage?.blocked, true) - - // Absent key = the capture never asked; it must not read as (0, 0), which - // would warn "0 of 0" on every default capture. - XCTAssertNil(Self.privateAXCustomActionCoverage(nil)) - // The bridge in this target always writes all four keys, so a partial - // dictionary is malformed and is dropped whole. - for key in complete.keys { - var partial = complete - partial.removeValue(forKey: key) - XCTAssertNil(Self.privateAXCustomActionCoverage(partial), "missing \(key)") - } - } - - /// The AX call cannot be cancelled once issued, so the read deadline frees - /// only the caller — the call keeps running. Without containment, repeating - /// `snapshot --actions` against a wedged element would stack orphaned reads, - /// all sharing one XCAXClient. This pins the containment: one serial queue and - /// a single-flight refusal that adds no work while a read is outstanding. - func testHungCustomActionReadIsContainedAndRecovers() { - let hung = HungAXClientForTesting() - let element = NSObject() - let dispatchesBefore = RunnerAXSnapshotBridge.customActionReadDispatchCount() - let blockedBefore = RunnerAXSnapshotBridge.customActionReadBlockedCount() - defer { hung.release() } - - // 1. First read wedges. The caller is freed by the deadline, but the call is - // still out there, so it stays counted in flight. - var completed = ObjCBool(true) - let firstStarted = Date() - let first = RunnerAXSnapshotBridge.customActionNames( - forElement: element, axClient: hung, completed: &completed) - XCTAssertNil(first) - XCTAssertFalse(completed.boolValue) - XCTAssertGreaterThanOrEqual(-firstStarted.timeIntervalSinceNow, 0.9) - XCTAssertEqual(RunnerAXSnapshotBridge.customActionReadsInFlight(), 1) - XCTAssertEqual( - RunnerAXSnapshotBridge.customActionReadDispatchCount(), dispatchesBefore + 1) - - // 2. Repeats do NOT accumulate: no new dispatch, still exactly one in - // flight, and every repeat is refused by single-flight admission. - for _ in 0..<5 { - XCTAssertNil( - RunnerAXSnapshotBridge.customActionNames( - forElement: element, axClient: hung, completed: &completed)) - XCTAssertFalse(completed.boolValue) - } - XCTAssertEqual(RunnerAXSnapshotBridge.customActionReadsInFlight(), 1) - XCTAssertEqual( - RunnerAXSnapshotBridge.customActionReadDispatchCount(), dispatchesBefore + 1) - XCTAssertEqual(RunnerAXSnapshotBridge.customActionReadBlockedCount(), blockedBefore + 5) - - // 3. A capture in that state discloses the skip rather than presenting the - // unread elements as action-free — and spends no read budget doing it. - let leaf = RunnerAXSnapshotFrontier() - leaf.snapshot = FrontierSnapshotWithElementForTesting() - leaf.node = NSMutableDictionary(dictionary: ["label": "feedItem", "children": []]) - let coverage = RunnerAXSnapshotBridge.annotateCustomActions( - onMergedLeaves: [leaf], axClient: hung, limit: 12, rootFrame: .zero, deadline: nil) - XCTAssertEqual(coverage[RunnerAXSnapshotCustomActionsBlockedKey] as? Bool, true) - XCTAssertEqual(coverage[RunnerAXSnapshotCustomActionsReadKey] as? Int, 0) - XCTAssertEqual(coverage[RunnerAXSnapshotCustomActionsCandidatesKey] as? Int, 1) - XCTAssertEqual( - RunnerAXSnapshotBridge.customActionReadDispatchCount(), dispatchesBefore + 1) - XCTAssertEqual( - Self.privateAXCustomActionCoverage(coverage), - SnapshotCustomActionCoverage(read: 0, candidates: 1, truncated: 0, blocked: true)) - - // 4. Recovery: once the wedged call returns, reads resume by themselves. - hung.release() - let recovered = expectation(description: "in-flight drains") - DispatchQueue.global().async { - while RunnerAXSnapshotBridge.customActionReadsInFlight() > 0 { - usleep(20_000) - } - recovered.fulfill() - } - wait(for: [recovered], timeout: 5) - - completed = ObjCBool(false) - XCTAssertNil( - RunnerAXSnapshotBridge.customActionNames( - forElement: element, axClient: hung, completed: &completed)) - // Completed (the fake answers nil actions), which is the point: the pass is - // live again rather than latched off. - XCTAssertTrue(completed.boolValue) - XCTAssertEqual( - RunnerAXSnapshotBridge.customActionReadDispatchCount(), dispatchesBefore + 2) - } - - /// The element budget bounds how many elements we read; these caps bound what - /// any ONE element can put in the response. Clipping must be reported, since - /// a clipped list looks exactly like a complete one. - func testActionNamesAreCappedPerElementAndReported() { - var truncated = ObjCBool(true) - - // Under both caps: untouched, nothing to report. - let small = ["Reply", "Repost"] - XCTAssertEqual( - RunnerAXSnapshotBridge.cappedActionNames(small, truncated: &truncated), small) - XCTAssertFalse(truncated.boolValue) - - // More actions than the per-element cap: clipped to the first 8, reported. - let many = (1...20).map { "Action \($0)" } - let cappedMany = RunnerAXSnapshotBridge.cappedActionNames(many, truncated: &truncated) - XCTAssertEqual(cappedMany.count, 8) - XCTAssertEqual(cappedMany.first, "Action 1") - XCTAssertTrue(truncated.boolValue) - - // A single very long name is shortened, reported, and stays one string. - let long = String(repeating: "a", count: 500) - let cappedLong = RunnerAXSnapshotBridge.cappedActionNames([long], truncated: &truncated) - XCTAssertEqual(cappedLong.count, 1) - XCTAssertTrue(truncated.boolValue) - XCTAssertLessThan(cappedLong[0].count, long.count) - XCTAssertTrue(cappedLong[0].hasSuffix("…")) - - // Empty input is not "truncated". - XCTAssertEqual(RunnerAXSnapshotBridge.cappedActionNames([], truncated: &truncated), []) - XCTAssertFalse(truncated.boolValue) - } - - /// Action names annotated by the bridge must survive into the emitted node — - /// the whole point of the capture is that the merged card names its hidden - /// affordances. - func testPrivateAXNodesCarryAnnotatedCustomActions() { - let tree: [String: Any] = [ - "type": Int(XCUIElement.ElementType.application.rawValue), - "label": "Blue Sky", - "frame": ["x": 0, "y": 0, "width": 390, "height": 844], - "children": [ - [ - "type": Int(XCUIElement.ElementType.link.rawValue), - "label": "feedItem-by-whiskers.test", - "frame": ["x": 0, "y": 100, "width": 390, "height": 200], - "actions": ["Reply", "Repost", "Open post options menu"], - "children": [], - ], - [ - "type": Int(XCUIElement.ElementType.button.rawValue), - "label": "Compose", - "frame": ["x": 300, "y": 700, "width": 60, "height": 60], - "children": [], - ], - ], - ] - let nodes = privateAXAcquisition( - rawRoot: tree, - hint: CaptureHint( - projection: .regular, depth: nil, regularPresentedDepth: nil, - interactiveOnly: false, customActions: false) - ) - - let card = nodes.first { $0.label == "feedItem-by-whiskers.test" } - XCTAssertEqual(card?.actions, ["Reply", "Repost", "Open post options menu"]) - // A node the bridge did not annotate stays absent, not empty. - XCTAssertNil(nodes.first { $0.label == "Compose" }?.actions) - } - - func testPrivateAXAcquisitionDoesNotInterpretScope() { - let tree: [String: Any] = [ - "type": 1, "label": "App", - "children": [ - [ - "type": 9, "identifier": "homeScreen", - "children": [ - ["type": 48, "label": "Post body without the scope text", "children": []] - ], - ], - ["type": 9, "label": "unrelated sibling", "children": []], - ], - ] - // Scope never reaches acquisition: the hint derived for a scoped request carries no scope, - // and the backend has no way to interpret one. - let nodes = privateAXAcquisition( - rawRoot: tree, - hint: SnapshotPresentation.captureHint( - for: PresentationOptions( - interactiveOnly: false, - depth: nil, - scope: "homeScreen", - raw: false - ) - ) - ) - - let labels = nodes.compactMap { $0.label ?? $0.identifier } - XCTAssertTrue(labels.contains("homeScreen")) - // Descendants of the matched scope are included even when they do not contain the text. - XCTAssertTrue(labels.contains("Post body without the scope text")) - XCTAssertTrue(labels.contains("unrelated sibling")) - } - - func testPrivateAXInteractiveFiltersLoginLikeHiddenDrawer() throws { - let tree: [String: Any] = [ - "type": Int(XCUIElement.ElementType.application.rawValue), - "label": "Blue Sky", - "frame": ["x": 0, "y": 0, "width": 390, "height": 844], - "children": [ - [ - "type": Int(XCUIElement.ElementType.scrollView.rawValue), - "frame": ["x": 0, "y": 0, "width": 390, "height": 844], - "children": [ - [ - "type": Int(XCUIElement.ElementType.image.rawValue), - "label": "Callstack", - "frame": ["x": 145, "y": 104, "width": 100, "height": 100], - "children": [], - ], - [ - "type": Int(XCUIElement.ElementType.staticText.rawValue), - "label": "Welcome back", - "frame": ["x": 32, "y": 260, "width": 326, "height": 32], - "children": [], - ], - [ - "type": Int(XCUIElement.ElementType.textField.rawValue), - "label": "Email", - "identifier": "login.email", - "frame": ["x": 32, "y": 348, "width": 326, "height": 48], - "children": [], - ], - [ - "type": Int(XCUIElement.ElementType.secureTextField.rawValue), - "label": "Password", - "identifier": "login.password", - "frame": ["x": 32, "y": 412, "width": 326, "height": 48], - "children": [], - ], - [ - "type": Int(XCUIElement.ElementType.button.rawValue), - "label": "Sign in", - "identifier": "login.submit", - "frame": ["x": 32, "y": 492, "width": 326, "height": 52], - "children": [], - ], - [ - "type": Int(XCUIElement.ElementType.link.rawValue), - "label": "Forgot password?", - "frame": ["x": 128, "y": 568, "width": 134, "height": 32], - "children": [], - ], - [ - "type": Int(XCUIElement.ElementType.button.rawValue), - "label": "Admin settings", - "frame": ["x": -260, "y": 184, "width": 220, "height": 44], - "children": [], - ], - [ - "type": Int(XCUIElement.ElementType.other.rawValue), - "frame": ["x": 16, "y": 184, "width": 220, "height": 44], - "children": [], - ], - ], - ] - ], - ] - let viewport = CGRect(x: 0, y: 0, width: 390, height: 844) - let hint = CaptureHint( - projection: .regular, depth: nil, regularPresentedDepth: nil, - interactiveOnly: true, customActions: false) - let acquired = SnapshotGeometrySpace.normalized( - nodes: privateAXAcquisition(rawRoot: tree, hint: hint), - viewport: viewport, - interfaceOrientation: RunnerInterfaceOrientation.portrait - ) - // Acquisition serializes the drawer too; the shared fold is what hides it (#1797). - XCTAssertTrue(acquired.compactMap(\.label).contains("Admin settings")) - - let capture = try SnapshotPresentation.presentRegular( - SnapshotAcquisition( - hint: hint, nodes: acquired, truncated: false, effectiveDepth: nil, viewport: viewport), - options: PresentationOptions(interactiveOnly: true, depth: nil, scope: nil, raw: false), - policy: .cursorProjected - ) - let labels = capture.nodes.compactMap { $0.label } - XCTAssertEqual( - labels, - ["Blue Sky", "Callstack", "Welcome back", "Email", "Password", "Sign in", "Forgot password?"] - ) - XCTAssertFalse(labels.contains("Admin settings")) - } -} - -/// Stands in for an AX client whose `attributesForElement:` never returns — -/// the wedged-server case the containment exists for. `release()` lets the -/// hung call finish so recovery is observable. -private final class HungAXClientForTesting: NSObject { - private let gate = DispatchSemaphore(value: 0) - private let releasedOnce = NSLock() - private var released = false - - @objc(attributesForElement:attributes:error:) - func attributes(forElement element: Any, attributes: Any, error: NSErrorPointer) -> Any? { - releasedOnce.lock() - let alreadyReleased = released - releasedOnce.unlock() - // Once the wedge clears, the server answers normally again — that is what - // makes the recovery leg a recovery rather than a second hang. - if alreadyReleased { - return nil - } - gate.wait() - return nil - } - - func release() { - releasedOnce.lock() - defer { releasedOnce.unlock() } - guard !released else { return } - released = true - gate.signal() - } -} - -/// Minimal snapshot stand-in whose accessibilityElement resolves (so the -/// extension proceeds to the request) while the paired fake client cannot -/// serve it — the failed-re-root miss path. -private final class FrontierSnapshotWithElementForTesting: NSObject { - @objc let accessibilityElement = NSObject() -} - -/// The vanished-element case: KVC resolves the property and gets nil. -private final class FrontierSnapshotWithoutElementForTesting: NSObject { - @objc let accessibilityElement: NSObject? = nil -} -#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift index 483862a3af..60df7eea72 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift @@ -188,7 +188,7 @@ extension RunnerTests { return buttons.first(where: { isDismissButton($0.label) }) ?? buttons.last } - private func isAcceptButton(_ label: String) -> Bool { + func isAcceptButton(_ label: String) -> Bool { let normalized = label.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() return [ "ok", @@ -201,12 +201,6 @@ extension RunnerTests { ].contains(normalized) || normalized.hasPrefix("confirm") } -#if AGENT_DEVICE_RUNNER_UNIT_TESTS - func testAlertAcceptTreatsOpenAsAffirmative() { - XCTAssertTrue(isAcceptButton("Open")) - } -#endif - private func isDismissButton(_ label: String) -> Bool { [ "cancel", diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+BlockingSystemModalResolution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+BlockingSystemModalResolution.swift index 2b92c09bdc..687d36d8b8 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+BlockingSystemModalResolution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+BlockingSystemModalResolution.swift @@ -79,39 +79,3 @@ enum RemoteHostedSystemModalPolicy { state == .runningForeground } } - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS -extension RunnerTests { - func testRemoteHostProbeRunsOnlyWhenSpringboardModalHasNoActions() { - XCTAssertTrue(RemoteHostedSystemModalPolicy.shouldProbeRemoteHost(springboardActionCount: 0)) - XCTAssertFalse(RemoteHostedSystemModalPolicy.shouldProbeRemoteHost(springboardActionCount: 1)) - XCTAssertFalse(RemoteHostedSystemModalPolicy.shouldProbeRemoteHost(springboardActionCount: 3)) - } - - func testRemoteHostStateGateFailsClosedToForeground() { - XCTAssertTrue(RemoteHostedSystemModalPolicy.isEligibleHostState(.runningForeground)) - XCTAssertFalse(RemoteHostedSystemModalPolicy.isEligibleHostState(.runningBackground)) - XCTAssertFalse(RemoteHostedSystemModalPolicy.isEligibleHostState(.notRunning)) - XCTAssertFalse(RemoteHostedSystemModalPolicy.isEligibleHostState(.unknown)) - } - - // No SpringBoard host (`hasSpringBoardSystemModalHost`) means modal resolution must return - // `.absent` without probing com.apple.springboard (#1351). Written for tvOS, where no lane - // ever executed it; `resolveBlockingSystemModal` takes that decision at RUNTIME off the same - // flag on macOS, so the host lane runs the real branch on every PR. - // - // Its former sibling `testBlockingSystemAlertSnapshotIsNilOnTvOS` is deleted rather than - // widened: `blockingSystemAlertSnapshot` is `#if os(macOS) return nil`, so on the only lane - // that could run it the assertion would pin a compile-time literal — a green that no change - // to the runner could turn red. The runtime gate it meant to cover is this test's subject, - // and the nil it returns on macOS is the compiler's business, not a test's. - #if os(tvOS) || os(macOS) - func testResolveBlockingSystemModalIsAbsentWithoutSpringBoardHost() { - guard case .absent = resolveBlockingSystemModal(deadline: .distantFuture) else { - XCTFail("blocking system-modal resolution must be .absent without a SpringBoard host") - return - } - } - #endif -} -#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index 2e87e03529..c1aa92ca27 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -1,32 +1,6 @@ import XCTest import AgentDeviceSnapshotPresentation -#if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS) -import ObjectiveC.runtime - -private final class RunnerSynthesizedSwipeFailureStub: NSObject { - @objc(synthesizeSwipeWithApplication:resolvedWindow:x:y:x2:y2:durationMs:) - class func synthesizeSwipe( - application: XCUIApplication, - resolvedWindow: Any?, - x: Double, - y: Double, - x2: Double, - y2: Double, - durationMs: Double - ) -> String? { - "forced private synthesis failure" - } -} - -private final class RunnerSynthesizedTapFailureStub: NSObject { - @objc(synthesizeTapWithApplication:resolvedWindow:x:y:) - class func synthesizeTap(application: XCUIApplication, resolvedWindow: Any?, x: Double, y: Double) -> String? { - "forced private synthesis failure" - } -} -#endif - extension RunnerTests { // MARK: - Main Thread Dispatch @@ -110,7 +84,7 @@ extension RunnerTests { /// Single factory for the success payload every gesture returns (message + gesture timing + /// an optional touch/drag visualization frame), so the field shape lives in one place. - private func gestureResponse( + func gestureResponse( message: String, timing: (gestureStartUptimeMs: Double, gestureEndUptimeMs: Double), frame: GestureFrame = .none, @@ -165,7 +139,7 @@ extension RunnerTests { /// Gesture plans already return canonical centroid endpoints from the portable runtime. /// Keep runner timing/fallback diagnostics, but do not leak the coordinate-drag adapter's /// visualization frame into only the fast-fling response shape. - private func canonicalPlannedGestureResponse(_ response: Response) -> Response { + func canonicalPlannedGestureResponse(_ response: Response) -> Response { guard response.ok, let data = response.data else { return response } return Response( ok: true, @@ -191,678 +165,6 @@ extension RunnerTests { return gestureResponse(message: plan.intent, timing: timing) } -#if AGENT_DEVICE_RUNNER_UNIT_TESTS - func testGestureResponseIncludesSynthesizedTapFallbackDiagnostics() { - let response = gestureResponse( - message: "tapped", - timing: (gestureStartUptimeMs: 1, gestureEndUptimeMs: 2), - fallback: GestureFallback( - strategy: "xctest-coordinate-tap", - message: "Runner synthesized coordinate tap is unavailable", - hint: "Using XCTest coordinate tap fallback." - ) - ) - - XCTAssertEqual(response.ok, true) - XCTAssertEqual(response.data?.gestureFallback, "xctest-coordinate-tap") - XCTAssertEqual( - response.data?.gestureFallbackMessage, - "Runner synthesized coordinate tap is unavailable" - ) - XCTAssertEqual(response.data?.gestureFallbackHint, "Using XCTest coordinate tap fallback.") - } - - func testGestureResponseIncludesMaestroNonHittableFallbackUsage() { - let response = gestureResponse( - message: "tapped via non-hittable coordinate fallback", - timing: (gestureStartUptimeMs: 1, gestureEndUptimeMs: 2), - frame: .touch(nil), - maestroNonHittableCoordinateFallbackUsed: true - ) - - XCTAssertEqual(response.data?.maestroNonHittableCoordinateFallbackUsed, true) - } - - func testCanonicalPlannedGestureResponseOmitsDragFrameAndPreservesDiagnostics() { - let response = gestureResponse( - message: "fling", - timing: (gestureStartUptimeMs: 1, gestureEndUptimeMs: 2), - frame: .drag( - DragVisualizationFrame( - x: 160, - y: 150, - x2: 40, - y2: 150, - referenceWidth: 200, - referenceHeight: 300 - ) - ), - fallback: GestureFallback( - strategy: "xctest-coordinate-drag", - message: "Private synthesis unavailable", - hint: "Using XCTest coordinate fallback." - ) - ) - - let canonical = canonicalPlannedGestureResponse(response) - - XCTAssertEqual(canonical.data?.gestureStartUptimeMs, 1) - XCTAssertEqual(canonical.data?.gestureEndUptimeMs, 2) - XCTAssertEqual(canonical.data?.gestureFallback, "xctest-coordinate-drag") - XCTAssertEqual(canonical.data?.gestureFallbackMessage, "Private synthesis unavailable") - XCTAssertEqual(canonical.data?.gestureFallbackHint, "Using XCTest coordinate fallback.") - XCTAssertNil(canonical.data?.x) - XCTAssertNil(canonical.data?.y) - XCTAssertNil(canonical.data?.x2) - XCTAssertNil(canonical.data?.y2) - XCTAssertNil(canonical.data?.referenceWidth) - XCTAssertNil(canonical.data?.referenceHeight) - } - -#if os(iOS) - func testSinglePointerFlingFallsBackToXCTestCoordinateDragWhenPrivateSynthesisFails() throws { - let selector = NSSelectorFromString( - "synthesizeSwipeWithApplication:resolvedWindow:x:y:x2:y2:durationMs:" - ) - guard - let synthesizedSwipeMethod = class_getClassMethod(RunnerSynthesizedGesture.self, selector), - let failureStubMethod = class_getClassMethod(RunnerSynthesizedSwipeFailureStub.self, selector) - else { - XCTFail("unable to install synthesized swipe failure stub") - return - } - let originalImplementation = method_getImplementation(synthesizedSwipeMethod) - method_setImplementation( - synthesizedSwipeMethod, - method_getImplementation(failureStubMethod) - ) - app.launch() - runnerAccessibilityHealth = .healthy - defer { - method_setImplementation(synthesizedSwipeMethod, originalImplementation) - invalidateCachedTarget(reason: "unit_test_cleanup") - app.terminate() - } - let command = try runnerCommandFixture( - """ - {"command":"gesture","commandId":"gesture-fling-fallback","gesturePlan":{"topology":"single","intent":"fling","executionProfile":"endpoint-hold","durationMs":100,"viewport":{"x":0,"y":0,"width":200,"height":300},"pointers":[{"pointerId":0,"samples":[{"offsetMs":0,"point":{"x":160,"y":150}},{"offsetMs":100,"point":{"x":40,"y":150}}]}]}} - """ - ) - - let response = try executeOnMainPrepared(command: command, activeApp: app) - - XCTAssertTrue(response.ok) - XCTAssertEqual(response.data?.message, "fling") - XCTAssertEqual(response.data?.gestureFallback, "xctest-coordinate-drag") - XCTAssertEqual(response.data?.gestureFallbackMessage, "forced private synthesis failure") - XCTAssertEqual( - response.data?.gestureFallbackHint, - "Private XCTest event synthesis is required for AX-free coordinate drag on iOS; update Xcode if this persists." - ) - XCTAssertNil(response.data?.x) - XCTAssertNil(response.data?.y) - XCTAssertNil(response.data?.x2) - XCTAssertNil(response.data?.y2) - } - - func testSelectorTapFallsBackToXCTestCoordinateWhenPrivateSynthesisFails() throws { - let selector = NSSelectorFromString("synthesizeTapWithApplication:resolvedWindow:x:y:") - guard - let synthesizedTapMethod = class_getClassMethod(RunnerSynthesizedGesture.self, selector), - let failureStubMethod = class_getClassMethod(RunnerSynthesizedTapFailureStub.self, selector) - else { - XCTFail("unable to install synthesized tap failure stub") - return - } - let originalImplementation = method_getImplementation(synthesizedTapMethod) - method_setImplementation( - synthesizedTapMethod, - method_getImplementation(failureStubMethod) - ) - app.launch() - currentApp = app - runnerAccessibilityHealth = .healthy - defer { - method_setImplementation(synthesizedTapMethod, originalImplementation) - invalidateCachedTarget(reason: "unit_test_cleanup") - app.terminate() - } - let command = try runnerCommandFixture( - #"{"command":"tap","commandId":"selector-tap-fallback","selectorKey":"label","selectorValue":"Agent Device Runner","synthesized":true}"# - ) - - let response = try executeOnMainPrepared(command: command, activeApp: app) - - XCTAssertTrue(response.ok) - XCTAssertEqual(response.data?.message, "tapped") - XCTAssertEqual(response.data?.gestureFallback, "xctest-coordinate-tap") - XCTAssertEqual(response.data?.gestureFallbackMessage, "forced private synthesis failure") - XCTAssertEqual( - response.data?.gestureFallbackHint, - "Falling back to XCTest coordinate tap may be slower and can still need a healthy accessibility tree." - ) - } -#endif - -#if os(iOS) - // `waitForTextEntryReadiness`'s hardware-keyboard fallback returns early only on confirmed - // focus (#1874), and `keyboardFocusConfirmed` reads that from the app-wide focus predicate this - // bundle otherwise refuses to trust. Two XCTest facts it rests on, neither a repository - // invariant: the predicate reports a responder that shows NO software keyboard at all, and it - // names the element well enough to tell the tapped field from another one. The fixture field is - // the exact shape the fallback exists for — a real responder with an empty `inputView` — so this - // is where both are observable. If either regressed, readiness would silently stop taking the - // fallback and spend the full readinessTimeout on every hardware-keyboard field, which no other - // assertion would notice. - func testHardwareKeyboardResponderConfirmsItsOwnKeyboardFocus() throws { - app.launchArguments = ["--agent-device-text-entry-regression"] - app.launch() - defer { - invalidateCachedTarget(reason: "unit_test_cleanup") - app.terminate() - } - XCTAssertTrue(app.waitForExistence(timeout: appExistenceTimeout)) - - let textField = app.textFields["agent-device-hardware-keyboard-input"] - XCTAssertTrue(textField.waitForExistence(timeout: appExistenceTimeout)) - let otherElement = app.staticTexts["Agent Device Runner"] - XCTAssertTrue(otherElement.waitForExistence(timeout: appExistenceTimeout)) - XCTAssertFalse( - keyboardFocusConfirmed(app: app, element: textField), - "an untapped field must not confirm focus, or the fallback would fire immediately" - ) - - let tapCommand = try runnerCommandFixture( - #"{"command":"tap","commandId":"tap-focus-confirmation","selectorKey":"id","selectorValue":"agent-device-hardware-keyboard-input"}"# - ) - let tapResponse = try executeOnMainPrepared(command: tapCommand, activeApp: app) - XCTAssertTrue(tapResponse.ok, String(describing: tapResponse.error)) - try XCTSkipIf( - isKeyboardVisible(app: app), - "software keyboard is up: this simulator cannot exercise the hidden-keyboard responder path" - ) - - let deadline = Date().addingTimeInterval(TextEntryTiming.readinessTimeout) - var confirmed = keyboardFocusConfirmed(app: app, element: textField) - while !confirmed && Date() < deadline { - sleepFor(TextEntryTiming.pollInterval) - confirmed = keyboardFocusConfirmed(app: app, element: textField) - } - XCTAssertTrue(confirmed, "a tapped responder must confirm its own keyboard focus") - XCTAssertFalse( - keyboardFocusConfirmed(app: app, element: otherElement), - "focus held by another element must read as a refusal, never as this element's focus" - ) - } -#endif - - func testInjectedTapRecordedFailureGateIsTapOnlyAndCountGated() { - // The seam's recording side cannot run in-bundle (a real XCTIssue would - // fail this very test run — same constraint the record(_:) suppression - // tests document); the live daemon proof covers it. This pins the gate. - XCTAssertFalse(RunnerTests.shouldInjectTapRecordedFailure(command: .tap, remaining: 0)) - XCTAssertTrue(RunnerTests.shouldInjectTapRecordedFailure(command: .tap, remaining: 1)) - XCTAssertFalse(RunnerTests.shouldInjectTapRecordedFailure(command: .type, remaining: 1)) - XCTAssertFalse(RunnerTests.shouldInjectTapRecordedFailure(command: .snapshot, remaining: 1)) - } - - func testXCTestRecordedFailureResponseFailsMutatingSuccesses() throws { - let command = try runnerCommandFixture(#"{"command":"tap","commandId":"tap-1"}"#) - let response = Response(ok: true, data: DataPayload(message: "tapped")) - - let failureResponse = xctestRecordedFailureResponse(command: command, response: response) - - XCTAssertEqual(failureResponse?.ok, false) - XCTAssertEqual(failureResponse?.error?.code, "XCTEST_RECORDED_FAILURE") - XCTAssertEqual( - failureResponse?.error?.message, - "XCTest recorded a failure while executing tap; the action may not have been performed." - ) - } - - func testXCTestRecordedFailureResponseFailsActionButtonSuccess() throws { - // The Action Button press carries no settle and no post-action observation, so this conversion is - // the only evidence the press landed. That is why the press is not classified runner-lifecycle: - // `isLifecycle` would silence the conversion here (#2699, #2702 review). - let command = try runnerCommandFixture(#"{"command":"actionButton","commandId":"action-button-1"}"#) - let response = Response(ok: true, data: DataPayload(message: "actionButton")) - - let failureResponse = xctestRecordedFailureResponse(command: command, response: response) - - XCTAssertEqual(failureResponse?.ok, false) - XCTAssertEqual(failureResponse?.error?.code, "XCTEST_RECORDED_FAILURE") - XCTAssertEqual( - failureResponse?.error?.message, - "XCTest recorded a failure while executing actionButton; the action may not have been performed." - ) - } - - func testXCTestRecordedFailureResponseDoesNotWrapReadOnlyOrRunnerFatalResponses() throws { - let snapshotCommand = try runnerCommandFixture(#"{"command":"snapshot","commandId":"snapshot-1"}"#) - let tapCommand = try runnerCommandFixture(#"{"command":"tap","commandId":"tap-1"}"#) - let runnerFatalResponse = Response( - ok: true, - data: DataPayload(runnerFatal: true, runnerFatalReason: "ax_snapshot_unavailable") - ) - - XCTAssertNil( - xctestRecordedFailureResponse( - command: snapshotCommand, - response: Response(ok: true, data: DataPayload(nodes: [], truncated: false)) - ) - ) - XCTAssertNil(xctestRecordedFailureResponse(command: tapCommand, response: runnerFatalResponse)) - } - - // Simulator-only from here to the matching #endif: these launch the host app, route through - // SpringBoard, or assert the iOS-only alert/system-modal branches. Tests outside the - // `os(iOS)` regions in this file are pure runner decisions and also run on the macOS host - // lane (ci.yml) — see the classification convention in RunnerTests.swift. -#if os(iOS) - func testMissingBundleCommandInvalidatesCompleteCachedTargetState() throws { - app.launch() - currentApp = app - currentBundleId = "com.example.stale-target" - currentAppProcessIdentifier = 42 - snapshotXCTestPenaltyWarmupExemptionPending = true - defer { - invalidateCachedTarget(reason: "unit_test_cleanup") - app.terminate() - } - let command = try runnerCommandFixture( - #"{"command":"snapshot","commandId":"snapshot-without-bundle"}"# - ) - - _ = prepareActiveCommandContext(command: command) - - XCTAssertNil(currentApp) - XCTAssertNil(currentBundleId) - XCTAssertNil(currentAppProcessIdentifier) - XCTAssertFalse(snapshotXCTestPenaltyWarmupExemptionPending) - } - - func testSkipAppActivationPreflightIncludesForegroundCachedCoordinateOnlyTaps() throws { - app.launch() - currentApp = app - currentBundleId = nil - defer { - currentApp = nil - currentBundleId = nil - app.terminate() - } - let tap = try runnerCommandFixture( - #"{"command":"tap","commandId":"tap-1","x":10,"y":20}"# - ) - - XCTAssertTrue(shouldSkipAppActivationPreflight(tap)) - } - - func testSkipAppActivationPreflightRejectsMissingChangedAndBackgroundTargets() throws { - let coordinateTap = try runnerCommandFixture( - #"{"command":"tap","commandId":"tap-1","x":10,"y":20}"# - ) - currentApp = nil - currentBundleId = nil - XCTAssertFalse(shouldSkipAppActivationPreflight(coordinateTap)) - - app.launch() - currentApp = app - currentBundleId = "com.example.current" - defer { - currentApp = nil - currentBundleId = nil - app.terminate() - } - let changedBundleTap = try runnerCommandFixture( - #"{"command":"tap","commandId":"tap-2","appBundleId":"com.example.other","x":10,"y":20}"# - ) - - XCTAssertFalse(shouldSkipAppActivationPreflight(changedBundleTap)) - - app.terminate() - currentApp = app - currentBundleId = nil - - XCTAssertFalse(shouldSkipAppActivationPreflight(coordinateTap)) - } - - func testActionButtonPressSkipsAppActivationPreflightWithoutBeingRunnerLifecycle() throws { - currentApp = nil - currentBundleId = nil - let press = try runnerCommandFixture(#"{"command":"actionButton","commandId":"action-button-1"}"#) - - // The skip is its own decision, reached without the lifecycle flag that would also drop the - // recorded-failure conversion; no cached target and no foreground app is required for it. - XCTAssertFalse(isRunnerLifecycleCommand(.actionButton)) - XCTAssertTrue(shouldSkipAppActivationPreflight(press)) - } - - func testPrepareActiveCommandContextRoutesBlockingSystemModalToSpringboard() throws { - blockingSystemModalPresenceOverrideForTesting = true - currentApp = nil - currentBundleId = nil - defer { - blockingSystemModalPresenceOverrideForTesting = nil - currentApp = nil - currentBundleId = nil - } - let tap = try runnerCommandFixture( - #"{"command":"tap","commandId":"tap-1","x":10,"y":20}"# - ) - - let preparation = prepareActiveCommandContext( - command: tap, - routeToSpringboard: shouldRouteToSpringboardBlockingSystemModal(tap) - ) - - guard case .context(let context) = preparation else { - XCTFail("expected command context") - return - } - XCTAssertTrue(context.app === springboard) - } - - func testExecuteDispatchedReturnsBusyBeforeBlockingSystemModalProbeDrains() throws { - app.launch() - currentApp = app - currentBundleId = nil - defer { - currentApp = nil - currentBundleId = nil - systemModalProbeOverrideForTesting = nil - clearSnapshotXCTestChannelPenalty(reason: "test-cleanup") - app.terminate() - } - - final class ResultBox { - var response: Response? - var error: Error? - var commandRecoveredBeforeRelease = false - var wasBusyBeforeRelease = false - var hadAbandonedProbeBeforeRelease = false - var drained = false - } - let box = ResultBox() - let probeStarted = expectation(description: "system-modal routing probe started") - let verificationFinished = expectation(description: "command recovery and modal probe drain verified") - let probeReleaseGate = DispatchSemaphore(value: 0) - let commandFinishedGate = DispatchSemaphore(value: 0) - systemModalProbeOverrideForTesting = { _ in - probeStarted.fulfill() - _ = probeReleaseGate.wait(timeout: .now() + 15) - return DataPayload(message: "late system modal") - } - - let command = try runnerCommandFixture( - #"{"command":"tap","commandId":"bounded-modal-routing","x":10,"y":20}"# - ) - DispatchQueue(label: "agent-device.runner.tests.modal-routing-probe").async { - do { - box.response = try self.executeDispatched(command: command) - } catch { - box.error = error - } - commandFinishedGate.signal() - } - DispatchQueue(label: "agent-device.runner.tests.modal-routing-probe-verifier").async { - let commandWait = commandFinishedGate.wait( - timeout: .now() + self.systemModalProbeBudget + 3 - ) - box.commandRecoveredBeforeRelease = commandWait == .success - && box.error == nil - && box.response?.error?.code == "RUNNER_BUSY" - if case .busy = self.currentMainThreadBusyState() { - box.wasBusyBeforeRelease = true - } - box.hadAbandonedProbeBeforeRelease = self.hasAbandonedMainThreadWork() - - // The XCTest main thread is blocked inside the injected probe, so this verifier owns the - // ordered release after recording the command result and abandoned-work state above. - probeReleaseGate.signal() - let deadline = Date().addingTimeInterval(5) - while self.hasAbandonedMainThreadWork(), Date() < deadline { - self.sleepFor(0.002) - } - box.drained = !self.hasAbandonedMainThreadWork() - verificationFinished.fulfill() - } - - wait(for: [probeStarted, verificationFinished], timeout: 15) - XCTAssertTrue( - box.commandRecoveredBeforeRelease, - "the public coordinate tap must return RUNNER_BUSY before the blocked modal probe drains" - ) - XCTAssertTrue(box.wasBusyBeforeRelease) - XCTAssertTrue(box.hadAbandonedProbeBeforeRelease) - XCTAssertTrue(box.drained) - guard case .idle = currentMainThreadBusyState() else { - return XCTFail("expected the runner to become idle after the routing probe drained") - } - XCTAssertFalse(hasAbandonedMainThreadWork()) - } - - func testSkipAppActivationPreflightRejectsSelectorAndMixedSequenceGestures() throws { - app.launch() - currentApp = app - currentBundleId = nil - defer { - currentApp = nil - currentBundleId = nil - app.terminate() - } - let selectorTap = try runnerCommandFixture( - #"{"command":"tap","commandId":"tap-1","selectorKey":"label","selectorValue":"Search","synthesized":true}"# - ) - let standardDrag = try runnerCommandFixture( - #"{"command":"drag","commandId":"drag-1","x":10,"y":20,"x2":30,"y2":40}"# - ) - let mixedSequence = try runnerCommandFixture( - """ - {"command":"sequence","commandId":"seq-1","steps":[ - {"kind":"tap","x":10,"y":20,"synthesized":true}, - {"kind":"doubleTap","x":30,"y":40} - ]} - """ - ) - - XCTAssertFalse(shouldSkipAppActivationPreflight(selectorTap)) - XCTAssertFalse(shouldSkipAppActivationPreflight(standardDrag)) - XCTAssertFalse(shouldSkipAppActivationPreflight(mixedSequence)) - } - - // Launches nothing, but still simulator-only: `shouldSkipAppActivationPreflight` is - // `#if os(iOS) …guards… #else return false #endif`, so on macOS this asserts a compile-time - // literal and no edit to the iOS body could make it red. Its five siblings above and below - // are gated for the same reason. - func testSkipAppActivationPreflightRequiresCachedForegroundTarget() throws { - currentApp = nil - currentBundleId = nil - let scroll = try runnerCommandFixture( - #"{"command":"scroll","commandId":"scroll-1","direction":"down","pixels":400}"# - ) - - XCTAssertFalse(shouldSkipAppActivationPreflight(scroll)) - } - - func testSkipAppActivationPreflightKeepsDragScrollAndSequenceOnForegroundGuard() throws { - app.launch() - currentApp = app - currentBundleId = nil - defer { - currentApp = nil - currentBundleId = nil - app.terminate() - } - let drag = try runnerCommandFixture( - #"{"command":"drag","commandId":"drag-1","x":10,"y":20,"x2":30,"y2":40}"# - ) - let scroll = try runnerCommandFixture( - #"{"command":"scroll","commandId":"scroll-1","direction":"down","pixels":400}"# - ) - let sequence = try runnerCommandFixture( - """ - {"command":"sequence","commandId":"seq-1","steps":[ - {"kind":"tap","x":10,"y":20,"synthesized":true}, - {"kind":"longPress","x":10,"y":200,"durationMs":300} - ]} - """ - ) - - XCTAssertFalse(shouldSkipAppActivationPreflight(drag)) - XCTAssertFalse(shouldSkipAppActivationPreflight(scroll)) - XCTAssertFalse(shouldSkipAppActivationPreflight(sequence)) - } - - func testSkipAppActivationPreflightIncludesAlertCommands() throws { - let alert = try runnerCommandFixture( - #"{"command":"alert","commandId":"alert-1","action":"get"}"# - ) - - XCTAssertTrue(shouldSkipAppActivationPreflight(alert)) - } -#endif - - func testDispatchReturnsBusyBeforeQueueingMainThreadWork() throws { - let command = try runnerCommandFixture(#"{"command":"snapshot","commandId":"snapshot-busy"}"#) - abandonedMainThreadWorkCount = 1 - abandonedMainThreadWorkSince = Date(timeIntervalSinceNow: -2) - defer { - abandonedMainThreadWorkCount = 0 - abandonedMainThreadWorkSince = nil - } - - let response = try execute(command: command) - - XCTAssertFalse(response.ok) - XCTAssertEqual(response.error?.code, "RUNNER_BUSY") - XCTAssertTrue(response.error?.message.contains("previous command") == true) - } - - func testDispatchReturnsWedgedBeforeQueueingMainThreadWork() throws { - let command = try runnerCommandFixture(#"{"command":"snapshot","commandId":"snapshot-wedged"}"#) - abandonedMainThreadWorkCount = 1 - abandonedMainThreadWorkSince = Date(timeIntervalSinceNow: -(mainThreadWedgeThreshold + 1)) - defer { - abandonedMainThreadWorkCount = 0 - abandonedMainThreadWorkSince = nil - } - - let response = try execute(command: command) - - XCTAssertFalse(response.ok) - XCTAssertEqual(response.error?.code, "RUNNER_WEDGED") - XCTAssertTrue(response.error?.hint?.contains("runner session will be restarted") == true) - } - - func testPostSnapshotDelayMarkDoesNotQueueBehindAbandonedMainThreadWork() { - abandonedMainThreadWorkCount = 1 - defer { - abandonedMainThreadWorkCount = 0 - needsPostSnapshotInteractionDelay = false - } - - let finished = expectation(description: "off-main caller finished") - DispatchQueue(label: "agent-device.runner.tests.post-snapshot-delay").async { - self.setNeedsPostSnapshotInteractionDelay() - finished.fulfill() - } - - wait(for: [finished], timeout: 1) - mainThreadWorkLock.lock() - let abandonedWorkCount = abandonedMainThreadWorkCount - mainThreadWorkLock.unlock() - XCTAssertEqual(abandonedWorkCount, 1, "the skipped mark must not add an abandoned unit") - XCTAssertFalse(needsPostSnapshotInteractionDelay) - } - - func testSnapshotFailureInvalidationQueuesBehindAbandonedMainThreadWorkWithoutWaiting() { - currentBundleId = "com.example.stale-target" - defer { currentBundleId = nil } - - final class ResultBox { - var elapsed: TimeInterval? - var bundleStillCachedWhileBlocked: Bool? - var abandonedWhileBlocked: Int? - } - let box = ResultBox() - let mainBlocked = DispatchSemaphore(value: 0) - let releaseMain = DispatchSemaphore(value: 0) - let finished = expectation(description: "invalidation returned while main was blocked") - - DispatchQueue(label: "agent-device.runner.tests.snapshot-invalidation").async { - _ = try? self.runMainThreadWork( - "command_execution", - timeout: 0, - timeoutError: self.mainThreadExecutionTimeoutError - ) { - mainBlocked.signal() - _ = releaseMain.wait(timeout: .now() + 5) - return true - } - _ = mainBlocked.wait(timeout: .now() + 2) - let startedAt = Date() - self.invalidateCachedTargetAfterSnapshotFailure() - box.elapsed = Date().timeIntervalSince(startedAt) - box.bundleStillCachedWhileBlocked = self.currentBundleId != nil - self.mainThreadWorkLock.lock() - box.abandonedWhileBlocked = self.abandonedMainThreadWorkCount - self.mainThreadWorkLock.unlock() - releaseMain.signal() - finished.fulfill() - } - - wait(for: [finished], timeout: 8) - let drainDeadline = Date().addingTimeInterval(2) - while hasAbandonedMainThreadWork() || currentBundleId != nil, Date() < drainDeadline { - sleepFor(0.005) - } - - XCTAssertLessThan( - box.elapsed ?? .infinity, - 0.5, - "the failed capture must not wait behind abandoned main-thread work" - ) - XCTAssertEqual( - box.bundleStillCachedWhileBlocked, - true, - "the drop must queue behind the blocked main thread, not run early" - ) - XCTAssertEqual(box.abandonedWhileBlocked, 1, "the deferred drop must not add an abandoned unit") - XCTAssertFalse(hasAbandonedMainThreadWork()) - XCTAssertNil(currentBundleId, "the drop must run once the main thread frees") - } -#endif - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS - /// Routes `command` through the transport's inline and queued paths. The calling test's main - /// thread serves the command's main-thread work while it waits. - func execute(command: Command) throws -> Response { - dispatchPrecondition(condition: .onQueue(.main)) - if let response = inlineResponse(for: command) { - return response - } - final class ResultBox { - var result: Result? - } - let box = ResultBox() - let executed = XCTestExpectation(description: "\(command.command.rawValue) executed off main") - enqueueAccepted(command: command) { result in - box.result = result - executed.fulfill() - } - guard XCTWaiter.wait(for: [executed], timeout: mainThreadExecutionTimeout + 5) == .completed, - let result = box.result - else { - throw NSError( - domain: RunnerErrorDomain.general, - code: RunnerErrorCode.commandReturnedNoResponse, - userInfo: [NSLocalizedDescriptionKey: "command did not finish on the command queue"] - ) - } - return try result.get() - } -#endif - func executeAccepted(command: Command) throws -> Response { commandJournal.start(command: command) pendingTargetActivation = nil @@ -967,7 +269,7 @@ extension RunnerTests { } } - private func executeDispatched(command: Command) throws -> Response { + func executeDispatched(command: Command) throws -> Response { // XCTest work cannot be cancelled mid-flight: once the watchdog abandons a main-queue // block, queueing more main-thread commands behind it only buries the runner deeper. // Refuse fast instead so the daemon backs off while the abandoned work drains; past the @@ -1238,7 +540,7 @@ extension RunnerTests { } } - private func setNeedsPostSnapshotInteractionDelay() { + func setNeedsPostSnapshotInteractionDelay() { guard !hasAbandonedMainThreadWork() else { NSLog("AGENT_DEVICE_RUNNER_POST_SNAPSHOT_DELAY_MARK_SKIPPED_XCTEST_OCCUPIED") return @@ -1256,7 +558,7 @@ extension RunnerTests { } } - private func invalidateCachedTargetAfterSnapshotFailure() { + func invalidateCachedTargetAfterSnapshotFailure() { // Abandoned work ahead of this hop cannot be cancelled: queue the drop behind it without // waiting, so the failed capture answers now and the next command still finds the target gone. guard !hasAbandonedMainThreadWork() else { @@ -1424,7 +726,7 @@ extension RunnerTests { ) } - private func prepareActiveCommandContext( + func prepareActiveCommandContext( command: Command, routeToSpringboard: Bool = false ) -> ActiveCommandPreparation { @@ -2446,7 +1748,7 @@ extension RunnerTests { return currentXCTestFailureCount() > failureCountBefore } - private func xctestRecordedFailureResponse(command: Command, response: Response) -> Response? { + func xctestRecordedFailureResponse(command: Command, response: Response) -> Response? { guard response.ok else { return nil } if response.data?.runnerFatal == true { return nil @@ -2464,13 +1766,7 @@ extension RunnerTests { ) } -#if AGENT_DEVICE_RUNNER_UNIT_TESTS - func runnerCommandFixture(_ json: String) throws -> Command { - try JSONDecoder().decode(Command.self, from: Data(json.utf8)) - } -#endif - - private func shouldSkipAppActivationPreflight(_ command: Command) -> Bool { + func shouldSkipAppActivationPreflight(_ command: Command) -> Bool { #if os(iOS) if command.command == .alert { return true @@ -2495,7 +1791,7 @@ extension RunnerTests { #endif } - private func shouldRouteToSpringboardBlockingSystemModal( + func shouldRouteToSpringboardBlockingSystemModal( _ command: Command ) -> Bool { #if os(iOS) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandJournal.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandJournal.swift index 9ed3b9f51f..c91955116d 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandJournal.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandJournal.swift @@ -144,340 +144,3 @@ final class RunnerCommandJournal { } } } - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS -extension RunnerTests { - func testUptimeBypassesCommandJournal() throws { - let command = runnerJournalCommand("uptime", id: "uptime-probe") - - let response = try execute(command: command) - let status = commandJournal.status(normalizedCommandId: "uptime-probe") - - XCTAssertEqual(response.ok, true) - XCTAssertNotNil(response.data?.currentUptimeMs) - XCTAssertEqual(status.lifecycleState, RunnerCommandLifecycleState.notAccepted.rawValue) - } - - func testStampingCurrentUptimePreservesPayload() { - let stamped = Response(ok: true, data: DataPayload(message: "recording started")) - .stampingCurrentUptimeMs(123.5) - - XCTAssertEqual(stamped.ok, true) - XCTAssertEqual(stamped.data?.message, "recording started") - XCTAssertEqual(stamped.data?.currentUptimeMs, 123.5) - } - - func testStampingCurrentUptimeCreatesPayloadWhenNil() { - let stamped = Response(ok: true).stampingCurrentUptimeMs(456.0) - - XCTAssertEqual(stamped.ok, true) - XCTAssertEqual(stamped.data?.currentUptimeMs, 456.0) - } - - func testStampingCurrentUptimeSkipsErrorResponses() { - let response = Response(ok: false, error: ErrorPayload(message: "boom")) - let stamped = response.stampingCurrentUptimeMs(789.0) - - XCTAssertEqual(stamped.ok, false) - XCTAssertNil(stamped.data) - XCTAssertEqual(stamped.error?.message, "boom") - } - - func testStampingCurrentMainThreadBusyPreservesPayload() { - let stamped = Response(ok: true, data: DataPayload(nodes: [], truncated: false)) - .stampingCurrentMainThreadBusy(true) - - XCTAssertEqual(stamped.ok, true) - XCTAssertEqual(stamped.data?.runnerMainThreadBusy, true) - } - - func testStampingCurrentMainThreadBusySkipsErrorResponses() { - let response = Response(ok: false, error: ErrorPayload(code: "RUNNER_BUSY", message: "busy")) - let stamped = response.stampingCurrentMainThreadBusy(true) - - XCTAssertEqual(stamped.ok, false) - XCTAssertNil(stamped.data) - XCTAssertEqual(stamped.error?.code, "RUNNER_BUSY") - } - - func testMainThreadBusyStateReportsOccupancy() { - XCTAssertFalse(MainThreadBusyState.idle.reportsMainThreadBusy) - XCTAssertTrue(MainThreadBusyState.busy(abandonedForSeconds: 5).reportsMainThreadBusy) - XCTAssertTrue(MainThreadBusyState.wedged(abandonedForSeconds: 200).reportsMainThreadBusy) - XCTAssertEqual( - Response(ok: true).stampingCurrentMainThreadBusy(false).data?.runnerMainThreadBusy, false) - } - - func testCommandFailedResponseTagsMainThreadTimeoutWithTypedCode() { - let timeout = NSError( - domain: RunnerErrorDomain.general, - code: RunnerErrorCode.mainThreadExecutionTimedOut, - userInfo: [NSLocalizedDescriptionKey: "main thread execution timed out"] - ) - - let response = commandFailedResponse(from: timeout) - - XCTAssertEqual(response.ok, false) - XCTAssertEqual(response.error?.code, RunnerWireErrorCode.mainThreadTimeout) - } - - func testCommandFailedResponseKeepsGenericCodeForOtherErrors() { - let other = NSError(domain: "SomeOtherDomain", code: 99, userInfo: nil) - - let response = commandFailedResponse(from: other) - - XCTAssertEqual(response.error?.code, "COMMAND_FAILED") - } - - func testJournalStoredResponseStaysUnstamped() throws { - let journal = RunnerCommandJournal() - let recordStart = runnerJournalCommand("recordStart", id: "record-start-anchor") - - journal.accept(command: recordStart) - journal.finish( - command: recordStart, - response: Response(ok: true, data: DataPayload(message: "recording started")) - ) - - let status = journal.status(normalizedCommandId: "record-start-anchor") - let responseJson = try XCTUnwrap(status.lifecycleResponseJson) - XCTAssertFalse(responseJson.contains("currentUptimeMs")) - } - - func testCommandJournalRetentionPolicy() throws { - let journal = RunnerCommandJournal() - - let uptime = runnerJournalCommand("uptime", id: "small-scalar") - journal.accept(command: uptime) - journal.finish( - command: uptime, - response: Response(ok: true, data: DataPayload(currentUptimeMs: 12.5)) - ) - - let scalarStatus = journal.status(normalizedCommandId: "small-scalar") - XCTAssertEqual(scalarStatus.lifecycleState, RunnerCommandLifecycleState.completed.rawValue) - XCTAssertEqual(scalarStatus.lifecycleResponseOk, true) - XCTAssertNotNil(scalarStatus.lifecycleResponseJson) - let scalarResponse = try decodeRunnerJournalResponse(scalarStatus.lifecycleResponseJson) - XCTAssertEqual(scalarResponse.data?.currentUptimeMs, 12.5) - - let querySelector = runnerJournalCommand("querySelector", id: "small-object") - journal.accept(command: querySelector) - journal.finish( - command: querySelector, - response: Response(ok: true, data: DataPayload(found: true, nodes: [runnerJournalNode()])) - ) - - let objectStatus = journal.status(normalizedCommandId: "small-object") - XCTAssertNotNil(objectStatus.lifecycleResponseJson) - let objectResponse = try decodeRunnerJournalResponse(objectStatus.lifecycleResponseJson) - XCTAssertEqual(objectResponse.data?.found, true) - XCTAssertEqual(objectResponse.data?.nodes?.count, 1) - - let snapshot = runnerJournalCommand("snapshot", id: "snapshot-tree") - journal.accept(command: snapshot) - journal.finish( - command: snapshot, - response: Response(ok: true, data: DataPayload(nodes: [runnerJournalNode()], truncated: false)) - ) - - let snapshotStatus = journal.status(normalizedCommandId: "snapshot-tree") - XCTAssertEqual(snapshotStatus.lifecycleState, RunnerCommandLifecycleState.completed.rawValue) - XCTAssertEqual(snapshotStatus.lifecycleResponseOk, true) - XCTAssertNil(snapshotStatus.lifecycleResponseJson) - - let screenshot = runnerJournalCommand("screenshot", id: "screenshot-artifact") - journal.accept(command: screenshot) - journal.finish( - command: screenshot, - response: Response(ok: true, data: DataPayload(message: "tmp/screenshot-1.png")) - ) - - let screenshotStatus = journal.status(normalizedCommandId: "screenshot-artifact") - XCTAssertEqual(screenshotStatus.lifecycleState, RunnerCommandLifecycleState.completed.rawValue) - XCTAssertEqual(screenshotStatus.lifecycleResponseOk, true) - XCTAssertNil(screenshotStatus.lifecycleResponseJson) - - let scroll = runnerJournalCommand("scroll", id: "scroll-drag") - journal.accept(command: scroll) - journal.finish( - command: scroll, - response: Response( - ok: true, - data: DataPayload( - message: "scrolled", - gestureStartUptimeMs: 1, - gestureEndUptimeMs: 2, - x: 155, - y: 420, - x2: 155, - y2: 301, - referenceWidth: 300, - referenceHeight: 600 - ) - ) - ) - - let scrollStatus = journal.status(normalizedCommandId: "scroll-drag") - XCTAssertEqual(scrollStatus.lifecycleState, RunnerCommandLifecycleState.completed.rawValue) - XCTAssertEqual(scrollStatus.lifecycleResponseOk, true) - XCTAssertNotNil(scrollStatus.lifecycleResponseJson) - let scrollResponse = try decodeRunnerJournalResponse(scrollStatus.lifecycleResponseJson) - XCTAssertEqual(scrollResponse.data?.x, 155) - XCTAssertEqual(scrollResponse.data?.y, 420) - XCTAssertEqual(scrollResponse.data?.x2, 155) - XCTAssertEqual(scrollResponse.data?.y2, 301) - XCTAssertEqual(scrollResponse.data?.referenceWidth, 300) - XCTAssertEqual(scrollResponse.data?.referenceHeight, 600) - - let largeRead = runnerJournalCommand("readText", id: "large-read") - journal.accept(command: largeRead) - journal.finish( - command: largeRead, - response: Response(ok: true, data: DataPayload(text: String(repeating: "x", count: 17 * 1024))) - ) - - let largeReadStatus = journal.status(normalizedCommandId: "large-read") - XCTAssertEqual(largeReadStatus.lifecycleState, RunnerCommandLifecycleState.completed.rawValue) - XCTAssertEqual(largeReadStatus.lifecycleResponseOk, true) - XCTAssertNil(largeReadStatus.lifecycleResponseJson) - } - - func testCommandJournalKeepsErrorMetadataWhenResponseJsonIsDropped() { - let journal = RunnerCommandJournal() - let snapshot = runnerJournalCommand("snapshot", id: "snapshot-error") - let hint = "Try a smaller read such as snapshot -s -d 8." - - journal.accept(command: snapshot) - journal.finish( - command: snapshot, - response: Response( - ok: false, - error: ErrorPayload( - code: "IOS_AX_SNAPSHOT_FAILED", - message: "iOS XCTest snapshot failed while serializing the accessibility tree.", - hint: hint - ) - ) - ) - - let status = journal.status(normalizedCommandId: "snapshot-error") - XCTAssertEqual(status.lifecycleState, RunnerCommandLifecycleState.failed.rawValue) - XCTAssertEqual(status.lifecycleResponseOk, false) - XCTAssertNil(status.lifecycleResponseJson) - XCTAssertEqual(status.lifecycleErrorCode, "IOS_AX_SNAPSHOT_FAILED") - XCTAssertEqual( - status.lifecycleErrorMessage, - "iOS XCTest snapshot failed while serializing the accessibility tree." - ) - XCTAssertEqual(status.lifecycleErrorHint, hint) - } - - func testCommandJournalRetainsCompletedSequenceResults() throws { - let journal = RunnerCommandJournal() - let sequence = runnerJournalCommand("sequence", id: "sequence-completed") - let results = (0..<20).map { _ in - SequenceStepResult( - ok: true, - kind: "tap", - errorCode: nil, - errorMessage: nil, - gestureStartUptimeMs: 100, - gestureEndUptimeMs: 120 - ) - } - - journal.accept(command: sequence) - journal.finish( - command: sequence, - response: Response( - ok: true, - data: DataPayload( - message: "sequence", - completedSteps: 20, - failedStepIndex: nil, - sequenceResults: results - ) - ) - ) - - let status = journal.status(normalizedCommandId: "sequence-completed") - XCTAssertEqual(status.lifecycleState, RunnerCommandLifecycleState.completed.rawValue) - XCTAssertEqual(status.lifecycleResponseOk, true) - let json = try XCTUnwrap(status.lifecycleResponseJson) - // Worst-case 20-step response must stay under the 16KB journal retention cap. - XCTAssertLessThan(json.utf8.count, 16 * 1024) - let decoded = try decodeRunnerJournalResponse(status.lifecycleResponseJson) - XCTAssertEqual(decoded.data?.completedSteps, 20) - XCTAssertEqual(decoded.data?.sequenceResults?.count, 20) - } - - func testCommandJournalRetainsFailedSequenceResults() throws { - let journal = RunnerCommandJournal() - let sequence = runnerJournalCommand("sequence", id: "sequence-failed") - let longError = String(repeating: "z", count: 200) - let results: [SequenceStepResult] = [ - SequenceStepResult(ok: true, kind: "tap", errorCode: nil, errorMessage: nil, - gestureStartUptimeMs: 100, gestureEndUptimeMs: 120), - SequenceStepResult(ok: true, kind: "tap", errorCode: nil, errorMessage: nil, - gestureStartUptimeMs: 130, gestureEndUptimeMs: 150), - SequenceStepResult(ok: false, kind: "longPress", errorCode: "UNSUPPORTED_OPERATION", - errorMessage: longError, gestureStartUptimeMs: 160, gestureEndUptimeMs: 180), - ] - - journal.accept(command: sequence) - journal.finish( - command: sequence, - response: Response( - ok: true, - data: DataPayload( - message: "sequence", - completedSteps: 2, - failedStepIndex: 2, - sequenceResults: results - ) - ) - ) - - let status = journal.status(normalizedCommandId: "sequence-failed") - XCTAssertEqual(status.lifecycleState, RunnerCommandLifecycleState.completed.rawValue) - let decoded = try decodeRunnerJournalResponse(status.lifecycleResponseJson) - XCTAssertEqual(decoded.data?.completedSteps, 2) - XCTAssertEqual(decoded.data?.failedStepIndex, 2) - XCTAssertEqual(decoded.data?.sequenceResults?.count, 3) - XCTAssertEqual(decoded.data?.sequenceResults?[2].ok, false) - XCTAssertEqual(decoded.data?.sequenceResults?[2].errorCode, "UNSUPPORTED_OPERATION") - } - - private func runnerJournalCommand(_ command: String, id: String) -> Command { - let json = #"{"command":"\#(command)","commandId":"\#(id)"}"# - return try! JSONDecoder().decode(Command.self, from: Data(json.utf8)) - } - - private func runnerJournalNode() -> PresentedNode { - SnapshotPresentation.singleElementRead( - RawAXNode( - index: 0, - type: "button", - label: "Continue", - identifier: "continue", - value: nil, - rect: SnapshotRect(x: 10, y: 20, width: 100, height: 44), - enabled: true, - focused: nil, - selected: nil, - hittable: true, - depth: 0, - parentIndex: nil, - hiddenContentAbove: nil, - hiddenContentBelow: nil - ) - ) - } - - private func decodeRunnerJournalResponse(_ responseJson: String?) throws -> Response { - let responseJson = try XCTUnwrap(responseJson) - return try JSONDecoder().decode(Response.self, from: Data(responseJson.utf8)) - } -} -#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index 8f0a101429..002459b19c 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -602,27 +602,4 @@ extension RunnerTests { #endif } #endif - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS - func testDesktopScrollWheelDeltasMapDirections() { - XCTAssertEqual(desktopScrollWheelDeltas(direction: .up, pixels: 120).vertical, 120) - XCTAssertEqual(desktopScrollWheelDeltas(direction: .down, pixels: 120).vertical, -120) - XCTAssertEqual(desktopScrollWheelDeltas(direction: .left, pixels: 120).horizontal, 120) - XCTAssertEqual(desktopScrollWheelDeltas(direction: .right, pixels: 120).horizontal, -120) - } - - func testDesktopScrollWheelDeltaEventsHonorDurationAndPreservePixels() { - let events = desktopScrollWheelDeltaEvents(direction: .down, pixels: 200, durationMs: 50) - XCTAssertEqual(events.count, 4) - XCTAssertEqual(events.map(\.vertical).reduce(0, +), -200) - XCTAssertEqual(events.map(\.horizontal).reduce(0, +), 0) - XCTAssertEqual(desktopScrollEventIntervalSeconds(durationMs: 50, eventCount: events.count), 0.05 / 3.0) - } - - func testDesktopScrollWheelDeltaEventsKeepInstantScrollSingleEvent() { - let events = desktopScrollWheelDeltaEvents(direction: .down, pixels: 200, durationMs: 0) - XCTAssertEqual(events.count, 1) - XCTAssertEqual(events.first?.vertical, -200) - } -#endif } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Keyboard.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Keyboard.swift index a3fab3fad7..d743ec2a7c 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Keyboard.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Keyboard.swift @@ -447,145 +447,3 @@ func runnerScreenshotStabilitySettled( guard let first = window.first, let firstData = first else { return false } return window.allSatisfy { $0 == firstData } } - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS -extension RunnerTests { - func testKeyboardBandFactReadFailureIsStatedAsUnmeasurableNotAbsence() { - // Absence would let a consumer claim the screen is clear of a keyboard on the strength of a read - // that never answered. - let fact = runnerKeyboardBandFact( - readSucceeded: false, - exists: false, - frame: CGRect(x: 0, y: 198, width: 402, height: 204) - ) - XCTAssertEqual(fact, .unmeasurable(RunnerKeyboardBandReason.queryFailed)) - XCTAssertEqual(fact.payload.kind, "unmeasurable") - XCTAssertEqual(fact.payload.reason, RunnerKeyboardBandReason.queryFailed) - XCTAssertNil(fact.payload.frame) - } - - func testKeyboardBandFactPublishesAbsenceWhenTheQueryFindsNoKeyboard() { - let fact = runnerKeyboardBandFact(readSucceeded: true, exists: false, frame: .zero) - XCTAssertEqual(fact, .absent) - XCTAssertEqual(fact.payload.kind, "absent") - XCTAssertNil(fact.payload.frame) - XCTAssertNil(fact.payload.reason) - } - - func testKeyboardBandFactPublishesTheMeasuredBandInAppOrientationSpace() { - // The landscape band measured on iPhone 17 Pro (iOS 26.2) after #2653: full width across the - // bottom of a 402 pt-tall app, which is what the tree reported as a strip down the left edge. - let frame = CGRect(x: 0, y: 198, width: 874, height: 204) - let fact = runnerKeyboardBandFact(readSucceeded: true, exists: true, frame: frame) - XCTAssertEqual(fact, .visible(frame)) - let payload = fact.payload - XCTAssertEqual(payload.kind, "visible") - XCTAssertEqual(payload.frame, SnapshotRect(x: 0, y: 198, width: 874, height: 204)) - XCTAssertNil(payload.reason) - } - - func testKeyboardBandFactRefusesUnusableGeometryInsteadOfClaimingAbsence() { - let nan = CGFloat(Double.nan) - let infinite = CGFloat.infinity - let unusable: [CGRect] = [ - .zero, - CGRect(x: 0, y: 198, width: 0, height: 204), - CGRect(x: 0, y: 198, width: 874, height: -1), - CGRect(x: 0, y: 198, width: -874, height: 204), - CGRect(x: 0, y: nan, width: 874, height: 204), - CGRect(x: 0, y: 198, width: nan, height: 204), - CGRect(x: infinite, y: 198, width: 874, height: 204) - ] - for frame in unusable { - let fact = runnerKeyboardBandFact(readSucceeded: true, exists: true, frame: frame) - XCTAssertEqual( - fact, - .unmeasurable(RunnerKeyboardBandReason.unusableFrame), - "expected \(frame) to be refused as a band" - ) - } - } - - func testKeyboardBandFactPayloadRoundTripsThroughTheWireShape() throws { - let cases: [RunnerKeyboardBandFact] = [ - .visible(CGRect(x: 0, y: 583, width: 402, height: 291)), - .absent, - .unmeasurable(RunnerKeyboardBandReason.queryTimeout) - ] - for fact in cases { - let data = try JSONEncoder().encode(fact.payload) - // `encodeIfPresent` for the two optional fields: a fact carries its own evidence and nothing - // else, so the daemon never has to distinguish a null from an absent key. - let object = try XCTUnwrap(JSONSerialization.jsonObject(with: data) as? [String: Any]) - switch fact { - case .visible: - XCTAssertNil(object["reason"]) - XCTAssertNotNil(object["frame"]) - case .absent: - XCTAssertNil(object["frame"]) - XCTAssertNil(object["reason"]) - case .unmeasurable: - XCTAssertNil(object["frame"]) - XCTAssertNotNil(object["reason"]) - } - XCTAssertEqual(try JSONDecoder().decode(KeyboardBandFactPayload.self, from: data), fact.payload) - } - } - - func testRunnerScreenshotStabilitySettledNeedsEnoughSamples() { - XCTAssertFalse(runnerScreenshotStabilitySettled([], requiredConsecutiveMatches: 3)) - XCTAssertFalse(runnerScreenshotStabilitySettled([Data([1])], requiredConsecutiveMatches: 3)) - let frame = Data([1, 2, 3]) - XCTAssertFalse( - runnerScreenshotStabilitySettled([frame, frame], requiredConsecutiveMatches: 3) - ) - } - - func testRunnerScreenshotStabilitySettledTrueWhenWindowMatches() { - let frame = Data([1, 2, 3]) - XCTAssertTrue( - runnerScreenshotStabilitySettled([Data([9]), frame, frame, frame], requiredConsecutiveMatches: 3) - ) - } - - func testRunnerScreenshotStabilitySettledFalseOnMidWindowMismatch() { - // A momentary pause (two matching samples) followed by resumed movement - // must not read as settled: the 3-sample window still spans the mismatch. - let frame = Data([1, 2, 3]) - let moved = Data([4, 5, 6]) - XCTAssertFalse( - runnerScreenshotStabilitySettled([frame, frame, moved], requiredConsecutiveMatches: 3) - ) - } - - func testRunnerScreenshotStabilitySettledFalseOnFailedCapture() { - // A nil sample (failed screenshot) never counts as a match, even against - // other nils — an unverifiable run must not look "stable". - XCTAssertFalse(runnerScreenshotStabilitySettled([nil, nil, nil], requiredConsecutiveMatches: 3)) - let frame = Data([1, 2, 3]) - XCTAssertFalse( - runnerScreenshotStabilitySettled([frame, frame, nil], requiredConsecutiveMatches: 3) - ) - } - - func testRunnerScreenshotStabilitySettledOnlyLooksAtTheTrailingWindow() { - // An older mismatch before the trailing window must not block settlement - // once the required run of most-recent samples agrees. - let frame = Data([9]) - XCTAssertTrue( - runnerScreenshotStabilitySettled( - [Data([1]), Data([2]), frame, frame, frame], - requiredConsecutiveMatches: 3 - ) - ) - } - - func testRunnerScreenshotStabilitySettledRejectsDegenerateRequirement() { - // Fewer than 2 required matches would make any single sample "settled" — - // guard against a misconfigured caller rather than silently no-op the wait. - XCTAssertFalse( - runnerScreenshotStabilitySettled([Data([1])], requiredConsecutiveMatches: 1) - ) - } -} -#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift index 57e4467cd3..0314cbab9b 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift @@ -505,33 +505,3 @@ extension RunnerTests { usleep(useconds_t(delay * 1_000_000)) } } - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS -extension RunnerTests { - func testResettingTargetBoundStateForgetsTheLastWrittenMarkers() { - defer { invalidateCachedTarget(reason: "unit_test_cleanup") } - lastLoggedFastAppGuardLine = "AGENT_DEVICE_RUNNER_FAST_APP_GUARD bundle=app state=4" - lastLoggedGesturePolicyLines[.scroll] = "AGENT_DEVICE_RUNNER_SYNTHESIZED_GESTURE_POLICY kind=scroll" - resetTargetBoundState() - XCTAssertNil(lastLoggedFastAppGuardLine, "a rebind must state the guard once more") - XCTAssertTrue(lastLoggedGesturePolicyLines.isEmpty, "a rebind must state the policy once more") - } - - func testFastAppGuardMarkerWritesOnceUntilTheFactChanges() { - var written: [String] = [] - runnerMarkerWriter = { written.append($0) } - defer { - runnerMarkerWriter = { NSLog("%@", $0) } - invalidateCachedTarget(reason: "unit_test_cleanup") - } - writeFastAppGuardMarker(bundleId: "com.example.app", state: .runningForeground) - writeFastAppGuardMarker(bundleId: "com.example.app", state: .runningForeground) - XCTAssertEqual(written.count, 1, "a repeated fact writes no second line") - writeFastAppGuardMarker(bundleId: "com.example.other", state: .runningForeground) - XCTAssertEqual(written.count, 2, "a changed fact writes a new line") - resetTargetBoundState() - writeFastAppGuardMarker(bundleId: "com.example.other", state: .runningForeground) - XCTAssertEqual(written.count, 3, "a rebind states the same fact once more") - } -} -#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+MainThreadWork.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+MainThreadWork.swift index c0dafe8f05..d624b10ba6 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+MainThreadWork.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+MainThreadWork.swift @@ -135,95 +135,3 @@ extension RunnerTests { ) } } - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS -extension RunnerTests { - func testRunMainThreadWorkExecutesOffMainCallerOnMainThread() { - final class ResultBox { - var observedMainThread: Bool? - var error: Error? - } - let box = ResultBox() - let finished = expectation(description: "off-main caller finished") - - DispatchQueue(label: "agent-device.runner.tests.off-main").async { - do { - box.observedMainThread = try self.runMainThreadWork( - "command_execution", - timeout: 1, - timeoutError: self.mainThreadExecutionTimeoutError - ) { - Thread.isMainThread - } - } catch { - box.error = error - } - finished.fulfill() - } - - wait(for: [finished], timeout: 2) - XCTAssertNil(box.error) - XCTAssertEqual(box.observedMainThread, true) - } - - func testRunMainThreadWorkTimeoutMarksAbandonedUntilDrained() { - final class ResultBox { - var error: Error? - var abandonedCount: Int? - var abandonedSinceSet: Bool? - var busyWhileAbandoned = false - } - let box = ResultBox() - let releaseWork = DispatchSemaphore(value: 0) - let observedAbandoned = DispatchSemaphore(value: 0) - let timedOut = expectation(description: "off-main caller timed out") - - DispatchQueue(label: "agent-device.runner.tests.timeout").async { - do { - _ = try self.runMainThreadWork( - "command_execution", - timeout: 0, - timeoutError: self.mainThreadExecutionTimeoutError - ) { - _ = releaseWork.wait(timeout: .now() + 2) - return true - } - } catch { - box.error = error - } - self.mainThreadWorkLock.lock() - box.abandonedCount = self.abandonedMainThreadWorkCount - box.abandonedSinceSet = self.abandonedMainThreadWorkSince != nil - self.mainThreadWorkLock.unlock() - if case .busy = self.currentMainThreadBusyState() { - box.busyWhileAbandoned = true - } - observedAbandoned.signal() - timedOut.fulfill() - } - DispatchQueue(label: "agent-device.runner.tests.release-timeout").async { - _ = observedAbandoned.wait(timeout: .now() + 2) - releaseWork.signal() - } - - wait(for: [timedOut], timeout: 3) - let drainDeadline = Date().addingTimeInterval(2) - while hasAbandonedMainThreadWork(), Date() < drainDeadline { - sleepFor(0.005) - } - - XCTAssertEqual((box.error as NSError?)?.code, RunnerErrorCode.mainThreadExecutionTimedOut) - XCTAssertEqual(box.abandonedCount, 1) - XCTAssertEqual(box.abandonedSinceSet, true) - XCTAssertTrue(box.busyWhileAbandoned) - XCTAssertFalse(hasAbandonedMainThreadWork(), "drained work must release the main thread") - mainThreadWorkLock.lock() - let sinceCleared = abandonedMainThreadWorkSince == nil - mainThreadWorkLock.unlock() - XCTAssertTrue(sinceCleared) - guard case .idle = currentMainThreadBusyState() else { - return XCTFail("expected the runner idle once the abandoned work drained") - } - } -} -#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift index 650d7559bb..a4df398737 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift @@ -303,170 +303,4 @@ extension RunnerTests { let element = app.descendants(matching: .any).matching(predicate).firstMatch return element.exists ? element : nil } - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS - func testTopLeadingNavigationFallbackPointTargetsHeaderControlBand() throws { - let point = try XCTUnwrap( - Self.topLeadingNavigationFallbackPoint( - in: CGRect(x: 0, y: 0, width: 430, height: 932) - ) - ) - - XCTAssertEqual(point.x, 34.4, accuracy: 0.01) - XCTAssertEqual(point.y, 132, accuracy: 0.01) - } - - func testTopLeadingNavigationFallbackPointRejectsInvalidFrame() { - XCTAssertNil(Self.topLeadingNavigationFallbackPoint(in: .infinite)) - XCTAssertNil(Self.topLeadingNavigationFallbackPoint(in: .zero)) - } - - func testNavigationBackControlRankPrefersBackThenCloseThenCancel() { - XCTAssertEqual(Self.navigationBackControlRank(label: "Back", identifier: ""), 0) - XCTAssertEqual(Self.navigationBackControlRank(label: "Close", identifier: ""), 1) - XCTAssertEqual(Self.navigationBackControlRank(label: "Cancel search", identifier: ""), 2) - XCTAssertNil(Self.navigationBackControlRank(label: "Search for more feeds", identifier: "")) - } - - func testNavigationBackPredicateUsesTheSharedKeywordTable() { - let predicate = Self.navigationBackPredicate() - - XCTAssertTrue(predicate.evaluate(with: ["label": "Back", "identifier": ""])) - XCTAssertTrue(predicate.evaluate(with: ["label": "", "identifier": "close-button"])) - XCTAssertFalse(predicate.evaluate(with: ["label": "Search for more feeds", "identifier": ""])) - } - - func testTopNavigationControlFrameAcceptsOnlyHeaderBand() { - let window = CGRect(x: 0, y: 0, width: 430, height: 932) - - XCTAssertTrue( - Self.isTopNavigationControlFrame( - CGRect(x: 340, y: 84, width: 72, height: 44), - in: window - ) - ) - XCTAssertFalse( - Self.isTopNavigationControlFrame( - CGRect(x: 20, y: 760, width: 72, height: 44), - in: window - ) - ) - XCTAssertFalse(Self.isTopNavigationControlFrame(.infinite, in: window)) - } - - func testNavigationVisualVerificationSeparatesNoChangeFromNoSample() { - XCTAssertEqual( - Self.navigationVisualObservation(before: Data([1, 2, 3]), after: Data([1, 2, 4])), - .changed - ) - XCTAssertEqual( - Self.navigationVisualObservation(before: Data([1, 2, 3]), after: Data([1, 2, 3])), - .unchanged - ) - // A missing sample is neither a change nor a no-change; treating it as "unchanged" would let a - // capture that refused become the reason the `back` command claims no control exists (#2728). - XCTAssertEqual(Self.navigationVisualObservation(before: nil, after: Data([1])), .unobserved) - XCTAssertEqual(Self.navigationVisualObservation(before: Data([1]), after: nil), .unobserved) - XCTAssertEqual(Self.navigationVisualObservation(before: nil, after: nil), .unobserved) - } - - func testNavigationFallbackReportsTheRefusalItHitNotADefaultCode() { - // The refusal from the most recent sample wins, so the code names what the fallback last looked at - // before giving up; an earlier refusal is reported only when the later sample carried none (#2728). - let after = NavigationVisualSample( - data: nil, - refusalCode: "APP_SCREEN_WINDOW_UNRESOLVED", - refusalHint: "after hint" - ) - let before = NavigationVisualSample( - data: nil, - refusalCode: "APP_SCREEN_UNRESOLVED", - refusalHint: "before hint" - ) - let laterWins = Self.navigationFallbackErrorPayload(after: after, before: before) - XCTAssertEqual(laterWins.code, "APP_SCREEN_WINDOW_UNRESOLVED") - XCTAssertEqual(laterWins.hint, "after hint") - - let onlyBefore = Self.navigationFallbackErrorPayload( - after: NavigationVisualSample(data: nil), - before: before - ) - XCTAssertEqual(onlyBefore.code, "APP_SCREEN_UNRESOLVED") - XCTAssertEqual(onlyBefore.hint, "before hint") - - // Neither side named a reason (unreachable on iOS): a real capture code, never a bare failure. - let unnamed = Self.navigationFallbackErrorPayload( - after: NavigationVisualSample(data: nil), - before: NavigationVisualSample(data: nil) - ) - XCTAssertEqual(unnamed.code, "APP_SCREEN_UNRESOLVED") - XCTAssertTrue(unnamed.message.contains("unknown outcome")) - } - - func testVerifyNavigationFallbackOutcomeReportsUnresolvedWindowWithoutSystemSurface() { - // The in-app `back` fallback ran its tap but the app resolved no window. It must name that refusal - // as an unknown outcome AND must not have sampled the system surface: capturing SpringBoard's home - // screen twice reads as "unchanged" and launders a wrong-process frame into "no back control - // exists" (#2728). This drives the SAME `navigationFallbackSample` production calls, handing it a - // system surface that fails the test if consulted — so reverting the fallback to sample SpringBoard - // (or dropping the refusal code) turns this red, which an inline `.never` re-creation could not. - var askedSystemSurface = false - let sample = Self.navigationFallbackSample( - resolvingApp: { .failure(.unresolvedWindow) }, - systemSurface: { - askedSystemSurface = true - return .failure(.unresolvedWindow) - }, - encoding: { _ in Data([1, 2, 3]) } - ) - XCTAssertFalse(askedSystemSurface, "the in-app fallback samples the app only, never SpringBoard") - - XCTAssertNil(sample.data) - XCTAssertEqual(sample.refusalCode, "APP_SCREEN_WINDOW_UNRESOLVED") - - let observation = Self.navigationVisualObservation(before: sample.data, after: sample.data) - XCTAssertEqual(observation, .unobserved) - - switch Self.inAppBackOutcome(observation: observation, before: sample, after: sample) { - case .unverified(let payload): - XCTAssertEqual(payload.code, "APP_SCREEN_WINDOW_UNRESOLVED") - case .performed, .unavailable: - XCTFail("a refused capture must report an unknown outcome, not 'no back control'") - } - } - - func testNavigationVisualSampleDistinguishesEncodedFrameFromRefusal() { - // The same capture entry point yields three different samples, and only the refusal ones may carry - // a code: an encoded frame is evidence, a resolved-but-unencodable image and a refusal are not - // (#2728). Reverting the mapping to a plain no-sample loses the reason a host keys on. - let captured = CapturedAppScreen( - image: RunnerImage(), - displayID: 3, - pixelWidth: 12, - pixelHeight: 24, - pixelsPerPoint: 3 - ) - - let encoded = Self.navigationVisualSample( - from: .success(captured), - encoding: { _ in Data([7, 7]) } - ) - XCTAssertEqual(encoded.data, Data([7, 7])) - XCTAssertNil(encoded.refusalCode) - - let unencodable = Self.navigationVisualSample( - from: .success(captured), - encoding: { _ in nil } - ) - XCTAssertNil(unencodable.data) - XCTAssertEqual(unencodable.refusalCode, "APP_SCREEN_CAPTURE_UNRENDERABLE") - - let refused = Self.navigationVisualSample( - from: .failure(.unresolvedScreen), - encoding: { _ in Data([7, 7]) } - ) - XCTAssertNil(refused.data) - XCTAssertEqual(refused.refusalCode, "APP_SCREEN_UNRESOLVED") - } -#endif } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+PrivateAXPresentation.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+PrivateAXPresentation.swift index 202e2d061b..b95b1dc5a1 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+PrivateAXPresentation.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+PrivateAXPresentation.swift @@ -89,212 +89,3 @@ extension RunnerTests { (value as? Bool) ?? (value as? NSNumber)?.boolValue } } - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS -extension RunnerTests { - fileprivate static func privateAXFrame( - _ x: Double, _ y: Double, _ width: Double, _ height: Double - ) -> [String: Any] { - ["x": x, "y": y, "width": width, "height": height] - } - - /// A scroll container whose second row is scrolled out of the viewport, plus an unlabeled - /// decoration: the shapes the regular projection folds away and the raw projection must keep. - fileprivate static var privateAXScrolledFixture: [String: Any] { - let frame = privateAXFrame - return ["type": Int(XCUIElement.ElementType.application.rawValue), - "label": "Element", "frame": frame(0, 0, 402, 874), "children": [[ - "type": Int(XCUIElement.ElementType.scrollView.rawValue), "frame": frame(0, 96, 402, 700), - "actions": ["Scroll down"], - "children": [ - ["type": Int(XCUIElement.ElementType.button.rawValue), "label": "Profile picture", - "frame": frame(16, 120, 44, 44), - "children": [["type": Int(XCUIElement.ElementType.image.rawValue), - "frame": frame(16, 120, 1, 1)]]], - ["type": Int(XCUIElement.ElementType.button.rawValue), "label": "Theme", - "frame": frame(16, 900, 360, 44)]]]]] - } - - /// Acquire with the private-AX serializer, run the one normalization pass the production capture - /// plan runs (`captureWithBackend`), then present through the shared regular fold -- the production - /// route for this backend since the fold moved into presentation (#1797, #2661). - fileprivate func privateAXRegularPresentation( - rawRoot: [String: Any], - viewport: CGRect, - interactiveOnly: Bool = false - ) throws -> [PresentedNode] { - let hint = CaptureHint( - projection: .regular, depth: nil, regularPresentedDepth: nil, - interactiveOnly: interactiveOnly, customActions: false) - let nodes = privateAXNormalizedAcquisition( - rawRoot: rawRoot, hint: hint, viewport: viewport, - interfaceOrientation: RunnerInterfaceOrientation.portrait) - return try SnapshotPresentation.presentRegular( - SnapshotAcquisition( - hint: hint, nodes: nodes, truncated: false, effectiveDepth: nil, viewport: viewport, - interfaceOrientation: RunnerInterfaceOrientation.portrait), - options: PresentationOptions( - interactiveOnly: interactiveOnly, depth: nil, scope: nil, raw: false), - policy: .cursorProjected - ).nodes - } - - /// Acquire, then normalize once -- the exact pair `captureWithBackend` runs for this backend. - fileprivate func privateAXNormalizedAcquisition( - rawRoot: [String: Any], - hint: CaptureHint, - viewport: CGRect, - interfaceOrientation: Int - ) -> [RawAXNode] { - SnapshotGeometrySpace.normalized( - nodes: privateAXAcquisition(rawRoot: rawRoot, hint: hint), - viewport: viewport, - interfaceOrientation: interfaceOrientation - ) - } - - /// The one normalization pass has to reach the node it publishes: this asserts the rotated rect of - /// a key under a turned surface host, and an untouched sibling under the app's own window, so a - /// pass that drops the space a subtree declared fails here. - func testPrivateAXAcquisitionPublishesATurnedSurfaceHostInAppOrientationSpace() { - let frame = Self.privateAXFrame - let appWindow: [String: Any] = [ - "type": Int(XCUIElement.ElementType.window.rawValue), - "frame": frame(0, 0, 874, 402), - "children": [ - ["type": Int(XCUIElement.ElementType.button.rawValue), "label": "Home", - "frame": frame(204, 323, 91, 55), "children": []] - ] - ] - let keyboardWindow: [String: Any] = [ - "type": Int(XCUIElement.ElementType.window.rawValue), - "frame": frame(0, 0, 874, 402), - "children": [ - ["type": Int(XCUIElement.ElementType.other.rawValue), - "frame": frame(0, 0, 402, 874), - "children": [ - ["type": Int(XCUIElement.ElementType.key.rawValue), "label": "q", - "frame": frame(154, 77, 45, 72), "children": []] - ]] - ] - ] - let hint = CaptureHint( - projection: .raw, depth: nil, regularPresentedDepth: nil, - interactiveOnly: false, customActions: false) - let nodes = privateAXNormalizedAcquisition( - rawRoot: [ - "type": Int(XCUIElement.ElementType.application.rawValue), - "label": "Element", "frame": frame(0, 0, 874, 402), - "children": [appWindow, keyboardWindow] - ], - hint: hint, - viewport: CGRect(x: 0, y: 0, width: 874, height: 402), - interfaceOrientation: RunnerInterfaceOrientation.landscapeRight - ) - - // Measured on iPhone 17 Pro (26.2): the key plane's left column arrives 154 pt along the device's - // long axis and comes back 203 pt down the app's short one. - XCTAssertEqual( - nodes.first { $0.label == "q" }?.rect, - SnapshotRect(x: 77, y: 203, width: 72, height: 45) - ) - XCTAssertEqual( - nodes.first { $0.label == "Home" }?.rect, - SnapshotRect(x: 204, y: 323, width: 91, height: 55) - ) - } - - func testPrivateAXRegularPresentationProjectsToViewportAndKeepsScrollHint() throws { - let nodes = try privateAXRegularPresentation( - rawRoot: Self.privateAXScrolledFixture, - viewport: CGRect(x: 0, y: 0, width: 402, height: 874)) - XCTAssertEqual(nodes.compactMap(\.label), ["Element", "Profile picture"]) - let scrollView = nodes.first { $0.type == "ScrollView" } - XCTAssertEqual(scrollView?.hiddenContentBelow, true) - XCTAssertEqual(scrollView?.actions, ["Scroll down"]) - } - - /// #1797 D4: the raw projection is the acquired tree. The offscreen row and the sub-pixel - /// decoration the regular projection folds away are both present, at traversal depth, and every - /// regular node still appears -- `regular ⊆ raw` on the same capture. - func testPrivateAXRawProjectionKeepsEveryAcquiredNode() throws { - let viewport = CGRect(x: 0, y: 0, width: 402, height: 874) - let root = Self.privateAXScrolledFixture - let regular = try privateAXRegularPresentation(rawRoot: root, viewport: viewport, - interactiveOnly: true) - let raw = privateAXNormalizedAcquisition(rawRoot: root, - hint: CaptureHint( - projection: .raw, depth: nil, regularPresentedDepth: nil, - interactiveOnly: false, customActions: false), - viewport: viewport, - interfaceOrientation: RunnerInterfaceOrientation.portrait) - - XCTAssertEqual(raw.map(\.type), ["Application", "ScrollView", "Button", "Image", "Button"]) - XCTAssertEqual(raw.map(\.depth), [0, 1, 2, 3, 2]) - XCTAssertEqual(raw.map(\.parentIndex), [nil, 0, 1, 2, 1]) - XCTAssertEqual(raw.compactMap(\.label), ["Element", "Profile picture", "Theme"]) - // The offscreen row is a reported fact in raw, and reported facts do not become hittable - // just because the projection kept them. - XCTAssertEqual(raw.last?.hittable, false) - XCTAssertTrue(raw.allSatisfy { $0.hiddenContentAbove == nil && $0.hiddenContentBelow == nil }) - - let rawKeys = Set(raw.map { "\($0.type)-\($0.label ?? "")-\($0.rect.y)" }) - for node in regular { - XCTAssertTrue( - rawKeys.contains("\(node.type)-\(node.label ?? "")-\(node.rect.y)"), - "regular node \(node.type)/\(node.label ?? "") is missing from the raw projection" - ) - } - XCTAssertGreaterThan(raw.count, regular.count) - } - - /// Raw depth is traversal depth, so a raw `--depth` request is the one narrowing this backend - /// can prove complete. - func testPrivateAXRawProjectionAppliesRequestedTraversalDepth() { - let raw = privateAXNormalizedAcquisition(rawRoot: Self.privateAXScrolledFixture, - hint: CaptureHint( - projection: .raw, depth: 2, regularPresentedDepth: nil, - interactiveOnly: false, customActions: false), - viewport: CGRect(x: 0, y: 0, width: 402, height: 874), - interfaceOrientation: RunnerInterfaceOrientation.portrait) - XCTAssertEqual(raw.map(\.type), ["Application", "ScrollView", "Button", "Button"]) - XCTAssertEqual(raw.map(\.depth), [0, 1, 2, 2]) - } - - func testPrivateAXPresentationKeepsOffscreenSubtreeExcludedWhenChildFramesAreClamped() throws { - let frame = Self.privateAXFrame - let root: [String: Any] = ["type": Int(XCUIElement.ElementType.application.rawValue), - "label": "Element", "frame": frame(0, 0, 402, 874), "children": [[ - "type": Int(XCUIElement.ElementType.table.rawValue), "frame": frame(0, 96, 402, 700), - "children": [["type": Int(XCUIElement.ElementType.cell.rawValue), - "label": "Theme", "frame": frame(0, 900, 402, 44), "children": [[ - "type": Int(XCUIElement.ElementType.staticText.rawValue), - "label": "Theme", "frame": frame(16, 96, 120, 44)], - ["type": Int(XCUIElement.ElementType.switch.rawValue), - "label": "Theme", "frame": frame(340, 96, 46, 44)]]]]]]] - - let nodes = try privateAXRegularPresentation( - rawRoot: root, viewport: CGRect(x: 0, y: 0, width: 402, height: 874)) - - XCTAssertEqual(nodes.compactMap(\.label), ["Element"]) - XCTAssertEqual(nodes.first { $0.type == "Table" }?.hiddenContentBelow, true) - } - - func testPrivateAXGeometrylessSemanticsAreNeverActionableOrScrollContexts() throws { - let zero = ["x": 0, "y": 0, "width": 0, "height": 0] - let root: [String: Any] = ["type": Int(XCUIElement.ElementType.application.rawValue), - "label": "Element", "frame": ["x": 0, "y": 0, "width": 402, "height": 874], - "children": [["type": Int(XCUIElement.ElementType.scrollView.rawValue), - "label": "Settings semantics", "frame": zero, "children": [[ - "type": Int(XCUIElement.ElementType.button.rawValue), "label": "Theme", "frame": zero], - ["type": Int(XCUIElement.ElementType.other.rawValue), "frame": zero]]]]] - let nodes = try privateAXRegularPresentation( - rawRoot: root, viewport: CGRect(x: 0, y: 0, width: 402, height: 874), - interactiveOnly: true) - XCTAssertEqual(nodes.compactMap(\.label), ["Element", "Settings semantics", "Theme"]) - XCTAssertEqual(nodes.filter { $0.index != 0 }.map(\.hittable), [false, false]) - XCTAssertFalse(nodes.contains { $0.type == "Other" }) - XCTAssertTrue(nodes.allSatisfy { $0.hiddenContentAbove == nil && $0.hiddenContentBelow == nil }) - } -} -#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollGesture.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollGesture.swift index f6ebb6866b..326eec698f 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollGesture.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollGesture.swift @@ -5,8 +5,9 @@ import XCTest // This is a deliberate two-place invariant: the daemon keeps the TS implementation (for Android, // recording, and reported-pixels), and the runner places the gesture with this Swift copy. Both // ports are asserted against the same table, contracts/fixtures/scroll-gesture.json (gated -// XCTest at the bottom of this file, vitest twin packages/contracts/src/scroll-gesture.test.ts) — -// if you change the math in either language, update the other and the table. +// XCTest in UnitTests/RunnerTests+ScrollGestureTests.swift, vitest twin +// packages/contracts/src/scroll-gesture.test.ts) — if you change the math in either language, +// update the other and the table. // // All inputs here are positive (reference dims, travel, center), so Swift's `.rounded()` // (half away from zero) matches JS `Math.round` (half up) on every value computed below. @@ -96,198 +97,3 @@ func runnerDragCommandDefaults(_ command: Command) -> RunnerDragCommandDefaults durationMs: command.durationMs ?? runnerDefaultDragDurationMs ) } - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS -private struct ScrollGestureFixture: Decodable { - struct Constants: Decodable { - let defaultIosScrollAmount: Double - let defaultMobileScrollDurationMs: Double - let defaultIosScrollDurationMs: Double - let defaultScrollAmount: Double - let defaultEdgePaddingFraction: Double - let ordinaryScrollReleaseBehavior: String - let edgeScrollReleaseBehavior: String - } - struct Expected: Decodable { - let x1: Double - let y1: Double - let x2: Double - let y2: Double - let pixels: Double - } - struct Case: Decodable { - let name: String - let direction: String - let amount: Double? - let pixels: Double? - let referenceWidth: Double - let referenceHeight: Double - let expected: Expected - } - - let constants: Constants - let cases: [Case] -} - -extension RunnerTests { - // Cross-language parity table: every case in contracts/fixtures/scroll-gesture.json must agree - // with the vitest twin (packages/contracts/src/scroll-gesture.test.ts). Add vectors there, - // never fork the math. - private func loadScrollGestureFixture() throws -> ScrollGestureFixture { - let fixtureURL = URL(fileURLWithPath: #filePath) - .deletingLastPathComponent() // AgentDeviceRunnerUITests - .deletingLastPathComponent() // AgentDeviceRunner - .deletingLastPathComponent() // runner - .deletingLastPathComponent() // apple - .deletingLastPathComponent() // repo root - .appendingPathComponent("contracts") - .appendingPathComponent("fixtures") - .appendingPathComponent("scroll-gesture.json") - return try JSONDecoder().decode(ScrollGestureFixture.self, from: Data(contentsOf: fixtureURL)) - } - - func testRunnerScrollGesturePlanMatchesParityTable() throws { - let fixture = try loadScrollGestureFixture() - XCTAssertFalse(fixture.cases.isEmpty, "parity table must not be empty") - for testCase in fixture.cases { - let plan = try XCTUnwrap( - runnerScrollGesturePlan( - direction: try XCTUnwrap(RunnerScrollDirection(rawValue: testCase.direction)), - amount: testCase.amount, - pixels: testCase.pixels, - referenceWidth: testCase.referenceWidth, - referenceHeight: testCase.referenceHeight - ), - testCase.name - ) - XCTAssertEqual(plan.x1, testCase.expected.x1, testCase.name) - XCTAssertEqual(plan.y1, testCase.expected.y1, testCase.name) - XCTAssertEqual(plan.x2, testCase.expected.x2, testCase.name) - XCTAssertEqual(plan.y2, testCase.expected.y2, testCase.name) - XCTAssertEqual(plan.travelPixels, testCase.expected.pixels, testCase.name) - } - } - - // The planner constants are private on both sides; the table pins them behaviourally on a - // 1000px axis where every rounding step is exact. - func testRunnerScrollGesturePlanUsesParityTableConstants() throws { - let constants = try loadScrollGestureFixture().constants - let defaultScroll = try JSONDecoder().decode( - Command.self, from: Data(#"{"command":"scroll"}"#.utf8)) - let defaults = runnerDragCommandDefaults(defaultScroll) - XCTAssertEqual(defaults.durationMs, constants.defaultIosScrollDurationMs) - XCTAssertEqual(defaults.scrollAmount, constants.defaultIosScrollAmount) - let defaulted = try XCTUnwrap( - runnerScrollGesturePlan( - direction: .down, amount: nil, pixels: nil, referenceWidth: 1000, referenceHeight: 1000 - ) - ) - XCTAssertEqual(defaulted.travelPixels, 1000 * constants.defaultScrollAmount) - let saturated = try XCTUnwrap( - runnerScrollGesturePlan( - direction: .down, amount: 10, pixels: nil, referenceWidth: 1000, referenceHeight: 1000 - ) - ) - XCTAssertEqual( - saturated.travelPixels, 1000 - 2 * 1000 * constants.defaultEdgePaddingFraction) - } - - func testRunnerScrollAndDragCommandDefaultsStayDistinct() throws { - func command(_ json: String) throws -> Command { - try JSONDecoder().decode(Command.self, from: Data(json.utf8)) - } - - let pixelScroll = runnerDragCommandDefaults( - try command(#"{"command":"scroll","pixels":120}"#)) - XCTAssertNil(pixelScroll.scrollAmount) - XCTAssertEqual(pixelScroll.durationMs, 400) - - let drag = runnerDragCommandDefaults(try command(#"{"command":"drag"}"#)) - XCTAssertNil(drag.scrollAmount) - XCTAssertEqual(drag.durationMs, 250) - - let explicitScroll = runnerDragCommandDefaults( - try command(#"{"command":"scroll","amount":0.5,"durationMs":125}"#)) - XCTAssertEqual(explicitScroll.scrollAmount, 0.5) - XCTAssertEqual(explicitScroll.durationMs, 125) - - let explicitDrag = runnerDragCommandDefaults( - try command(#"{"command":"drag","durationMs":125}"#)) - XCTAssertEqual(explicitDrag.durationMs, 125) - } - - func testRunnerScrollReleaseBehaviorSelectsTheDragProfile() throws { - let constants = try loadScrollGestureFixture().constants - let controlled = try XCTUnwrap(ScrollReleaseBehavior(rawValue: constants.ordinaryScrollReleaseBehavior)) - let inertial = try XCTUnwrap(ScrollReleaseBehavior(rawValue: constants.edgeScrollReleaseBehavior)) - XCTAssertEqual( - scrollDragProfile(releaseBehavior: nil), - .controlledScroll - ) - XCTAssertEqual( - scrollDragProfile(releaseBehavior: controlled), - .controlledScroll - ) - XCTAssertEqual( - scrollDragProfile(releaseBehavior: inertial), - .fastSwipe - ) - } - - func testControlledScrollProfileUsesReliableCadenceAndMonotonicDeceleration() { - XCTAssertEqual(RunnerControlledScrollFrameCount(350), 21) - XCTAssertEqual(RunnerControlledScrollFrameCount(400), 24) - XCTAssertEqual(RunnerControlledScrollFrameCount(500), 30) - XCTAssertEqual(RunnerControlledScrollFrameCount(1_000), 30) - XCTAssertEqual(RunnerControlledScrollFrameCount(10_000), 30) - - let frameCount = RunnerControlledScrollFrameCount(400) - let progress = (0...frameCount).map { - RunnerControlledScrollProgress(Double($0) / Double(frameCount)) - } - let deltas = zip(progress.dropFirst(), progress).map { $0.0 - $0.1 } - XCTAssertEqual(progress.first, 0) - XCTAssertEqual(progress.last, 1) - XCTAssertTrue(zip(deltas, deltas.dropFirst()).allSatisfy { $0.1 <= $0.0 }) - - let iPhoneViewportPoints = 874.0 - let defaultFingerTravel = iPhoneViewportPoints * 0.65 - let finalSampleTravel = (1 - progress[progress.count - 2]) * defaultFingerTravel - XCTAssertLessThan(finalSampleTravel, 0.1) - } - - func testRunnerScrollGesturePlanRejectsUnknownDirection() { - XCTAssertNil(RunnerScrollDirection(rawValue: "sideways")) - } - - func testRunnerScrollGesturePlanRejectsInvalidAmountAndPixels() { - XCTAssertNil( - runnerScrollGesturePlan( - direction: .down, - amount: 0, - pixels: nil, - referenceWidth: 300, - referenceHeight: 600 - ) - ) - XCTAssertNil( - runnerScrollGesturePlan( - direction: .down, - amount: nil, - pixels: -10, - referenceWidth: 300, - referenceHeight: 600 - ) - ) - XCTAssertNil( - runnerScrollGesturePlan( - direction: .down, - amount: .infinity, - pixels: nil, - referenceWidth: 300, - referenceHeight: 600 - ) - ) - } -} -#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift index 8b0b5da622..b6ea551520 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift @@ -239,188 +239,3 @@ extension RunnerTests { Response(ok: false, error: ErrorPayload(code: "INVALID_ARGS", message: message)) } } - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS -// MARK: - In-bundle unit tests (device-free) - -extension RunnerTests { - func testSequenceDecodesStepsFromWire() throws { - let json = """ - {"command":"sequence","commandId":"seq-1","steps":[ - {"kind":"tap","x":100,"y":200}, - {"kind":"doubleTap","x":101,"y":200}, - {"kind":"longPress","x":102,"y":200,"durationMs":300,"pauseMs":50} - ]} - """ - let command = try JSONDecoder().decode(Command.self, from: Data(json.utf8)) - XCTAssertEqual(command.command, .sequence) - XCTAssertEqual(command.steps?.count, 3) - XCTAssertEqual(command.steps?[0].kind, "tap") - XCTAssertEqual(command.steps?[1].kind, "doubleTap") - XCTAssertEqual(command.steps?[2].pauseMs, 50) - } - - func testSequenceAcceptsDoubleTapKind() { - // A doubleTap step missing coords must fail on the coords check, not the kind allowlist — - // proving "doubleTap" passes validateSequenceStep without needing a device to execute on. - let response = executeSequenceForTest(steps: [ - sequenceStep(kind: "doubleTap", x: nil) - ]) - XCTAssertEqual(response.ok, false) - XCTAssertEqual(response.error?.code, "INVALID_ARGS") - XCTAssertTrue(response.error?.message.contains("requires finite x and y") ?? false) - XCTAssertFalse(response.error?.message.contains("unsupported kind") ?? true) - } - - func testSequenceRejectsUnknownKind() throws { - let response = executeSequenceForTest(steps: [ - sequenceStep(kind: "tap", x: 1, y: 2), - sequenceStep(kind: "pinch", x: 3, y: 4), - ]) - XCTAssertEqual(response.ok, false) - XCTAssertEqual(response.error?.code, "INVALID_ARGS") - XCTAssertTrue(response.error?.message.contains("step 1") ?? false) - XCTAssertTrue(response.error?.message.contains("pinch") ?? false) - } - - func testSequenceRejectsEmpty() { - let response = executeSequenceForTest(steps: []) - XCTAssertEqual(response.ok, false) - XCTAssertEqual(response.error?.code, "INVALID_ARGS") - } - - func testSequenceRejectsTooManySteps() { - let steps = (0..<21).map { _ in sequenceStep(kind: "tap", x: 1, y: 2) } - let response = executeSequenceForTest(steps: steps) - XCTAssertEqual(response.ok, false) - XCTAssertEqual(response.error?.code, "INVALID_ARGS") - XCTAssertTrue(response.error?.message.contains("at most 20") ?? false) - } - - func testSequenceHasSynthesizedCoordinateStep() { - XCTAssertTrue( - sequenceHasSynthesizedCoordinateStep([ - sequenceStep(kind: "tap", x: 1, y: 2, synthesized: true), - ]) - ) - XCTAssertFalse( - sequenceHasSynthesizedCoordinateStep([ - sequenceStep(kind: "tap", x: 1, y: 2), - sequenceStep(kind: "doubleTap", x: 1, y: 2, synthesized: true), - ]) - ) - } - - func testAssembleSequencePreservesOrderOnSuccess() { - let steps = [ - sequenceStep(kind: "tap", x: 1, y: 1), - sequenceStep(kind: "longPress", x: 2, y: 2), - sequenceStep(kind: "tap", x: 3, y: 3), - ] - var calls: [Int] = [] - let execution = assembleSequenceExecution(steps: steps) { index, _ in - calls.append(index) - return SequenceStepOutcome( - outcome: .performed, - gestureStartUptimeMs: Double(index * 10), - gestureEndUptimeMs: Double(index * 10 + 5) - ) - } - XCTAssertEqual(calls, [0, 1, 2]) - XCTAssertEqual(execution.completedSteps, 3) - XCTAssertNil(execution.failedStepIndex) - XCTAssertEqual(execution.results.map { $0.kind }, ["tap", "longPress", "tap"]) - XCTAssertEqual(execution.gestureStartUptimeMs, 0) - XCTAssertEqual(execution.gestureEndUptimeMs, 25) - } - - func testAssembleSequenceStopsAtFirstFailure() { - let steps = [ - sequenceStep(kind: "tap", x: 1, y: 1), - sequenceStep(kind: "longPress", x: 2, y: 2), - sequenceStep(kind: "tap", x: 3, y: 3), - ] - var calls: [Int] = [] - let execution = assembleSequenceExecution(steps: steps) { index, _ in - calls.append(index) - if index == 1 { - return SequenceStepOutcome( - outcome: .unsupported(message: "long press unsupported", hint: nil), - gestureStartUptimeMs: 10, - gestureEndUptimeMs: 15 - ) - } - return SequenceStepOutcome(outcome: .performed, gestureStartUptimeMs: 0, gestureEndUptimeMs: 5) - } - // Step 2 is never invoked. - XCTAssertEqual(calls, [0, 1]) - XCTAssertEqual(execution.completedSteps, 1) - XCTAssertEqual(execution.failedStepIndex, 1) - // results.count == completedSteps + 1 (the failed step). - XCTAssertEqual(execution.results.count, 2) - XCTAssertEqual(execution.results[1].ok, false) - XCTAssertEqual(execution.results[1].errorCode, "UNSUPPORTED_OPERATION") - XCTAssertEqual(execution.results[1].errorMessage, "long press unsupported") - } - - func testSequenceWorstCaseResponseStaysUnderJournalCap() throws { - let longMessage = String(repeating: "e", count: 200) - let results = (0..<20).map { index in - SequenceStepResult( - ok: index < 19, - kind: "longPress", - errorCode: index < 19 ? nil : "UNSUPPORTED_OPERATION", - errorMessage: index < 19 ? nil : longMessage, - gestureStartUptimeMs: 123456.789, - gestureEndUptimeMs: 123466.789 - ) - } - let response = Response( - ok: true, - data: DataPayload( - message: "sequence", - completedSteps: 19, - failedStepIndex: 19, - sequenceResults: results - ) - ) - let encoded = try JSONEncoder().encode(response) - XCTAssertLessThan(encoded.count, 16 * 1024) - } - - private func sequenceStep( - kind: String, - x: Double?, - y: Double? = nil, - synthesized: Bool? = nil - ) -> SequenceStep { - SequenceStep( - kind: kind, - x: x, - y: y, - durationMs: nil, - pauseMs: nil, - synthesized: synthesized - ) - } - - /// Validation runs before any executor call, so the INVALID_ARGS paths are exercised without - /// reaching the device executor (which is never invoked when validation rejects). - private func executeSequenceForTest(steps: [SequenceStep]) -> Response { - let command = makeSequenceCommand(steps: steps) - return executeSequence(command: command, activeApp: app) - } - - /// Build a sequence Command via JSON so the test does not depend on the memberwise init's - /// parameter order. - private func makeSequenceCommand(steps: [SequenceStep]) -> Command { - struct SequenceCommandFixture: Encodable { - let command = "sequence" - let commandId = "seq-test" - let steps: [SequenceStep] - } - let data = try! JSONEncoder().encode(SequenceCommandFixture(steps: steps)) - return try! JSONDecoder().decode(Command.self, from: data) - } -} -#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift index 83686731cf..14fe138387 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift @@ -5,12 +5,12 @@ extension RunnerTests { static let axSnapshotErrorCode = "IOS_AX_SNAPSHOT_FAILED" static let axSnapshotFailureMessage = "iOS XCTest snapshot failed while serializing the accessibility tree." - private static let axSnapshotUnavailableReason = "ax_snapshot_unavailable" + static let axSnapshotUnavailableReason = "ax_snapshot_unavailable" static let axSnapshotHint = "Snapshot state is unavailable because XCTest could not serialize this iOS accessibility tree. This can be specific to the current screen. Use plain screenshot, not screenshot --overlay-refs, as visual truth; navigate with coordinate commands if needed; then retry snapshot -i after reaching another screen. If you own the app and need full-tree inspection, simplify this screen's accessibility tree and expose stable ids on actionable controls." - private static let rawSnapshotTooLargeCode = "IOS_RAW_SNAPSHOT_TOO_LARGE" - private static let rawSnapshotMaxNodes = 5_000 - private static let rawSnapshotTooLargeHint = + static let rawSnapshotTooLargeCode = "IOS_RAW_SNAPSHOT_TOO_LARGE" + static let rawSnapshotMaxNodes = 5_000 + static let rawSnapshotTooLargeHint = "Raw iOS snapshot exceeded the runner payload guard. Use regular snapshot for visible UI, or scope/depth-limit raw snapshot when inspecting a large accessibility tree." // Runaway guard for the regular tree walk: a work bound only. A screen that trips it raises this // number in ADR 0004's name rather than bounding the walk by geometry again. @@ -523,11 +523,11 @@ extension RunnerTests { ) } - private func recoveredSnapshotMessage(_ failure: SnapshotCaptureFailure) -> String { + func recoveredSnapshotMessage(_ failure: SnapshotCaptureFailure) -> String { return "\(failure.message) Hint: \(failure.hint)" } - private func rawSnapshotTooLargeFailure(nodeCount: Int) -> SnapshotCaptureFailure { + func rawSnapshotTooLargeFailure(nodeCount: Int) -> SnapshotCaptureFailure { SnapshotCaptureFailure( code: Self.rawSnapshotTooLargeCode, message: "iOS raw snapshot exceeded \(Self.rawSnapshotMaxNodes) nodes while walking node \(nodeCount).", @@ -558,246 +558,4 @@ extension RunnerTests { runnerFatalReason: runnerFatalReason ) } - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS - func testSnapshotAccessibilityUnavailableMarksSparseSnapshotRunnerFatal() { - currentApp = app - currentBundleId = "com.example.app" - - let payload = snapshotAccessibilityUnavailable( - failure: SnapshotCaptureFailure( - code: Self.axSnapshotErrorCode, - message: Self.axSnapshotFailureMessage, - hint: Self.axSnapshotHint - ) - ) - - XCTAssertEqual(payload.message, "\(Self.axSnapshotFailureMessage) Hint: \(Self.axSnapshotHint)") - XCTAssertEqual(payload.nodes?.count, 1) - XCTAssertEqual(payload.nodes?.first?.type, "Application") - XCTAssertEqual(payload.truncated, true) - XCTAssertEqual(payload.runnerFatal, true) - XCTAssertEqual(payload.runnerFatalReason, Self.axSnapshotUnavailableReason) - // The planned terminal result carries the structured verdict like every other planned - // snapshot — downstream sparse handling keys off it, not off node shapes. - XCTAssertEqual(payload.snapshotQuality?.state, "sparse") - XCTAssertEqual(payload.snapshotQuality?.reasonCode, "ax-rejected") - XCTAssertEqual(payload.snapshotQuality?.reason, Self.axSnapshotFailureMessage) - XCTAssertNil(currentApp) - XCTAssertNil(currentBundleId) - } - - func testRecoveredSnapshotMessagePreservesHint() { - let message = recoveredSnapshotMessage( - SnapshotCaptureFailure( - code: Self.axSnapshotErrorCode, - message: Self.axSnapshotFailureMessage, - hint: Self.axSnapshotHint - ) - ) - - XCTAssertTrue(message.contains(Self.axSnapshotFailureMessage)) - XCTAssertTrue(message.contains(Self.axSnapshotHint)) - } - - func testRawSnapshotTooLargeFailureIsStructured() { - let failure = rawSnapshotTooLargeFailure(nodeCount: Self.rawSnapshotMaxNodes + 1) - - XCTAssertEqual(failure.code, Self.rawSnapshotTooLargeCode) - XCTAssertTrue(failure.message.contains("\(Self.rawSnapshotMaxNodes) nodes")) - XCTAssertEqual(failure.hint, Self.rawSnapshotTooLargeHint) - } - - func testSystemModalProbeSliceSharesAndClampsToPlanDeadline() { - // Fresh plan deadline: the probe gets its full dedicated budget. - XCTAssertEqual(Self.systemModalProbeSlice(budget: 4, deadlineRemaining: 20), 4) - // Nearly-spent plan deadline: the probe is clamped so it can't run past the shared budget. - XCTAssertEqual(Self.systemModalProbeSlice(budget: 4, deadlineRemaining: 1.5), 1.5) - // Exactly/already exhausted deadline: skip the probe entirely (0), never a negative timeout. - XCTAssertEqual(Self.systemModalProbeSlice(budget: 4, deadlineRemaining: 0), 0) - XCTAssertEqual(Self.systemModalProbeSlice(budget: 4, deadlineRemaining: -5), 0) - } - - // Simulator-only: the bounded probe body returns nil on macOS (no SpringBoard host), so the - // timeout/penalty/drain machinery below only exists on the iOS branch. -#if os(iOS) - /// Regression for #1244/#1248: drives the bounded system-modal probe through a real, - /// production-only command entry point (`snapshotFast` or `snapshotRaw` -- see the two test - /// methods below), not `boundedBlockingSystemAlertSnapshot` directly, with - /// `systemModalProbeOverrideForTesting` set to a closure that blocks past the probe's real - /// slice, forcing a real `runMainThreadWork` timeout. This is revert-sensitive on both halves - /// of the fix, for either entry point: - /// - if the entry point reverted to calling the unbounded `blockingSystemAlertSnapshot` - /// directly (or dropped the `runMainThreadWork` wrap), nothing here would ever time out, - /// so the mid-flight busy/penalty assertions below would never be met; - /// - if the `onAbandoned` penalty hook or the abandoned-work accounting were dropped, the - /// timeout would still fire, but the busy/penalty and drain assertions would not hold. - /// - /// The drain assertion is synchronized on the *real* release rather than raced: after - /// signaling the probe to finish, the background queue polls `hasAbandonedMainThreadWork()` - /// (bounded) and only then fulfills `drained`, which the test `wait(for:timeout:)`s on before - /// asserting `.idle`/`hasAbandonedMainThreadWork() == false` below -- so a slow drain fails that - /// assertion instead of racing a fixed-timing guess. - private func assertBoundedSystemModalProbeTimeoutRecoversThenReleasesOnDrain( - entryPointName: String, - callEntryPoint: @escaping (XCUIApplication, PresentationOptions) throws -> DataPayload - ) { - let targetBundleId = "com.callstack.agentdevice.runner.missing.snapshot-timeout-test" - let snapshotTarget = XCUIApplication(bundleIdentifier: targetBundleId) - let probeReleaseGate = DispatchSemaphore(value: 0) - currentApp = snapshotTarget - currentBundleId = targetBundleId - defer { - probeReleaseGate.signal() - currentApp = nil - currentBundleId = nil - systemModalProbeOverrideForTesting = nil - clearSnapshotXCTestChannelPenalty(reason: "test-cleanup") - } - - final class ResultBox { - var payload: DataPayload? - var wasBusyBeforeDrain = false - var hadAbandonedCaptureBeforeDrain = false - var wasPenalizedBeforeDrain = false - } - let box = ResultBox() - // The test owns release of the injected probe. A fixed timeout races the capture plan's - // independent fallback tiers on loaded CI hosts and can drain before the test records the - // abandoned-work state. The defer above still releases the probe if an earlier assertion or - // expectation fails. - systemModalProbeOverrideForTesting = { _ in - probeReleaseGate.wait() - return nil - } - - let completion = expectation( - description: "\(entryPointName) recovered while the probe was abandoned, then released it" - ) - let drained = expectation(description: "\(entryPointName) modal probe drained") - DispatchQueue(label: "agent-device.runner.tests.modal-probe-timeout").async { - box.payload = try? callEntryPoint( - snapshotTarget, - PresentationOptions(interactiveOnly: false, depth: nil, scope: nil, raw: false) - ) - - // 1) Penalty/busy accounting: must already be in place by the time the entry point - // returns, well before we release the still-blocked probe below. - if case .busy = self.currentMainThreadBusyState() { - box.wasBusyBeforeDrain = true - } - box.hadAbandonedCaptureBeforeDrain = self.hasAbandonedMainThreadWork() - box.wasPenalizedBeforeDrain = self.isSnapshotXCTestChannelPenalized(bundleId: self.currentBundleId) - - // 2) `box.payload` above was already produced -- through the capture plan's recovery - // tiers -- while the probe is still blocked on `probeReleaseGate`, i.e. recovered before - // drain, not queued behind it. - completion.fulfill() - - // 3) Only now let the abandoned probe finish, then block this queue (never the test's - // main-thread wait) on the *real* drain signal -- the abandoned-work count reaching zero - // -- bounded so a revert that never drains fulfills `drained` anyway and lets the - // assertions below report the regression explicitly instead of just timing out. - probeReleaseGate.signal() - let drainDeadline = Date().addingTimeInterval(5) - while self.hasAbandonedMainThreadWork(), Date() < drainDeadline { - self.sleepFor(0.002) - } - drained.fulfill() - } - - wait(for: [completion], timeout: 15) - - // 1) Penalty/busy accounting. - XCTAssertTrue( - box.wasBusyBeforeDrain, - "expected RUNNER_BUSY while the \(entryPointName) modal probe timeout is outstanding" - ) - XCTAssertTrue( - box.hadAbandonedCaptureBeforeDrain, - "onAbandoned must retain the abandoned XCTest channel work for \(entryPointName)" - ) - XCTAssertTrue( - box.wasPenalizedBeforeDrain, - "a timed-out modal probe must penalize the XCTest snapshot channel for \(entryPointName)" - ) - - // 2) Recovered response before drain. - XCTAssertNotNil( - box.payload, - "\(entryPointName) must recover a payload through the capture plan while the probe drains" - ) - - // 3) Bounded, deterministic drain barrier, then release assertions. - wait(for: [drained], timeout: 6) - guard case .idle = currentMainThreadBusyState() else { - return XCTFail("expected the runner to be idle once the abandoned \(entryPointName) probe drained") - } - XCTAssertFalse( - hasAbandonedMainThreadWork(), - "the drained probe must release the main thread for \(entryPointName)" - ) - } - - func testBoundedSystemModalProbeTimeoutRecoversThenReleasesOnDrain() { - assertBoundedSystemModalProbeTimeoutRecoversThenReleasesOnDrain(entryPointName: "snapshotFast") { - target, options in - try self.snapshotFast(app: target, options: options) - } - } - - func testBoundedSystemModalProbeTimeoutRecoversThenReleasesOnDrainForSnapshotRaw() { - assertBoundedSystemModalProbeTimeoutRecoversThenReleasesOnDrain(entryPointName: "snapshotRaw") { - target, options in - try self.snapshotRaw(app: target, options: options) - } - } -#endif - - func testDispatchRecoverySkipsBookkeepingWhileXCTestChannelOccupied() { - // The #1244 recovery shape: the modal probe abandoned an XCTest query that is still grinding on - // main, the capture recovered independently, and its response is ready. The recovery loop must - // return it without re-entering the main queue for recorded-failure/retry bookkeeping (that hop - // would block behind the abandoned query and re-stall the command), and a later command must - // still see the runner busy until the abandoned work drains. Removing the guard regresses this. - let command = try! JSONDecoder().decode( - Command.self, - from: Data(#"{"command":"snapshot","commandId":"recovery-guard"}"#.utf8) - ) - let recovered = Response(ok: false, error: .targetAppUnavailable(bundleId: nil)) - - setAbandonedMainThreadWork(1) - defer { setAbandonedMainThreadWork(0) } - guard case .busy = currentMainThreadBusyState() else { - return XCTFail("expected RUNNER_BUSY while abandoned XCTest work is outstanding") - } - - var occupiedCalls = 0 - let occupied = try! executeDispatchedWithRecovery(command: command) { - occupiedCalls += 1 - return recovered - } - XCTAssertEqual(occupiedCalls, 1, "recovered response must not retry behind abandoned XCTest work") - XCTAssertEqual(occupied.ok, false) - - setAbandonedMainThreadWork(0) - guard case .idle = currentMainThreadBusyState() else { - return XCTFail("runner should be idle once the abandoned work drained") - } - var drainedCalls = 0 - _ = try! executeDispatchedWithRecovery(command: command) { - drainedCalls += 1 - return recovered - } - XCTAssertEqual(drainedCalls, 2, "with the channel free the read-only retry runs once") - } - - private func setAbandonedMainThreadWork(_ count: Int) { - mainThreadWorkLock.lock() - abandonedMainThreadWorkCount = count - abandonedMainThreadWorkSince = count > 0 ? Date(timeIntervalSinceNow: -1) : nil - mainThreadWorkLock.unlock() - } -#endif - } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotBackendCapabilities.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotBackendCapabilities.swift index e4c43b54bd..4a5b478639 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotBackendCapabilities.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotBackendCapabilities.swift @@ -86,81 +86,3 @@ enum SnapshotBackendKind: String, CaseIterable { } } } - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS -private struct SnapshotBackendParityFixture: Decodable { - struct Availability: Decodable { - let simulator: Bool - let physicalDevice: Bool - } - - struct Backend: Decodable { - let name: String - let forceable: Bool - let supportsRawProjection: Bool - let regularDepth: String - let hittable: String - let availability: Availability - } - - let backends: [Backend] -} - -extension RunnerTests { - private func loadSnapshotBackendParityFixture() throws -> SnapshotBackendParityFixture { - let fixtureURL = URL(fileURLWithPath: #filePath) - .deletingLastPathComponent() // AgentDeviceRunnerUITests - .deletingLastPathComponent() // AgentDeviceRunner - .deletingLastPathComponent() // runner - .deletingLastPathComponent() // apple - .deletingLastPathComponent() // repo root - .appendingPathComponent("contracts") - .appendingPathComponent("fixtures") - .appendingPathComponent("ios-snapshot-backends.json") - return try JSONDecoder().decode( - SnapshotBackendParityFixture.self, - from: Data(contentsOf: fixtureURL) - ) - } - - /// The JSON table is the cross-runtime declaration used by the TypeScript capability registry - /// and this runner. A backend case, forceability branch, projection/depth claim, or availability - /// change that is not classified in both implementations fails before an iOS smoke can drift. - func testSnapshotBackendDeclarationsMatchCapabilityFixture() throws { - let fixture = try loadSnapshotBackendParityFixture() - XCTAssertEqual( - fixture.backends.map(\.name), - SnapshotBackendKind.allCases.map(\.rawValue) - ) - - for expected in fixture.backends { - guard let backend = SnapshotBackendKind(rawValue: expected.name) else { - XCTFail("fixture contains an unknown snapshot backend: \(expected.name)") - continue - } - XCTAssertEqual(backend.isForceable, expected.forceable, expected.name) - XCTAssertEqual(backend.supportsRawProjection, expected.supportsRawProjection, expected.name) - XCTAssertEqual( - backend.regularDepthCapability.rawValue, - expected.regularDepth, - "regular depth capability: \(expected.name)" - ) - XCTAssertEqual( - backend.hittableSemantics, - expected.hittable, - "hittable semantics: \(expected.name)" - ) - XCTAssertEqual( - backend.isAvailable(on: .simulator), - expected.availability.simulator, - "simulator availability: \(expected.name)" - ) - XCTAssertEqual( - backend.isAvailable(on: .physicalDevice), - expected.availability.physicalDevice, - "physical-device availability: \(expected.name)" - ) - } - } -} -#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift index 16a999ab5d..b6e9946957 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift @@ -629,7 +629,7 @@ extension RunnerTests { // MARK: Outcome stamping - private func stampedSnapshotPayload( + func stampedSnapshotPayload( _ capture: SnapshotBackendCapture, backend: SnapshotBackendKind, state: String, @@ -666,493 +666,3 @@ extension RunnerTests { ) } } - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS -// MARK: - In-bundle unit tests - -extension RunnerTests { - private func planTestNode( - index: Int, - type: String, - label: String? = nil, - identifier: String? = nil, - hittable: Bool = false, - parentIndex: Int? = nil - ) -> PresentedNode { - SnapshotPresentation.singleElementRead( - RawAXNode( - index: index, - type: type, - label: label, - identifier: identifier, - value: nil, - rect: SnapshotRect(.zero), - enabled: true, - focused: nil, - selected: nil, - hittable: hittable, - depth: parentIndex == nil ? 0 : 1, - parentIndex: parentIndex, - hiddenContentAbove: nil, - hiddenContentBelow: nil - ) - ) - } - - func testSparsePayloadReasonMatrix() { - let root = planTestNode(index: 0, type: "Application", label: "Example App", hittable: true) - let window = planTestNode(index: 1, type: "Window", parentIndex: 0) - let button = planTestNode(index: 1, type: "Button", label: "Ok", hittable: true, parentIndex: 0) - let shell = planTestNode( - index: 1, - type: "Other", - identifier: "appShell", - parentIndex: 0 - ) - let serializationPlaceholder = planTestNode( - index: 2, - type: "Other", - label: "[object Object]", - parentIndex: 1 - ) - - // Labeled, hittable root over a bare window is still sparse. - XCTAssertNotNil(Self.sparsePayloadReason(DataPayload(nodes: [root, window], truncated: false))) - // Deadline-truncated near-empty sweep needs recovery even with one real control. - XCTAssertNotNil(Self.sparsePayloadReason(DataPayload(nodes: [root, button], truncated: true))) - // The same tiny tree from a completed sweep is a legitimately minimal screen. - XCTAssertNil(Self.sparsePayloadReason(DataPayload(nodes: [root, button], truncated: false))) - // Container metadata plus a stringified serialization placeholder is not readable UI. - XCTAssertNotNil( - Self.sparsePayloadReason( - DataPayload(nodes: [root, shell, serializationPlaceholder], truncated: false) - ) - ) - let actionableShell = planTestNode( - index: 1, - type: "Other", - identifier: "checkout", - hittable: true, - parentIndex: 0 - ) - XCTAssertNil( - Self.sparsePayloadReason(DataPayload(nodes: [root, actionableShell], truncated: false)) - ) - // Empty payloads are degraded. - XCTAssertNotNil(Self.sparsePayloadReason(DataPayload(nodes: [], truncated: false))) - } - - func testCollapsedLeafIndexesFlagsMergedContainersOnly() { - let root = planTestNode(index: 0, type: "Application", label: "App") - let merged = planTestNode( - index: 1, - type: "Other", - label: (0...30).map { "Row \($0), Tap" }.joined(separator: ", "), - parentIndex: 0 - ) - let prose = planTestNode( - index: 2, - type: "StaticText", - label: (0...30).map { "clause \($0)" }.joined(separator: ", "), - parentIndex: 0 - ) - XCTAssertEqual(Self.collapsedLeafIndexes([root, merged, prose]), [1]) - XCTAssertNil(Self.collapsedLeafIndexes([root, prose])) - } - - func testTerminalFailsClosedOnInteractiveAxFailureRegardlessOfSparseBest() { - // Interactive AX failure must invalidate + fail closed; a later tier's sparse synthetic-root - // "best" must never downgrade this to a returned-sparse payload (regression: best == nil guard). - XCTAssertEqual( - Self.resolveSnapshotPlanTerminal(terminal: .sparseWithFatalOnAXFailure, interactiveOnly: true), - .failClosed - ) - XCTAssertEqual( - Self.resolveSnapshotPlanTerminal(terminal: .sparseWithFatalOnAXFailure, interactiveOnly: false), - .sparseBest - ) - XCTAssertEqual( - Self.resolveSnapshotPlanTerminal(terminal: .throwOnAXFailure, interactiveOnly: true), - .throwAxFailure - ) - } - - func testXCTestChannelStateFirstFailureStampsDeferredCodeOnlyForDeferral() { - XCTAssertNil(Self.xcTestChannelStateFirstFailure(.normal)) - XCTAssertEqual(Self.xcTestChannelStateFirstFailure(.deferredToIndependentBackend)?.code, "deferred") - XCTAssertEqual(Self.xcTestChannelStateFirstFailure(.boundedXCTestProbe)?.code, "budget") - } - - func testSnapshotQualityCarriesPhaseTimingAtResponseLevel() { - let timing = SnapshotCaptureTiming(acquisitionMs: 12, presentationMs: 34) - let capture = SnapshotBackendCapture( - payload: DataPayload( - nodes: [planTestNode(index: 0, type: "Application", label: "App")], - truncated: false - ), - effectiveDepth: nil, - timing: timing - ) - - let payload = stampedSnapshotPayload( - capture, - backend: .recursiveTree, - state: "healthy", - reason: nil - ) - - XCTAssertEqual(payload.snapshotQuality?.timing, timing) - XCTAssertEqual(payload.nodes?.count, 1) - } - - func testStampedPayloadCarriesDisclosuresOnlyInTheVerdict() { - let root = planTestNode(index: 0, type: "Application", label: "App") - let merged = planTestNode( - index: 1, - type: "Other", - label: (0...30).map { "Tab \($0)" }.joined(separator: ", "), - parentIndex: 0 - ) - let coverage = SnapshotCustomActionCoverage( - read: 12, candidates: 19, truncated: 0, blocked: false) - let silent = stampedSnapshotPayload( - SnapshotBackendCapture( - payload: DataPayload(nodes: [root, merged], truncated: false), - effectiveDepth: nil, - customActions: coverage - ), - backend: .recursiveTree, - state: "healthy", - reason: nil - ) - XCTAssertNil(silent.message) - XCTAssertEqual(silent.snapshotQuality?.customActions, coverage) - XCTAssertEqual(silent.snapshotQuality?.collapsedLeafIndexes, [1]) - - let underlying = stampedSnapshotPayload( - SnapshotBackendCapture( - payload: DataPayload(message: "underlying", nodes: [root], truncated: false), - effectiveDepth: 4 - ), - backend: .privateAX, - state: "recovered", - reason: (reason: "tree capture timed out", code: "budget") - ) - XCTAssertEqual(underlying.message, "underlying") - } - - func testStampedPayloadTruncationTracksCompletenessNotRecoveryProvenance() { - let complete = SnapshotBackendCapture( - payload: DataPayload( - nodes: [ - planTestNode(index: 0, type: "Application", label: "App"), - planTestNode(index: 1, type: "Button", label: "Open", parentIndex: 0), - ], - truncated: false - ), - effectiveDepth: nil - ) - let deferred: (reason: String, code: String) = ( - "XCTest-backed snapshot tiers were deferred after recent slow accessibility work", "deferred" - ) - - // The CI signature behind `is absent ... capture was truncated`: a complete private AX - // tree selected while the XCTest channel is penalized is whole, and must say so. - let recovered = stampedSnapshotPayload( - complete, backend: .privateAX, state: "recovered", reason: deferred) - XCTAssertEqual(recovered.snapshotQuality?.state, "recovered") - XCTAssertEqual(recovered.truncated, false) - - let depthLimited = stampedSnapshotPayload( - SnapshotBackendCapture(payload: complete.payload, effectiveDepth: 56), - backend: .privateAX, state: "recovered", reason: deferred) - XCTAssertEqual(depthLimited.truncated, true) - - let cappedPayload = stampedSnapshotPayload( - SnapshotBackendCapture( - payload: DataPayload(nodes: complete.payload.nodes ?? [], truncated: true), - effectiveDepth: nil), - backend: .recursiveTree, state: "healthy", reason: nil) - XCTAssertEqual(cappedPayload.truncated, true) - - let sparse = stampedSnapshotPayload( - complete, backend: .querySweep, state: "sparse", - reason: ("snapshot returned no semantic controls or content", "sparse-tree")) - XCTAssertEqual(sparse.truncated, true) - } - - func testSnapshotQualityCarriesUnscopedQualityPayload() { - let quality = DataPayload( - nodes: [planTestNode(index: 0, type: "Application", label: "App")], - truncated: false - ) - let capture = SnapshotBackendCapture( - payload: quality, - effectiveDepth: nil, - qualityPayload: quality - ) - - let payload = stampedSnapshotPayload( - capture, - backend: .recursiveTree, - state: "healthy", - reason: nil - ) - - XCTAssertEqual(payload.qualityPayload?.nodes.count, 1) - XCTAssertEqual(payload.qualityPayload?.truncated, false) - XCTAssertNil(payload.qualityPayload?.scope) - } - - func testDirectPresentationDoesNotClaimPlanTiming() { - let options = PresentationOptions( - interactiveOnly: false, - depth: nil, - scope: nil, - raw: true - ) - let result = SnapshotPresentation.presentRaw( - SnapshotAcquisition( - hint: SnapshotPresentation.captureHint(for: options), - nodes: [], - truncated: false, - effectiveDepth: nil, - viewport: .infinite - ), - options: options - ) - let capture = Self.makeSnapshotBackendCapture(from: result) - - let payload = stampedSnapshotPayload( - capture, - backend: .recursiveTree, - state: "healthy", - reason: nil - ) - - XCTAssertNil(payload.snapshotQuality?.timing) - } - - /// The raw plan is derived from what each backend can actually serve, not from a second - /// hand-maintained list. Non-vacuity: flipping `querySweep.supportsRawProjection` to true adds it - /// to the plan and fails the first two assertions — which is exactly the shape of #1797 D4, a - /// `--raw` request answered by a backend that has no hierarchy to return. - func testRawDiagnosticPlanCarriesOnlyBackendsThatCanServeRaw() { - XCTAssertEqual(Self.rawDiagnosticPlan, [.recursiveTree, .privateAX]) - XCTAssertEqual( - SnapshotBackendKind.allCases.filter { !$0.supportsRawProjection }, [.querySweep]) - XCTAssertTrue(Self.rawDiagnosticPlan.allSatisfy(\.supportsRawProjection)) - // Tree-first error propagation is the raw plan's other contract (ADR 0004). - XCTAssertEqual(Self.rawDiagnosticPlan.first, .recursiveTree) - } - - /// A projection mismatch is a runner bug, not an accessibility failure: it must not take the - /// AX-failure terminal route (rethrow / fail-closed), just drop its tier with a named reason. - func testProjectionMismatchFailureIsStructuredAndNotAnAxFailure() { - let failure = Self.snapshotProjectionMismatchFailure( - .querySweep, requested: .raw, acquired: .regular) - XCTAssertEqual(failure.code, "IOS_SNAPSHOT_PROJECTION_MISMATCH") - XCTAssertTrue(failure.message.contains("queries")) - XCTAssertTrue(failure.message.contains("raw")) - XCTAssertFalse(Self.isAxSnapshotFailure(failure)) - } - - /// #1634 P2: the decoded wire field must reach capture options and its - /// applicable plan. A pinned REGULAR capture defers to privateAX-first; the - /// RAW diagnostic plan is never rerouted by the pin — raw keeps tree-first - /// error propagation, which is exactly why raw baselines are excluded from - /// corroboration daemon-side. - func testDecodedPreferredBackendReachesOptionsAndApplicablePlan() throws { - let json = #"{"command":"snapshot","preferredBackend":"private-ax"}"# - let command = try JSONDecoder().decode(Command.self, from: Data(json.utf8)) - let options = Self.presentationOptions(from: command) - XCTAssertEqual(options.preferredBackend, "private-ax") - XCTAssertFalse(options.raw) - - let treated = Self.snapshotXCTestChannelTreatedAsPenalized( - penalized: false, preferredBackend: options.preferredBackend) - let pinned = Self.effectiveSnapshotCapturePlan( - Self.regularVisiblePlan, - xCTestChannelPenalized: treated, - preferredBackend: options.preferredBackend - ) - XCTAssertEqual(pinned.plan, [.privateAX]) - XCTAssertEqual(pinned.xCTestChannelState, .deferredToIndependentBackend) - - let raw = Self.effectiveSnapshotCapturePlan( - Self.rawDiagnosticPlan, - xCTestChannelPenalized: treated, - preferredBackend: options.preferredBackend - ) - XCTAssertEqual(raw.plan, Self.rawDiagnosticPlan) - - // A command without the field decodes to no pin and a normal plan. - let bare = try JSONDecoder().decode( - Command.self, from: Data(#"{"command":"snapshot"}"#.utf8)) - XCTAssertNil(Self.presentationOptions(from: bare).preferredBackend) - } - - /// #1635: the force seam must select the recursive tree even when the XCTest - /// channel is currently penalized. Without the preferred-backend argument, - /// this call returns the independent private-AX recovery plan instead. - func testPreferredTreeBackendPinsRegularPlanAndLeavesStructuredEvidence() { - let forced = Self.effectiveSnapshotCapturePlan( - Self.regularVisiblePlan, - xCTestChannelPenalized: true, - preferredBackend: SnapshotBackendKind.recursiveTree.rawValue - ) - XCTAssertEqual(forced.plan, [.recursiveTree]) - XCTAssertEqual(forced.xCTestChannelState, .normal) - XCTAssertEqual( - Self.xcTestChannelStateFirstFailure( - forced.xCTestChannelState, - preferredBackend: forced.preferredBackend?.rawValue - )?.code, - "requested-backend" - ) - } - - /// Same-backend evidence probes: a daemon-pinned private-AX capture takes the - /// penalized route even with a healthy channel, so tap-outcome corroboration - /// baselines and probes are always captured by the same backend (backends are - /// never comparable views of a screen). Composed with the plan rule, the pin - /// yields the privateAX-first deferred plan. - func testPreferredPrivateAXBackendPlansAsPenalized() { - XCTAssertTrue( - Self.snapshotXCTestChannelTreatedAsPenalized(penalized: false, preferredBackend: "private-ax")) - XCTAssertTrue( - Self.snapshotXCTestChannelTreatedAsPenalized(penalized: true, preferredBackend: nil)) - XCTAssertFalse( - Self.snapshotXCTestChannelTreatedAsPenalized(penalized: false, preferredBackend: nil)) - XCTAssertFalse( - Self.snapshotXCTestChannelTreatedAsPenalized(penalized: false, preferredBackend: "tree")) - - let pinned = Self.effectiveSnapshotCapturePlan( - Self.regularVisiblePlan, - xCTestChannelPenalized: Self.snapshotXCTestChannelTreatedAsPenalized( - penalized: false, preferredBackend: "private-ax" - ), - preferredBackend: "private-ax" - ) - XCTAssertEqual(pinned.plan, [.privateAX]) - XCTAssertEqual(pinned.xCTestChannelState, .deferredToIndependentBackend) - } - - func testEffectiveSnapshotCapturePlanDefersXCTestBackedTiersOnlyWhenPenalizedRegularPlan() { - let regular = Self.effectiveSnapshotCapturePlan( - Self.regularVisiblePlan, - xCTestChannelPenalized: true - ) - XCTAssertEqual(regular.plan, [.privateAX]) - XCTAssertEqual(regular.xCTestChannelState, .deferredToIndependentBackend) - XCTAssertNil(regular.treeCaptureSliceBudgetOverride) - - let unpenalized = Self.effectiveSnapshotCapturePlan( - Self.regularVisiblePlan, - xCTestChannelPenalized: false - ) - XCTAssertEqual(unpenalized.plan, Self.regularVisiblePlan) - XCTAssertEqual(unpenalized.xCTestChannelState, .normal) - XCTAssertNil(unpenalized.treeCaptureSliceBudgetOverride) - - // The raw diagnostic plan preserves tree-first error propagation even under penalty. - let raw = Self.effectiveSnapshotCapturePlan( - Self.rawDiagnosticPlan, - xCTestChannelPenalized: true - ) - XCTAssertEqual(raw.plan, Self.rawDiagnosticPlan) - XCTAssertEqual(raw.xCTestChannelState, .normal) - XCTAssertNil(raw.treeCaptureSliceBudgetOverride) - } - - func testEffectiveSnapshotCapturePlanUsesBoundedXCTestProbeWhenNoIndependentBackendRuns() { - let physicalDevicePlan = Self.effectiveSnapshotCapturePlan( - Self.regularVisiblePlan, - xCTestChannelPenalized: true, - availableBackends: [.recursiveTree, .querySweep] - ) - - XCTAssertEqual(physicalDevicePlan.plan, [.recursiveTree, .querySweep]) - XCTAssertEqual(physicalDevicePlan.xCTestChannelState, .boundedXCTestProbe) - XCTAssertEqual( - physicalDevicePlan.treeCaptureSliceBudgetOverride, - Self.penalizedXCTestProbeTreeSliceBudget - ) - } - - func testSnapshotXCTestChannelPenaltyMatchesBundleAndExpires() { - defer { - snapshotXCTestChannelPenaltyBundleId = nil - snapshotXCTestChannelPenaltyUntil = .distantPast - } - - penalizeSnapshotXCTestChannel(bundleId: "xyz.blueskyweb.app", reason: "test") - XCTAssertTrue(isSnapshotXCTestChannelPenalized(bundleId: "xyz.blueskyweb.app")) - XCTAssertFalse(isSnapshotXCTestChannelPenalized(bundleId: "com.other.app")) - - // A penalty recorded without a bundle applies to any current target. - penalizeSnapshotXCTestChannel(bundleId: nil, reason: "test") - XCTAssertTrue(isSnapshotXCTestChannelPenalized(bundleId: "com.other.app")) - - // Expired penalties stop applying. - snapshotXCTestChannelPenaltyUntil = Date(timeIntervalSinceNow: -1) - XCTAssertFalse(isSnapshotXCTestChannelPenalized(bundleId: "com.other.app")) - } - - func testAbandonedMainThreadWorkSkipsOnlyXCTestBackedSnapshotTiers() { - abandonedMainThreadWorkCount = 1 - defer { abandonedMainThreadWorkCount = 0 } - - XCTAssertTrue(shouldSkipSnapshotBackendForAbandonedMainThreadWork(.recursiveTree)) - XCTAssertTrue(shouldSkipSnapshotBackendForAbandonedMainThreadWork(.querySweep)) - XCTAssertFalse(shouldSkipSnapshotBackendForAbandonedMainThreadWork(.privateAX)) - } - -#if os(iOS) - /// #2403: a plan pinned to private AX serves a regular `--depth` request through acquisition - /// and presentation. With a backend depth gate in `captureWithBackend`, private AX returns no - /// capture, the plan falls through to the synthetic sparse root, and the daemon rejects that - /// zero-rect root as a missing viewport. - func testPrivateAXPinnedRegularDepthReachesAcquisitionAndPresentation() throws { - app.launchArguments = ["--agent-device-selector-read-regression"] - app.launch() - currentApp = app - currentBundleId = nil - defer { - currentApp = nil - clearPrivateAXAcceptedDepth(reason: "test-cleanup") - app.terminate() - } - func capture(depth: Int?) throws -> DataPayload { - try runSnapshotCapturePlan( - Self.regularVisiblePlan, - app: app, - options: PresentationOptions( - interactiveOnly: false, - depth: depth, - scope: nil, - raw: false, - preferredBackend: SnapshotBackendKind.privateAX.rawValue - ), - terminal: .sparseWithFatalOnAXFailure - ) - } - - let capped = try capture(depth: 1) - - let quality = try XCTUnwrap(capped.snapshotQuality) - XCTAssertEqual(quality.backend, SnapshotBackendKind.privateAX.rawValue) - XCTAssertNotEqual(quality.state, "sparse") - let nodes = try XCTUnwrap(capped.nodes) - XCTAssertGreaterThan(nodes.count, 1) - XCTAssertEqual(nodes.map(\.depth).max(), 1) - XCTAssertNotEqual(nodes[0].rect, SnapshotRect(x: 0, y: 0, width: 0, height: 0)) - XCTAssertTrue(nodes.contains { $0.label == "Readable target" }) - - // The presented cut only ever narrows the unscoped capture from the same backend. - let unscoped = try XCTUnwrap(try capture(depth: nil).nodes) - XCTAssertLessThanOrEqual(nodes.count, unscoped.count) - } -#endif -} -#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotTiming.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotTiming.swift index 442aa6d1b9..288e4cca5a 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotTiming.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotTiming.swift @@ -73,7 +73,7 @@ extension RunnerTests { /// The penalty breaker observes only acquisition facts. Presentation is a separate phase and /// cannot arm the breaker, even when it is slower than the acquisition that produced the tree. - private static func snapshotXCTestPenaltyReason( + static func snapshotXCTestPenaltyReason( kind: SnapshotBackendKind, attempt: SnapshotBackendAttempt, slowThresholdMs: Double @@ -107,79 +107,3 @@ extension RunnerTests { ) } } - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS -extension RunnerTests { - func testXCTestPenaltyDecisionSeparatesAcquisitionAndPresentation() { - let slowPresentation = SnapshotBackendAttempt( - outcome: .noCapture, - timing: SnapshotCaptureTiming(acquisitionMs: 100, presentationMs: 4_000) - ) - XCTAssertNil( - Self.snapshotXCTestPenaltyReason( - kind: .recursiveTree, - attempt: slowPresentation, - slowThresholdMs: 3_000 - ) - ) - - let slowAcquisition = SnapshotBackendAttempt( - outcome: .noCapture, - timing: SnapshotCaptureTiming(acquisitionMs: 3_001, presentationMs: 100) - ) - XCTAssertEqual( - Self.snapshotXCTestPenaltyReason( - kind: .recursiveTree, - attempt: slowAcquisition, - slowThresholdMs: 3_000 - ), - "slow_tree_capture_3001ms" - ) - - let timeout = SnapshotCaptureFailure( - code: Self.xCTestSnapshotTimeoutCode, - message: "test timeout", - hint: "test" - ) - let acquisitionFailure = SnapshotBackendAttempt( - outcome: .failed(timeout, phase: .acquisition), - timing: SnapshotCaptureTiming(acquisitionMs: 100, presentationMs: 100) - ) - XCTAssertEqual( - Self.snapshotXCTestPenaltyReason( - kind: .recursiveTree, - attempt: acquisitionFailure, - slowThresholdMs: 3_000 - ), - "tree_backend_timeout" - ) - - let presentationFailure = SnapshotBackendAttempt( - outcome: .failed(timeout, phase: .presentation), - timing: SnapshotCaptureTiming(acquisitionMs: 100, presentationMs: 100) - ) - XCTAssertNil( - Self.snapshotXCTestPenaltyReason( - kind: .recursiveTree, - attempt: presentationFailure, - slowThresholdMs: 3_000 - ) - ) - } - - func testSnapshotPhaseTimerReportsAcquisitionAndPresentationSeparately() { - var now = Date(timeIntervalSinceReferenceDate: 100) - var timer = SnapshotPhaseTimer(now: { now }) - - _ = timer.measure(.acquisition) { - now = now.addingTimeInterval(2) - } - _ = timer.measure(.presentation) { - now = now.addingTimeInterval(5) - } - - XCTAssertEqual(timer.timing.acquisitionMs, 2_000, accuracy: 0.001) - XCTAssertEqual(timer.timing.presentationMs, 5_000, accuracy: 0.001) - } -} -#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift index a74bcd7908..74723a7f44 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift @@ -170,107 +170,3 @@ extension RunnerTests { + " fallbackAttempted=\(fallbackAttempted)" } } - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS) -extension RunnerTests { - func testSynthesizedGesturePolicyMarkerWritesOncePerKindUntilTheDecisionChanges() { - var written: [String] = [] - runnerMarkerWriter = { written.append($0) } - defer { - runnerMarkerWriter = { NSLog("%@", $0) } - invalidateCachedTarget(reason: "unit_test_cleanup") - } - logSynthesizedGesturePolicyDecision(kind: .coordinateTap, context: nil, fallbackAttempted: false) - logSynthesizedGesturePolicyDecision(kind: .coordinateTap, context: nil, fallbackAttempted: false) - XCTAssertEqual(written.count, 1, "a repeated decision writes no second line") - logSynthesizedGesturePolicyDecision(kind: .scroll, context: nil, fallbackAttempted: false) - XCTAssertEqual(written.count, 2, "each gesture kind states its own decision") - logSynthesizedGesturePolicyDecision(kind: .coordinateTap, context: nil, fallbackAttempted: true) - XCTAssertEqual(written.count, 3, "a changed decision writes a new line") - resetTargetBoundState() - logSynthesizedGesturePolicyDecision(kind: .coordinateTap, context: nil, fallbackAttempted: true) - XCTAssertEqual(written.count, 4, "a rebind states the same decision once more") - } -} -#endif - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS -extension RunnerTests { - func testSynthesizedFallbackPolicyRequiresPrivateSynthesisForScrollWhenAxUnavailableOrUnknown() { - XCTAssertFalse( - SynthesizedFallbackPolicy.privateSynthesisRequired - .allowsXCTestCoordinateFallback(accessibilityHealth: .unavailable) - ) - XCTAssertFalse( - SynthesizedFallbackPolicy.privateSynthesisRequired - .allowsXCTestCoordinateFallback(accessibilityHealth: .unknown) - ) - XCTAssertFalse( - SynthesizedFallbackPolicy.privateSynthesisRequired - .allowsXCTestCoordinateFallback(accessibilityHealth: .healthy) - ) - } - - func testSynthesizedDragCoordinateFallbackAllowsUnknownButNotUnavailableAccessibility() { - XCTAssertTrue( - SynthesizedFallbackPolicy.xctestCoordinateWhenAccessibilityAvailable - .allowsXCTestCoordinateFallback(accessibilityHealth: .healthy) - ) - XCTAssertFalse( - SynthesizedFallbackPolicy.xctestCoordinateWhenAccessibilityAvailable - .allowsXCTestCoordinateFallback(accessibilityHealth: .unavailable) - ) - XCTAssertTrue( - SynthesizedFallbackPolicy.xctestCoordinateWhenAccessibilityAvailable - .allowsXCTestCoordinateFallback(accessibilityHealth: .unknown) - ) - } - - /// Keyboard-policy semantics only. Which command gets which policy is the table below; a probe - /// that is merely permitted still costs a live AX fetch, so the two questions stay separate. - func testSynthesizedKeyboardPolicyAllowsProbeOnlyWhenAccessibilityPermitsIt() { - XCTAssertFalse( - SynthesizedKeyboardPolicy.whenAccessibilityHealthy - .allowsProbe(accessibilityHealth: .unknown) - ) - XCTAssertTrue( - SynthesizedKeyboardPolicy.requiredWhenAvailable - .allowsProbe(accessibilityHealth: .unknown) - ) - XCTAssertFalse( - SynthesizedKeyboardPolicy.requiredWhenAvailable - .allowsProbe(accessibilityHealth: .unavailable) - ) - } - - func testSynthesizedGesturePoliciesMatchCommandContracts() { - XCTAssertEqual( - synthesizedGesturePolicy(.coordinateTap), - SynthesizedGesturePolicy( - keyboardPolicy: .never, - fallbackPolicy: .xctestCoordinateAllowed - ) - ) - XCTAssertEqual( - synthesizedGesturePolicy(.scroll), - SynthesizedGesturePolicy( - keyboardPolicy: .requiredWhenAvailable, - fallbackPolicy: .privateSynthesisRequired - ) - ) - XCTAssertEqual( - synthesizedGesturePolicy(.synthesizedDrag), - SynthesizedGesturePolicy( - keyboardPolicy: .requiredWhenAvailable, - fallbackPolicy: .xctestCoordinateWhenAccessibilityAvailable - ) - ) - } - - func testCoordinateTapTextInputProbeSkipsPenalizedXCTestChannel() { - XCTAssertTrue(shouldProbeCoordinateTapTextInput(xCTestChannelPenalized: false)) - XCTAssertFalse(shouldProbeCoordinateTapTextInput(xCTestChannelPenalized: true)) - } - -} -#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputCandidatePolicy.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputCandidatePolicy.swift index a46e48e80f..e8dc68912b 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputCandidatePolicy.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputCandidatePolicy.swift @@ -14,34 +14,3 @@ func isCoordinateTextInputCandidate( && point.y >= frame.minY - tolerance && point.y <= frame.maxY + tolerance } - -extension RunnerTests { -#if AGENT_DEVICE_RUNNER_UNIT_TESTS - func testCoordinateTextInputCandidateMustBeEnabledAndContainTheTouchPoint() { - let frame = CGRect(x: 10, y: 20, width: 100, height: 40) - let point = CGPoint(x: 50, y: 40) - - XCTAssertTrue( - isCoordinateTextInputCandidate( - enabled: true, - frame: frame, - point: point - ) - ) - XCTAssertFalse( - isCoordinateTextInputCandidate( - enabled: false, - frame: frame, - point: point - ) - ) - XCTAssertFalse( - isCoordinateTextInputCandidate( - enabled: true, - frame: frame, - point: CGPoint(x: 200, y: 200) - ) - ) - } -#endif -} diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Transport.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Transport.swift index e694f41baa..9db9f0f7e7 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Transport.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Transport.swift @@ -200,7 +200,7 @@ extension RunnerTests { /// Returns true when this send duplicated a still-executing commandId and was attached as a /// waiter of the in-flight execution. Otherwise marks the commandId in flight and returns /// false so the caller enqueues the (single) execution. - private func attachToInFlightCommandIfNeeded( + func attachToInFlightCommandIfNeeded( command: Command, completion: @escaping ((data: Data, shouldFinish: Bool)) -> Void ) -> Bool { @@ -221,7 +221,7 @@ extension RunnerTests { return false } - private func deliverCommandResult( + func deliverCommandResult( command: Command, result: (data: Data, shouldFinish: Bool), completion: ((data: Data, shouldFinish: Bool)) -> Void @@ -239,45 +239,6 @@ extension RunnerTests { } } -#if AGENT_DEVICE_RUNNER_UNIT_TESTS - func testDuplicateCommandIdCoalescesOntoInFlightExecution() throws { - let command = try JSONDecoder().decode( - Command.self, - from: Data(#"{"command":"snapshot","commandId":"snapshot-coalesce"}"#.utf8) - ) - var primaryData: Data? - var waiterData: Data? - defer { - inFlightCommandIds.removeAll() - inFlightCommandWaiters.removeAll() - } - - XCTAssertFalse( - attachToInFlightCommandIfNeeded(command: command) { result in - primaryData = result.data - } - ) - XCTAssertTrue( - attachToInFlightCommandIfNeeded(command: command) { result in - waiterData = result.data - } - ) - - let delivered = Data("single-result".utf8) - deliverCommandResult( - command: command, - result: (delivered, false) - ) { result in - primaryData = result.data - } - - XCTAssertEqual(primaryData, delivered) - XCTAssertEqual(waiterData, delivered) - XCTAssertFalse(inFlightCommandIds.contains("snapshot-coalesce")) - XCTAssertNil(inFlightCommandWaiters["snapshot-coalesce"]) - } -#endif - // MARK: - Response Encoding private func jsonResponse(status: Int, response: Response) -> Data { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TvRemote.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TvRemote.swift index 252352baf0..b312933bb8 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TvRemote.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TvRemote.swift @@ -18,28 +18,6 @@ enum TvRemoteButton: String { } extension RunnerTests { -#if AGENT_DEVICE_RUNNER_UNIT_TESTS - func testTvRemoteButtonMappingAcceptsSupportedNamesAndRejectsUnknown() { - let supported = [ - ("select", "select"), - ("SELECT", "select"), - ("menu", "menu"), - ("home", "home"), - ("up", "up"), - ("down", "down"), - ("left", "left"), - ("right", "right"), - ] - for (raw, expected) in supported { - XCTAssertEqual(tvRemoteButton(from: raw)?.rawValue, expected) - } - - for raw in [String?(nil), "", "volumeUp", "select "] { - XCTAssertNil(tvRemoteButton(from: raw)) - } - } -#endif - func resolveTvRemoteDoublePressDelay() -> TimeInterval { guard let raw = ProcessInfo.processInfo.environment["AGENT_DEVICE_TV_REMOTE_DOUBLE_PRESS_DELAY_MS"], diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AXSnapshotFallbackTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AXSnapshotFallbackTests.swift new file mode 100644 index 0000000000..cbcad722a9 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AXSnapshotFallbackTests.swift @@ -0,0 +1,531 @@ +import XCTest +import AgentDeviceSnapshotPresentation + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +// MARK: - In-bundle unit tests + +extension RunnerTests { + func testPrivateAXAttemptDepthsAppliesRememberedDepth() { + XCTAssertEqual( + Self.privateAXAttemptDepths(requestedDepth: 64, rememberedDepth: nil), + [64, 56, 40, 24, 12] + ) + XCTAssertEqual( + Self.privateAXAttemptDepths(requestedDepth: 64, rememberedDepth: 56), + [56, 40, 24, 12] + ) + XCTAssertEqual(Self.privateAXAttemptDepths(requestedDepth: 64, rememberedDepth: 12), [12]) + // Remembered at/above the requested depth changes nothing. + XCTAssertEqual( + Self.privateAXAttemptDepths(requestedDepth: 64, rememberedDepth: 64), + [64, 56, 40, 24, 12] + ) + // A shallower explicit request keeps its own rungs; deeper stale memory is ignored. + XCTAssertEqual(Self.privateAXAttemptDepths(requestedDepth: 24, rememberedDepth: 56), [24, 12]) + } + + /// Executed producer contract for the #1627 review blocker: a frontier whose + /// live element vanished, and one whose re-rooted request fails, must BOTH + /// count as missed — an all-miss extension reporting itself drained would + /// present a capped capture as complete. Goes red if either miss-path + /// increment in extendSnapshotFrontiers is removed. + func testDeepExtensionCountsMissedFrontiers() { + // Element vanished (list churn between serialization and extension): the + // fabricated snapshot answers nil for accessibilityElement — missed, and + // no request call is consumed. (An explicit nil property: bare NSObject + // resolves the key through a UIKit category and would take the call path.) + let orphan = RunnerAXSnapshotFrontier() + orphan.snapshot = FrontierSnapshotWithoutElementForTesting() + orphan.node = NSMutableDictionary() + // Re-rooted request fails: the element resolves but the client cannot + // serve requestSnapshotForElement — one consumed call AND a miss. + let unreachable = RunnerAXSnapshotFrontier() + unreachable.snapshot = FrontierSnapshotWithElementForTesting() + unreachable.node = NSMutableDictionary() + + var nodeCount = 0 + var truncated = ObjCBool(false) + let outcome = RunnerAXSnapshotBridge.extend( + NSMutableArray(array: [orphan, unreachable]), + axClient: NSObject(), + attributes: [], + maxDepth: 56, + maxNodes: 5_000, + nodeCount: &nodeCount, + truncated: &truncated, + callsAllowed: 8, + mergedLeaves: nil, + deadline: nil + ) + + XCTAssertEqual(outcome?[RunnerAXSnapshotDeepExtensionMissedKey] as? Int, 2) + XCTAssertEqual(outcome?[RunnerAXSnapshotDeepExtensionCallsKey] as? Int, 1) + XCTAssertEqual(outcome?[RunnerAXSnapshotDeepExtensionPendingKey] as? Int, 0) + XCTAssertEqual(outcome?[RunnerAXSnapshotDeepExtensionNodesAddedKey] as? Int, 0) + XCTAssertFalse(truncated.boolValue) + // And the consumer verdict over exactly this outcome: still depth-limited. + XCTAssertTrue( + Self.privateAXDepthLimited( + effectiveDepth: 56, requestedDepth: 64, pendingFrontiers: 0, missedFrontiers: 2)) + } + + func testPrivateAXDepthLimitedRequiresEveryFrontierResolved() { + // Un-capped capture is never depth-limited, extension or not. + XCTAssertFalse( + Self.privateAXDepthLimited( + effectiveDepth: 64, requestedDepth: 64, pendingFrontiers: nil, missedFrontiers: nil)) + // Capped with no extension outcome (never ran) stays depth-limited. + XCTAssertTrue( + Self.privateAXDepthLimited( + effectiveDepth: 56, requestedDepth: 64, pendingFrontiers: nil, missedFrontiers: nil)) + // Fully drained extension clears the verdict. + XCTAssertFalse( + Self.privateAXDepthLimited( + effectiveDepth: 56, requestedDepth: 64, pendingFrontiers: 0, missedFrontiers: 0)) + // Budget exhaustion (pending frontiers) keeps it. + XCTAssertTrue( + Self.privateAXDepthLimited( + effectiveDepth: 56, requestedDepth: 64, pendingFrontiers: 2, missedFrontiers: 0)) + // The #1627 review blocker: an all-miss extension (elements vanished or + // re-rooted requests failed) resolved nothing — it must NOT present the + // capture as complete just because the queue emptied. + XCTAssertTrue( + Self.privateAXDepthLimited( + effectiveDepth: 56, requestedDepth: 64, pendingFrontiers: 0, missedFrontiers: 8)) + } + + func testPrivateAXAcceptedDepthMemoryMatchesBundleProcessAndExpires() { + defer { clearPrivateAXAcceptedDepth(reason: "test-cleanup") } + + rememberPrivateAXAcceptedDepth(bundleId: "xyz.blueskyweb.app", processIdentifier: 111, depth: 56) + XCTAssertEqual( + rememberedPrivateAXAcceptedDepth(bundleId: "xyz.blueskyweb.app", processIdentifier: 111), + 56 + ) + XCTAssertNil(rememberedPrivateAXAcceptedDepth(bundleId: "com.other.app", processIdentifier: 111)) + // A relaunch changes the PID; the new process must re-probe the full depth even inside the + // TTL, and an unknown current PID (post-invalidation) must never match. + XCTAssertNil(rememberedPrivateAXAcceptedDepth(bundleId: "xyz.blueskyweb.app", processIdentifier: 222)) + XCTAssertNil(rememberedPrivateAXAcceptedDepth(bundleId: "xyz.blueskyweb.app", processIdentifier: nil)) + + // Expired memory stops applying (the expiry re-probes the full requested depth). + privateAXAcceptedDepthUntil = Date(timeIntervalSinceNow: -1) + XCTAssertNil(rememberedPrivateAXAcceptedDepth(bundleId: "xyz.blueskyweb.app", processIdentifier: 111)) + } + + func testPrivateAXAcceptedDepthMemoryRequiresProcessIdentifierToRecord() { + defer { clearPrivateAXAcceptedDepth(reason: "test-cleanup") } + + rememberPrivateAXAcceptedDepth(bundleId: "xyz.blueskyweb.app", processIdentifier: nil, depth: 56) + XCTAssertNil( + rememberedPrivateAXAcceptedDepth(bundleId: "xyz.blueskyweb.app", processIdentifier: nil) + ) + } + + func testViewportReadSkippedWhileXCTestChannelPenalized() { + // Pins the viewport fast path (#1587 review): every penalized private AX capture used to burn + // the full 1s main-thread timeout on a doomed viewport read before falling back. + currentBundleId = "xyz.blueskyweb.app" + defer { + currentBundleId = nil + clearSnapshotXCTestChannelPenalty(reason: "test-cleanup") + abandonedMainThreadWorkCount = 0 + } + + XCTAssertTrue(shouldReadPrivateAXViewportViaXCTest()) + + penalizeSnapshotXCTestChannel(bundleId: "xyz.blueskyweb.app", reason: "test") + XCTAssertFalse(shouldReadPrivateAXViewportViaXCTest()) + + clearSnapshotXCTestChannelPenalty(reason: "test") + XCTAssertTrue(shouldReadPrivateAXViewportViaXCTest()) + + abandonedMainThreadWorkCount = 1 + XCTAssertFalse(shouldReadPrivateAXViewportViaXCTest()) + } + + /// The wire field must reach both capture options AND the backend pin: custom + /// actions are only readable through the private AX client, so a capture that + /// asked for them but planned the XCTest tree backend would return a payload + /// that structurally cannot carry them. + func testCustomActionsRequestPinsPrivateAXBackend() throws { + let asked = try JSONDecoder().decode( + Command.self, from: Data(#"{"command":"snapshot","customActions":true}"#.utf8)) + let options = Self.presentationOptions(from: asked) + XCTAssertTrue(options.customActions) + XCTAssertEqual(options.preferredBackend, SnapshotBackendKind.privateAX.rawValue) + XCTAssertTrue( + Self.snapshotXCTestChannelTreatedAsPenalized( + penalized: false, preferredBackend: options.preferredBackend)) + + // An explicit pin is never overwritten by the implied one. + let pinned = try JSONDecoder().decode( + Command.self, + from: Data(#"{"command":"snapshot","customActions":true,"preferredBackend":"tree"}"#.utf8)) + XCTAssertEqual(Self.presentationOptions(from: pinned).preferredBackend, "tree") + + // And the default capture neither asks nor pins. + let bare = try JSONDecoder().decode(Command.self, from: Data(#"{"command":"snapshot"}"#.utf8)) + XCTAssertFalse(Self.presentationOptions(from: bare).customActions) + XCTAssertNil(Self.presentationOptions(from: bare).preferredBackend) + } + + /// A request-pinned backend degraded nothing, so its verdict must not claim + /// slow accessibility work — that reason drives a user-facing warning. + func testRequestPinnedBackendReportsItsOwnReason() { + let requested = Self.xcTestChannelStateFirstFailure( + .deferredToIndependentBackend, requestPinnedBackend: true) + XCTAssertEqual(requested?.code, "requested-backend") + XCTAssertFalse(requested?.reason.contains("slow accessibility work") ?? true) + + // The circuit breaker's own deferral keeps its established code and wording. + let breaker = Self.xcTestChannelStateFirstFailure(.deferredToIndependentBackend) + XCTAssertEqual(breaker?.code, "deferred") + + // The bounded probe and the healthy plan are untouched by the new flag. + XCTAssertEqual( + Self.xcTestChannelStateFirstFailure(.boundedXCTestProbe, requestPinnedBackend: true)?.code, + "budget") + XCTAssertNil(Self.xcTestChannelStateFirstFailure(.normal, requestPinnedBackend: true)) + } + + /// The disclosure only exists if the counts survive the bridge boundary, and + /// "did not ask" must stay distinguishable from "read none". + func testCustomActionCoverageParsesOnlyCompletePairs() { + let complete: [String: Any] = [ + RunnerAXSnapshotCustomActionsReadKey: 12, + RunnerAXSnapshotCustomActionsCandidatesKey: 19, + RunnerAXSnapshotCustomActionsTruncatedKey: 2, + RunnerAXSnapshotCustomActionsBlockedKey: true, + ] + let coverage = Self.privateAXCustomActionCoverage(complete) + XCTAssertEqual(coverage?.read, 12) + XCTAssertEqual(coverage?.candidates, 19) + XCTAssertEqual(coverage?.truncated, 2) + XCTAssertEqual(coverage?.blocked, true) + + // Absent key = the capture never asked; it must not read as (0, 0), which + // would warn "0 of 0" on every default capture. + XCTAssertNil(Self.privateAXCustomActionCoverage(nil)) + // The bridge in this target always writes all four keys, so a partial + // dictionary is malformed and is dropped whole. + for key in complete.keys { + var partial = complete + partial.removeValue(forKey: key) + XCTAssertNil(Self.privateAXCustomActionCoverage(partial), "missing \(key)") + } + } + + /// The AX call cannot be cancelled once issued, so the read deadline frees + /// only the caller — the call keeps running. Without containment, repeating + /// `snapshot --actions` against a wedged element would stack orphaned reads, + /// all sharing one XCAXClient. This pins the containment: one serial queue and + /// a single-flight refusal that adds no work while a read is outstanding. + func testHungCustomActionReadIsContainedAndRecovers() { + let hung = HungAXClientForTesting() + let element = NSObject() + let dispatchesBefore = RunnerAXSnapshotBridge.customActionReadDispatchCount() + let blockedBefore = RunnerAXSnapshotBridge.customActionReadBlockedCount() + defer { hung.release() } + + // 1. First read wedges. The caller is freed by the deadline, but the call is + // still out there, so it stays counted in flight. + var completed = ObjCBool(true) + let firstStarted = Date() + let first = RunnerAXSnapshotBridge.customActionNames( + forElement: element, axClient: hung, completed: &completed) + XCTAssertNil(first) + XCTAssertFalse(completed.boolValue) + XCTAssertGreaterThanOrEqual(-firstStarted.timeIntervalSinceNow, 0.9) + XCTAssertEqual(RunnerAXSnapshotBridge.customActionReadsInFlight(), 1) + XCTAssertEqual( + RunnerAXSnapshotBridge.customActionReadDispatchCount(), dispatchesBefore + 1) + + // 2. Repeats do NOT accumulate: no new dispatch, still exactly one in + // flight, and every repeat is refused by single-flight admission. + for _ in 0..<5 { + XCTAssertNil( + RunnerAXSnapshotBridge.customActionNames( + forElement: element, axClient: hung, completed: &completed)) + XCTAssertFalse(completed.boolValue) + } + XCTAssertEqual(RunnerAXSnapshotBridge.customActionReadsInFlight(), 1) + XCTAssertEqual( + RunnerAXSnapshotBridge.customActionReadDispatchCount(), dispatchesBefore + 1) + XCTAssertEqual(RunnerAXSnapshotBridge.customActionReadBlockedCount(), blockedBefore + 5) + + // 3. A capture in that state discloses the skip rather than presenting the + // unread elements as action-free — and spends no read budget doing it. + let leaf = RunnerAXSnapshotFrontier() + leaf.snapshot = FrontierSnapshotWithElementForTesting() + leaf.node = NSMutableDictionary(dictionary: ["label": "feedItem", "children": []]) + let coverage = RunnerAXSnapshotBridge.annotateCustomActions( + onMergedLeaves: [leaf], axClient: hung, limit: 12, rootFrame: .zero, deadline: nil) + XCTAssertEqual(coverage[RunnerAXSnapshotCustomActionsBlockedKey] as? Bool, true) + XCTAssertEqual(coverage[RunnerAXSnapshotCustomActionsReadKey] as? Int, 0) + XCTAssertEqual(coverage[RunnerAXSnapshotCustomActionsCandidatesKey] as? Int, 1) + XCTAssertEqual( + RunnerAXSnapshotBridge.customActionReadDispatchCount(), dispatchesBefore + 1) + XCTAssertEqual( + Self.privateAXCustomActionCoverage(coverage), + SnapshotCustomActionCoverage(read: 0, candidates: 1, truncated: 0, blocked: true)) + + // 4. Recovery: once the wedged call returns, reads resume by themselves. + hung.release() + let recovered = expectation(description: "in-flight drains") + DispatchQueue.global().async { + while RunnerAXSnapshotBridge.customActionReadsInFlight() > 0 { + usleep(20_000) + } + recovered.fulfill() + } + wait(for: [recovered], timeout: 5) + + completed = ObjCBool(false) + XCTAssertNil( + RunnerAXSnapshotBridge.customActionNames( + forElement: element, axClient: hung, completed: &completed)) + // Completed (the fake answers nil actions), which is the point: the pass is + // live again rather than latched off. + XCTAssertTrue(completed.boolValue) + XCTAssertEqual( + RunnerAXSnapshotBridge.customActionReadDispatchCount(), dispatchesBefore + 2) + } + + /// The element budget bounds how many elements we read; these caps bound what + /// any ONE element can put in the response. Clipping must be reported, since + /// a clipped list looks exactly like a complete one. + func testActionNamesAreCappedPerElementAndReported() { + var truncated = ObjCBool(true) + + // Under both caps: untouched, nothing to report. + let small = ["Reply", "Repost"] + XCTAssertEqual( + RunnerAXSnapshotBridge.cappedActionNames(small, truncated: &truncated), small) + XCTAssertFalse(truncated.boolValue) + + // More actions than the per-element cap: clipped to the first 8, reported. + let many = (1...20).map { "Action \($0)" } + let cappedMany = RunnerAXSnapshotBridge.cappedActionNames(many, truncated: &truncated) + XCTAssertEqual(cappedMany.count, 8) + XCTAssertEqual(cappedMany.first, "Action 1") + XCTAssertTrue(truncated.boolValue) + + // A single very long name is shortened, reported, and stays one string. + let long = String(repeating: "a", count: 500) + let cappedLong = RunnerAXSnapshotBridge.cappedActionNames([long], truncated: &truncated) + XCTAssertEqual(cappedLong.count, 1) + XCTAssertTrue(truncated.boolValue) + XCTAssertLessThan(cappedLong[0].count, long.count) + XCTAssertTrue(cappedLong[0].hasSuffix("…")) + + // Empty input is not "truncated". + XCTAssertEqual(RunnerAXSnapshotBridge.cappedActionNames([], truncated: &truncated), []) + XCTAssertFalse(truncated.boolValue) + } + + /// Action names annotated by the bridge must survive into the emitted node — + /// the whole point of the capture is that the merged card names its hidden + /// affordances. + func testPrivateAXNodesCarryAnnotatedCustomActions() { + let tree: [String: Any] = [ + "type": Int(XCUIElement.ElementType.application.rawValue), + "label": "Blue Sky", + "frame": ["x": 0, "y": 0, "width": 390, "height": 844], + "children": [ + [ + "type": Int(XCUIElement.ElementType.link.rawValue), + "label": "feedItem-by-whiskers.test", + "frame": ["x": 0, "y": 100, "width": 390, "height": 200], + "actions": ["Reply", "Repost", "Open post options menu"], + "children": [], + ], + [ + "type": Int(XCUIElement.ElementType.button.rawValue), + "label": "Compose", + "frame": ["x": 300, "y": 700, "width": 60, "height": 60], + "children": [], + ], + ], + ] + let nodes = privateAXAcquisition( + rawRoot: tree, + hint: CaptureHint( + projection: .regular, depth: nil, regularPresentedDepth: nil, + interactiveOnly: false, customActions: false) + ) + + let card = nodes.first { $0.label == "feedItem-by-whiskers.test" } + XCTAssertEqual(card?.actions, ["Reply", "Repost", "Open post options menu"]) + // A node the bridge did not annotate stays absent, not empty. + XCTAssertNil(nodes.first { $0.label == "Compose" }?.actions) + } + + func testPrivateAXAcquisitionDoesNotInterpretScope() { + let tree: [String: Any] = [ + "type": 1, "label": "App", + "children": [ + [ + "type": 9, "identifier": "homeScreen", + "children": [ + ["type": 48, "label": "Post body without the scope text", "children": []] + ], + ], + ["type": 9, "label": "unrelated sibling", "children": []], + ], + ] + // Scope never reaches acquisition: the hint derived for a scoped request carries no scope, + // and the backend has no way to interpret one. + let nodes = privateAXAcquisition( + rawRoot: tree, + hint: SnapshotPresentation.captureHint( + for: PresentationOptions( + interactiveOnly: false, + depth: nil, + scope: "homeScreen", + raw: false + ) + ) + ) + + let labels = nodes.compactMap { $0.label ?? $0.identifier } + XCTAssertTrue(labels.contains("homeScreen")) + // Descendants of the matched scope are included even when they do not contain the text. + XCTAssertTrue(labels.contains("Post body without the scope text")) + XCTAssertTrue(labels.contains("unrelated sibling")) + } + + func testPrivateAXInteractiveFiltersLoginLikeHiddenDrawer() throws { + let tree: [String: Any] = [ + "type": Int(XCUIElement.ElementType.application.rawValue), + "label": "Blue Sky", + "frame": ["x": 0, "y": 0, "width": 390, "height": 844], + "children": [ + [ + "type": Int(XCUIElement.ElementType.scrollView.rawValue), + "frame": ["x": 0, "y": 0, "width": 390, "height": 844], + "children": [ + [ + "type": Int(XCUIElement.ElementType.image.rawValue), + "label": "Callstack", + "frame": ["x": 145, "y": 104, "width": 100, "height": 100], + "children": [], + ], + [ + "type": Int(XCUIElement.ElementType.staticText.rawValue), + "label": "Welcome back", + "frame": ["x": 32, "y": 260, "width": 326, "height": 32], + "children": [], + ], + [ + "type": Int(XCUIElement.ElementType.textField.rawValue), + "label": "Email", + "identifier": "login.email", + "frame": ["x": 32, "y": 348, "width": 326, "height": 48], + "children": [], + ], + [ + "type": Int(XCUIElement.ElementType.secureTextField.rawValue), + "label": "Password", + "identifier": "login.password", + "frame": ["x": 32, "y": 412, "width": 326, "height": 48], + "children": [], + ], + [ + "type": Int(XCUIElement.ElementType.button.rawValue), + "label": "Sign in", + "identifier": "login.submit", + "frame": ["x": 32, "y": 492, "width": 326, "height": 52], + "children": [], + ], + [ + "type": Int(XCUIElement.ElementType.link.rawValue), + "label": "Forgot password?", + "frame": ["x": 128, "y": 568, "width": 134, "height": 32], + "children": [], + ], + [ + "type": Int(XCUIElement.ElementType.button.rawValue), + "label": "Admin settings", + "frame": ["x": -260, "y": 184, "width": 220, "height": 44], + "children": [], + ], + [ + "type": Int(XCUIElement.ElementType.other.rawValue), + "frame": ["x": 16, "y": 184, "width": 220, "height": 44], + "children": [], + ], + ], + ] + ], + ] + let viewport = CGRect(x: 0, y: 0, width: 390, height: 844) + let hint = CaptureHint( + projection: .regular, depth: nil, regularPresentedDepth: nil, + interactiveOnly: true, customActions: false) + let acquired = SnapshotGeometrySpace.normalized( + nodes: privateAXAcquisition(rawRoot: tree, hint: hint), + viewport: viewport, + interfaceOrientation: RunnerInterfaceOrientation.portrait + ) + // Acquisition serializes the drawer too; the shared fold is what hides it (#1797). + XCTAssertTrue(acquired.compactMap(\.label).contains("Admin settings")) + + let capture = try SnapshotPresentation.presentRegular( + SnapshotAcquisition( + hint: hint, nodes: acquired, truncated: false, effectiveDepth: nil, viewport: viewport), + options: PresentationOptions(interactiveOnly: true, depth: nil, scope: nil, raw: false), + policy: .cursorProjected + ) + let labels = capture.nodes.compactMap { $0.label } + XCTAssertEqual( + labels, + ["Blue Sky", "Callstack", "Welcome back", "Email", "Password", "Sign in", "Forgot password?"] + ) + XCTAssertFalse(labels.contains("Admin settings")) + } +} + +/// Stands in for an AX client whose `attributesForElement:` never returns — +/// the wedged-server case the containment exists for. `release()` lets the +/// hung call finish so recovery is observable. +private final class HungAXClientForTesting: NSObject { + private let gate = DispatchSemaphore(value: 0) + private let releasedOnce = NSLock() + private var released = false + + @objc(attributesForElement:attributes:error:) + func attributes(forElement element: Any, attributes: Any, error: NSErrorPointer) -> Any? { + releasedOnce.lock() + let alreadyReleased = released + releasedOnce.unlock() + // Once the wedge clears, the server answers normally again — that is what + // makes the recovery leg a recovery rather than a second hang. + if alreadyReleased { + return nil + } + gate.wait() + return nil + } + + func release() { + releasedOnce.lock() + defer { releasedOnce.unlock() } + guard !released else { return } + released = true + gate.signal() + } +} + +/// Minimal snapshot stand-in whose accessibilityElement resolves (so the +/// extension proceeds to the request) while the paired fake client cannot +/// serve it — the failed-re-root miss path. +private final class FrontierSnapshotWithElementForTesting: NSObject { + @objc let accessibilityElement = NSObject() +} + +/// The vanished-element case: KVC resolves the property and gets nil. +private final class FrontierSnapshotWithoutElementForTesting: NSObject { + @objc let accessibilityElement: NSObject? = nil +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertTests.swift new file mode 100644 index 0000000000..1beb0bec9d --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertTests.swift @@ -0,0 +1,9 @@ +import XCTest + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +extension RunnerTests { + func testAlertAcceptTreatsOpenAsAffirmative() { + XCTAssertTrue(isAcceptButton("Open")) + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+BlockingSystemModalResolutionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+BlockingSystemModalResolutionTests.swift new file mode 100644 index 0000000000..8a124bb669 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+BlockingSystemModalResolutionTests.swift @@ -0,0 +1,37 @@ +import XCTest + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +extension RunnerTests { + func testRemoteHostProbeRunsOnlyWhenSpringboardModalHasNoActions() { + XCTAssertTrue(RemoteHostedSystemModalPolicy.shouldProbeRemoteHost(springboardActionCount: 0)) + XCTAssertFalse(RemoteHostedSystemModalPolicy.shouldProbeRemoteHost(springboardActionCount: 1)) + XCTAssertFalse(RemoteHostedSystemModalPolicy.shouldProbeRemoteHost(springboardActionCount: 3)) + } + + func testRemoteHostStateGateFailsClosedToForeground() { + XCTAssertTrue(RemoteHostedSystemModalPolicy.isEligibleHostState(.runningForeground)) + XCTAssertFalse(RemoteHostedSystemModalPolicy.isEligibleHostState(.runningBackground)) + XCTAssertFalse(RemoteHostedSystemModalPolicy.isEligibleHostState(.notRunning)) + XCTAssertFalse(RemoteHostedSystemModalPolicy.isEligibleHostState(.unknown)) + } + + // No SpringBoard host (`hasSpringBoardSystemModalHost`) means modal resolution must return + // `.absent` without probing com.apple.springboard (#1351). Written for tvOS, where no lane + // ever executed it; `resolveBlockingSystemModal` takes that decision at RUNTIME off the same + // flag on macOS, so the host lane runs the real branch on every PR. + // + // Its former sibling `testBlockingSystemAlertSnapshotIsNilOnTvOS` is deleted rather than + // widened: `blockingSystemAlertSnapshot` is `#if os(macOS) return nil`, so on the only lane + // that could run it the assertion would pin a compile-time literal — a green that no change + // to the runner could turn red. The runtime gate it meant to cover is this test's subject, + // and the nil it returns on macOS is the compiler's business, not a test's. + #if os(tvOS) || os(macOS) + func testResolveBlockingSystemModalIsAbsentWithoutSpringBoardHost() { + guard case .absent = resolveBlockingSystemModal(deadline: .distantFuture) else { + XCTFail("blocking system-modal resolution must be .absent without a SpringBoard host") + return + } + } + #endif +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandExecutionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandExecutionTests.swift new file mode 100644 index 0000000000..b763b9e07e --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandExecutionTests.swift @@ -0,0 +1,704 @@ +import XCTest +import AgentDeviceSnapshotPresentation + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS) +import ObjectiveC.runtime + +private final class RunnerSynthesizedSwipeFailureStub: NSObject { + @objc(synthesizeSwipeWithApplication:resolvedWindow:x:y:x2:y2:durationMs:) + class func synthesizeSwipe( + application: XCUIApplication, + resolvedWindow: Any?, + x: Double, + y: Double, + x2: Double, + y2: Double, + durationMs: Double + ) -> String? { + "forced private synthesis failure" + } +} + +private final class RunnerSynthesizedTapFailureStub: NSObject { + @objc(synthesizeTapWithApplication:resolvedWindow:x:y:) + class func synthesizeTap(application: XCUIApplication, resolvedWindow: Any?, x: Double, y: Double) -> String? { + "forced private synthesis failure" + } +} +#endif + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +extension RunnerTests { + func testGestureResponseIncludesSynthesizedTapFallbackDiagnostics() { + let response = gestureResponse( + message: "tapped", + timing: (gestureStartUptimeMs: 1, gestureEndUptimeMs: 2), + fallback: GestureFallback( + strategy: "xctest-coordinate-tap", + message: "Runner synthesized coordinate tap is unavailable", + hint: "Using XCTest coordinate tap fallback." + ) + ) + + XCTAssertEqual(response.ok, true) + XCTAssertEqual(response.data?.gestureFallback, "xctest-coordinate-tap") + XCTAssertEqual( + response.data?.gestureFallbackMessage, + "Runner synthesized coordinate tap is unavailable" + ) + XCTAssertEqual(response.data?.gestureFallbackHint, "Using XCTest coordinate tap fallback.") + } + + func testGestureResponseIncludesMaestroNonHittableFallbackUsage() { + let response = gestureResponse( + message: "tapped via non-hittable coordinate fallback", + timing: (gestureStartUptimeMs: 1, gestureEndUptimeMs: 2), + frame: .touch(nil), + maestroNonHittableCoordinateFallbackUsed: true + ) + + XCTAssertEqual(response.data?.maestroNonHittableCoordinateFallbackUsed, true) + } + + func testCanonicalPlannedGestureResponseOmitsDragFrameAndPreservesDiagnostics() { + let response = gestureResponse( + message: "fling", + timing: (gestureStartUptimeMs: 1, gestureEndUptimeMs: 2), + frame: .drag( + DragVisualizationFrame( + x: 160, + y: 150, + x2: 40, + y2: 150, + referenceWidth: 200, + referenceHeight: 300 + ) + ), + fallback: GestureFallback( + strategy: "xctest-coordinate-drag", + message: "Private synthesis unavailable", + hint: "Using XCTest coordinate fallback." + ) + ) + + let canonical = canonicalPlannedGestureResponse(response) + + XCTAssertEqual(canonical.data?.gestureStartUptimeMs, 1) + XCTAssertEqual(canonical.data?.gestureEndUptimeMs, 2) + XCTAssertEqual(canonical.data?.gestureFallback, "xctest-coordinate-drag") + XCTAssertEqual(canonical.data?.gestureFallbackMessage, "Private synthesis unavailable") + XCTAssertEqual(canonical.data?.gestureFallbackHint, "Using XCTest coordinate fallback.") + XCTAssertNil(canonical.data?.x) + XCTAssertNil(canonical.data?.y) + XCTAssertNil(canonical.data?.x2) + XCTAssertNil(canonical.data?.y2) + XCTAssertNil(canonical.data?.referenceWidth) + XCTAssertNil(canonical.data?.referenceHeight) + } + +#if os(iOS) + func testSinglePointerFlingFallsBackToXCTestCoordinateDragWhenPrivateSynthesisFails() throws { + let selector = NSSelectorFromString( + "synthesizeSwipeWithApplication:resolvedWindow:x:y:x2:y2:durationMs:" + ) + guard + let synthesizedSwipeMethod = class_getClassMethod(RunnerSynthesizedGesture.self, selector), + let failureStubMethod = class_getClassMethod(RunnerSynthesizedSwipeFailureStub.self, selector) + else { + XCTFail("unable to install synthesized swipe failure stub") + return + } + let originalImplementation = method_getImplementation(synthesizedSwipeMethod) + method_setImplementation( + synthesizedSwipeMethod, + method_getImplementation(failureStubMethod) + ) + app.launch() + runnerAccessibilityHealth = .healthy + defer { + method_setImplementation(synthesizedSwipeMethod, originalImplementation) + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } + let command = try runnerCommandFixture( + """ + {"command":"gesture","commandId":"gesture-fling-fallback","gesturePlan":{"topology":"single","intent":"fling","executionProfile":"endpoint-hold","durationMs":100,"viewport":{"x":0,"y":0,"width":200,"height":300},"pointers":[{"pointerId":0,"samples":[{"offsetMs":0,"point":{"x":160,"y":150}},{"offsetMs":100,"point":{"x":40,"y":150}}]}]}} + """ + ) + + let response = try executeOnMainPrepared(command: command, activeApp: app) + + XCTAssertTrue(response.ok) + XCTAssertEqual(response.data?.message, "fling") + XCTAssertEqual(response.data?.gestureFallback, "xctest-coordinate-drag") + XCTAssertEqual(response.data?.gestureFallbackMessage, "forced private synthesis failure") + XCTAssertEqual( + response.data?.gestureFallbackHint, + "Private XCTest event synthesis is required for AX-free coordinate drag on iOS; update Xcode if this persists." + ) + XCTAssertNil(response.data?.x) + XCTAssertNil(response.data?.y) + XCTAssertNil(response.data?.x2) + XCTAssertNil(response.data?.y2) + } + + func testSelectorTapFallsBackToXCTestCoordinateWhenPrivateSynthesisFails() throws { + let selector = NSSelectorFromString("synthesizeTapWithApplication:resolvedWindow:x:y:") + guard + let synthesizedTapMethod = class_getClassMethod(RunnerSynthesizedGesture.self, selector), + let failureStubMethod = class_getClassMethod(RunnerSynthesizedTapFailureStub.self, selector) + else { + XCTFail("unable to install synthesized tap failure stub") + return + } + let originalImplementation = method_getImplementation(synthesizedTapMethod) + method_setImplementation( + synthesizedTapMethod, + method_getImplementation(failureStubMethod) + ) + app.launch() + currentApp = app + runnerAccessibilityHealth = .healthy + defer { + method_setImplementation(synthesizedTapMethod, originalImplementation) + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } + let command = try runnerCommandFixture( + #"{"command":"tap","commandId":"selector-tap-fallback","selectorKey":"label","selectorValue":"Agent Device Runner","synthesized":true}"# + ) + + let response = try executeOnMainPrepared(command: command, activeApp: app) + + XCTAssertTrue(response.ok) + XCTAssertEqual(response.data?.message, "tapped") + XCTAssertEqual(response.data?.gestureFallback, "xctest-coordinate-tap") + XCTAssertEqual(response.data?.gestureFallbackMessage, "forced private synthesis failure") + XCTAssertEqual( + response.data?.gestureFallbackHint, + "Falling back to XCTest coordinate tap may be slower and can still need a healthy accessibility tree." + ) + } +#endif + +#if os(iOS) + // `waitForTextEntryReadiness`'s hardware-keyboard fallback returns early only on confirmed + // focus (#1874), and `keyboardFocusConfirmed` reads that from the app-wide focus predicate this + // bundle otherwise refuses to trust. Two XCTest facts it rests on, neither a repository + // invariant: the predicate reports a responder that shows NO software keyboard at all, and it + // names the element well enough to tell the tapped field from another one. The fixture field is + // the exact shape the fallback exists for — a real responder with an empty `inputView` — so this + // is where both are observable. If either regressed, readiness would silently stop taking the + // fallback and spend the full readinessTimeout on every hardware-keyboard field, which no other + // assertion would notice. + func testHardwareKeyboardResponderConfirmsItsOwnKeyboardFocus() throws { + app.launchArguments = ["--agent-device-text-entry-regression"] + app.launch() + defer { + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } + XCTAssertTrue(app.waitForExistence(timeout: appExistenceTimeout)) + + let textField = app.textFields["agent-device-hardware-keyboard-input"] + XCTAssertTrue(textField.waitForExistence(timeout: appExistenceTimeout)) + let otherElement = app.staticTexts["Agent Device Runner"] + XCTAssertTrue(otherElement.waitForExistence(timeout: appExistenceTimeout)) + XCTAssertFalse( + keyboardFocusConfirmed(app: app, element: textField), + "an untapped field must not confirm focus, or the fallback would fire immediately" + ) + + let tapCommand = try runnerCommandFixture( + #"{"command":"tap","commandId":"tap-focus-confirmation","selectorKey":"id","selectorValue":"agent-device-hardware-keyboard-input"}"# + ) + let tapResponse = try executeOnMainPrepared(command: tapCommand, activeApp: app) + XCTAssertTrue(tapResponse.ok, String(describing: tapResponse.error)) + try XCTSkipIf( + isKeyboardVisible(app: app), + "software keyboard is up: this simulator cannot exercise the hidden-keyboard responder path" + ) + + let deadline = Date().addingTimeInterval(TextEntryTiming.readinessTimeout) + var confirmed = keyboardFocusConfirmed(app: app, element: textField) + while !confirmed && Date() < deadline { + sleepFor(TextEntryTiming.pollInterval) + confirmed = keyboardFocusConfirmed(app: app, element: textField) + } + XCTAssertTrue(confirmed, "a tapped responder must confirm its own keyboard focus") + XCTAssertFalse( + keyboardFocusConfirmed(app: app, element: otherElement), + "focus held by another element must read as a refusal, never as this element's focus" + ) + } +#endif + + func testInjectedTapRecordedFailureGateIsTapOnlyAndCountGated() { + // The seam's recording side cannot run in-bundle (a real XCTIssue would + // fail this very test run — same constraint the record(_:) suppression + // tests document); the live daemon proof covers it. This pins the gate. + XCTAssertFalse(RunnerTests.shouldInjectTapRecordedFailure(command: .tap, remaining: 0)) + XCTAssertTrue(RunnerTests.shouldInjectTapRecordedFailure(command: .tap, remaining: 1)) + XCTAssertFalse(RunnerTests.shouldInjectTapRecordedFailure(command: .type, remaining: 1)) + XCTAssertFalse(RunnerTests.shouldInjectTapRecordedFailure(command: .snapshot, remaining: 1)) + } + + func testXCTestRecordedFailureResponseFailsMutatingSuccesses() throws { + let command = try runnerCommandFixture(#"{"command":"tap","commandId":"tap-1"}"#) + let response = Response(ok: true, data: DataPayload(message: "tapped")) + + let failureResponse = xctestRecordedFailureResponse(command: command, response: response) + + XCTAssertEqual(failureResponse?.ok, false) + XCTAssertEqual(failureResponse?.error?.code, "XCTEST_RECORDED_FAILURE") + XCTAssertEqual( + failureResponse?.error?.message, + "XCTest recorded a failure while executing tap; the action may not have been performed." + ) + } + + func testXCTestRecordedFailureResponseFailsActionButtonSuccess() throws { + // The Action Button press carries no settle and no post-action observation, so this conversion is + // the only evidence the press landed. That is why the press is not classified runner-lifecycle: + // `isLifecycle` would silence the conversion here (#2699, #2702 review). + let command = try runnerCommandFixture(#"{"command":"actionButton","commandId":"action-button-1"}"#) + let response = Response(ok: true, data: DataPayload(message: "actionButton")) + + let failureResponse = xctestRecordedFailureResponse(command: command, response: response) + + XCTAssertEqual(failureResponse?.ok, false) + XCTAssertEqual(failureResponse?.error?.code, "XCTEST_RECORDED_FAILURE") + XCTAssertEqual( + failureResponse?.error?.message, + "XCTest recorded a failure while executing actionButton; the action may not have been performed." + ) + } + + func testXCTestRecordedFailureResponseDoesNotWrapReadOnlyOrRunnerFatalResponses() throws { + let snapshotCommand = try runnerCommandFixture(#"{"command":"snapshot","commandId":"snapshot-1"}"#) + let tapCommand = try runnerCommandFixture(#"{"command":"tap","commandId":"tap-1"}"#) + let runnerFatalResponse = Response( + ok: true, + data: DataPayload(runnerFatal: true, runnerFatalReason: "ax_snapshot_unavailable") + ) + + XCTAssertNil( + xctestRecordedFailureResponse( + command: snapshotCommand, + response: Response(ok: true, data: DataPayload(nodes: [], truncated: false)) + ) + ) + XCTAssertNil(xctestRecordedFailureResponse(command: tapCommand, response: runnerFatalResponse)) + } + + // Simulator-only from here to the matching #endif: these launch the host app, route through + // SpringBoard, or assert the iOS-only alert/system-modal branches. Tests outside the + // `os(iOS)` regions in this file are pure runner decisions and also run on the macOS host + // lane (ci.yml) — see the classification convention in RunnerTests.swift. +#if os(iOS) + func testMissingBundleCommandInvalidatesCompleteCachedTargetState() throws { + app.launch() + currentApp = app + currentBundleId = "com.example.stale-target" + currentAppProcessIdentifier = 42 + snapshotXCTestPenaltyWarmupExemptionPending = true + defer { + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } + let command = try runnerCommandFixture( + #"{"command":"snapshot","commandId":"snapshot-without-bundle"}"# + ) + + _ = prepareActiveCommandContext(command: command) + + XCTAssertNil(currentApp) + XCTAssertNil(currentBundleId) + XCTAssertNil(currentAppProcessIdentifier) + XCTAssertFalse(snapshotXCTestPenaltyWarmupExemptionPending) + } + + func testSkipAppActivationPreflightIncludesForegroundCachedCoordinateOnlyTaps() throws { + app.launch() + currentApp = app + currentBundleId = nil + defer { + currentApp = nil + currentBundleId = nil + app.terminate() + } + let tap = try runnerCommandFixture( + #"{"command":"tap","commandId":"tap-1","x":10,"y":20}"# + ) + + XCTAssertTrue(shouldSkipAppActivationPreflight(tap)) + } + + func testSkipAppActivationPreflightRejectsMissingChangedAndBackgroundTargets() throws { + let coordinateTap = try runnerCommandFixture( + #"{"command":"tap","commandId":"tap-1","x":10,"y":20}"# + ) + currentApp = nil + currentBundleId = nil + XCTAssertFalse(shouldSkipAppActivationPreflight(coordinateTap)) + + app.launch() + currentApp = app + currentBundleId = "com.example.current" + defer { + currentApp = nil + currentBundleId = nil + app.terminate() + } + let changedBundleTap = try runnerCommandFixture( + #"{"command":"tap","commandId":"tap-2","appBundleId":"com.example.other","x":10,"y":20}"# + ) + + XCTAssertFalse(shouldSkipAppActivationPreflight(changedBundleTap)) + + app.terminate() + currentApp = app + currentBundleId = nil + + XCTAssertFalse(shouldSkipAppActivationPreflight(coordinateTap)) + } + + func testActionButtonPressSkipsAppActivationPreflightWithoutBeingRunnerLifecycle() throws { + currentApp = nil + currentBundleId = nil + let press = try runnerCommandFixture(#"{"command":"actionButton","commandId":"action-button-1"}"#) + + // The skip is its own decision, reached without the lifecycle flag that would also drop the + // recorded-failure conversion; no cached target and no foreground app is required for it. + XCTAssertFalse(isRunnerLifecycleCommand(.actionButton)) + XCTAssertTrue(shouldSkipAppActivationPreflight(press)) + } + + func testPrepareActiveCommandContextRoutesBlockingSystemModalToSpringboard() throws { + blockingSystemModalPresenceOverrideForTesting = true + currentApp = nil + currentBundleId = nil + defer { + blockingSystemModalPresenceOverrideForTesting = nil + currentApp = nil + currentBundleId = nil + } + let tap = try runnerCommandFixture( + #"{"command":"tap","commandId":"tap-1","x":10,"y":20}"# + ) + + let preparation = prepareActiveCommandContext( + command: tap, + routeToSpringboard: shouldRouteToSpringboardBlockingSystemModal(tap) + ) + + guard case .context(let context) = preparation else { + XCTFail("expected command context") + return + } + XCTAssertTrue(context.app === springboard) + } + + func testExecuteDispatchedReturnsBusyBeforeBlockingSystemModalProbeDrains() throws { + app.launch() + currentApp = app + currentBundleId = nil + defer { + currentApp = nil + currentBundleId = nil + systemModalProbeOverrideForTesting = nil + clearSnapshotXCTestChannelPenalty(reason: "test-cleanup") + app.terminate() + } + + final class ResultBox { + var response: Response? + var error: Error? + var commandRecoveredBeforeRelease = false + var wasBusyBeforeRelease = false + var hadAbandonedProbeBeforeRelease = false + var drained = false + } + let box = ResultBox() + let probeStarted = expectation(description: "system-modal routing probe started") + let verificationFinished = expectation(description: "command recovery and modal probe drain verified") + let probeReleaseGate = DispatchSemaphore(value: 0) + let commandFinishedGate = DispatchSemaphore(value: 0) + systemModalProbeOverrideForTesting = { _ in + probeStarted.fulfill() + _ = probeReleaseGate.wait(timeout: .now() + 15) + return DataPayload(message: "late system modal") + } + + let command = try runnerCommandFixture( + #"{"command":"tap","commandId":"bounded-modal-routing","x":10,"y":20}"# + ) + DispatchQueue(label: "agent-device.runner.tests.modal-routing-probe").async { + do { + box.response = try self.executeDispatched(command: command) + } catch { + box.error = error + } + commandFinishedGate.signal() + } + DispatchQueue(label: "agent-device.runner.tests.modal-routing-probe-verifier").async { + let commandWait = commandFinishedGate.wait( + timeout: .now() + self.systemModalProbeBudget + 3 + ) + box.commandRecoveredBeforeRelease = commandWait == .success + && box.error == nil + && box.response?.error?.code == "RUNNER_BUSY" + if case .busy = self.currentMainThreadBusyState() { + box.wasBusyBeforeRelease = true + } + box.hadAbandonedProbeBeforeRelease = self.hasAbandonedMainThreadWork() + + // The XCTest main thread is blocked inside the injected probe, so this verifier owns the + // ordered release after recording the command result and abandoned-work state above. + probeReleaseGate.signal() + let deadline = Date().addingTimeInterval(5) + while self.hasAbandonedMainThreadWork(), Date() < deadline { + self.sleepFor(0.002) + } + box.drained = !self.hasAbandonedMainThreadWork() + verificationFinished.fulfill() + } + + wait(for: [probeStarted, verificationFinished], timeout: 15) + XCTAssertTrue( + box.commandRecoveredBeforeRelease, + "the public coordinate tap must return RUNNER_BUSY before the blocked modal probe drains" + ) + XCTAssertTrue(box.wasBusyBeforeRelease) + XCTAssertTrue(box.hadAbandonedProbeBeforeRelease) + XCTAssertTrue(box.drained) + guard case .idle = currentMainThreadBusyState() else { + return XCTFail("expected the runner to become idle after the routing probe drained") + } + XCTAssertFalse(hasAbandonedMainThreadWork()) + } + + func testSkipAppActivationPreflightRejectsSelectorAndMixedSequenceGestures() throws { + app.launch() + currentApp = app + currentBundleId = nil + defer { + currentApp = nil + currentBundleId = nil + app.terminate() + } + let selectorTap = try runnerCommandFixture( + #"{"command":"tap","commandId":"tap-1","selectorKey":"label","selectorValue":"Search","synthesized":true}"# + ) + let standardDrag = try runnerCommandFixture( + #"{"command":"drag","commandId":"drag-1","x":10,"y":20,"x2":30,"y2":40}"# + ) + let mixedSequence = try runnerCommandFixture( + """ + {"command":"sequence","commandId":"seq-1","steps":[ + {"kind":"tap","x":10,"y":20,"synthesized":true}, + {"kind":"doubleTap","x":30,"y":40} + ]} + """ + ) + + XCTAssertFalse(shouldSkipAppActivationPreflight(selectorTap)) + XCTAssertFalse(shouldSkipAppActivationPreflight(standardDrag)) + XCTAssertFalse(shouldSkipAppActivationPreflight(mixedSequence)) + } + + // Launches nothing, but still simulator-only: `shouldSkipAppActivationPreflight` is + // `#if os(iOS) …guards… #else return false #endif`, so on macOS this asserts a compile-time + // literal and no edit to the iOS body could make it red. Its five siblings above and below + // are gated for the same reason. + func testSkipAppActivationPreflightRequiresCachedForegroundTarget() throws { + currentApp = nil + currentBundleId = nil + let scroll = try runnerCommandFixture( + #"{"command":"scroll","commandId":"scroll-1","direction":"down","pixels":400}"# + ) + + XCTAssertFalse(shouldSkipAppActivationPreflight(scroll)) + } + + func testSkipAppActivationPreflightKeepsDragScrollAndSequenceOnForegroundGuard() throws { + app.launch() + currentApp = app + currentBundleId = nil + defer { + currentApp = nil + currentBundleId = nil + app.terminate() + } + let drag = try runnerCommandFixture( + #"{"command":"drag","commandId":"drag-1","x":10,"y":20,"x2":30,"y2":40}"# + ) + let scroll = try runnerCommandFixture( + #"{"command":"scroll","commandId":"scroll-1","direction":"down","pixels":400}"# + ) + let sequence = try runnerCommandFixture( + """ + {"command":"sequence","commandId":"seq-1","steps":[ + {"kind":"tap","x":10,"y":20,"synthesized":true}, + {"kind":"longPress","x":10,"y":200,"durationMs":300} + ]} + """ + ) + + XCTAssertFalse(shouldSkipAppActivationPreflight(drag)) + XCTAssertFalse(shouldSkipAppActivationPreflight(scroll)) + XCTAssertFalse(shouldSkipAppActivationPreflight(sequence)) + } + + func testSkipAppActivationPreflightIncludesAlertCommands() throws { + let alert = try runnerCommandFixture( + #"{"command":"alert","commandId":"alert-1","action":"get"}"# + ) + + XCTAssertTrue(shouldSkipAppActivationPreflight(alert)) + } +#endif + + func testDispatchReturnsBusyBeforeQueueingMainThreadWork() throws { + let command = try runnerCommandFixture(#"{"command":"snapshot","commandId":"snapshot-busy"}"#) + abandonedMainThreadWorkCount = 1 + abandonedMainThreadWorkSince = Date(timeIntervalSinceNow: -2) + defer { + abandonedMainThreadWorkCount = 0 + abandonedMainThreadWorkSince = nil + } + + let response = try execute(command: command) + + XCTAssertFalse(response.ok) + XCTAssertEqual(response.error?.code, "RUNNER_BUSY") + XCTAssertTrue(response.error?.message.contains("previous command") == true) + } + + func testDispatchReturnsWedgedBeforeQueueingMainThreadWork() throws { + let command = try runnerCommandFixture(#"{"command":"snapshot","commandId":"snapshot-wedged"}"#) + abandonedMainThreadWorkCount = 1 + abandonedMainThreadWorkSince = Date(timeIntervalSinceNow: -(mainThreadWedgeThreshold + 1)) + defer { + abandonedMainThreadWorkCount = 0 + abandonedMainThreadWorkSince = nil + } + + let response = try execute(command: command) + + XCTAssertFalse(response.ok) + XCTAssertEqual(response.error?.code, "RUNNER_WEDGED") + XCTAssertTrue(response.error?.hint?.contains("runner session will be restarted") == true) + } + + func testPostSnapshotDelayMarkDoesNotQueueBehindAbandonedMainThreadWork() { + abandonedMainThreadWorkCount = 1 + defer { + abandonedMainThreadWorkCount = 0 + needsPostSnapshotInteractionDelay = false + } + + let finished = expectation(description: "off-main caller finished") + DispatchQueue(label: "agent-device.runner.tests.post-snapshot-delay").async { + self.setNeedsPostSnapshotInteractionDelay() + finished.fulfill() + } + + wait(for: [finished], timeout: 1) + mainThreadWorkLock.lock() + let abandonedWorkCount = abandonedMainThreadWorkCount + mainThreadWorkLock.unlock() + XCTAssertEqual(abandonedWorkCount, 1, "the skipped mark must not add an abandoned unit") + XCTAssertFalse(needsPostSnapshotInteractionDelay) + } + + func testSnapshotFailureInvalidationQueuesBehindAbandonedMainThreadWorkWithoutWaiting() { + currentBundleId = "com.example.stale-target" + defer { currentBundleId = nil } + + final class ResultBox { + var elapsed: TimeInterval? + var bundleStillCachedWhileBlocked: Bool? + var abandonedWhileBlocked: Int? + } + let box = ResultBox() + let mainBlocked = DispatchSemaphore(value: 0) + let releaseMain = DispatchSemaphore(value: 0) + let finished = expectation(description: "invalidation returned while main was blocked") + + DispatchQueue(label: "agent-device.runner.tests.snapshot-invalidation").async { + _ = try? self.runMainThreadWork( + "command_execution", + timeout: 0, + timeoutError: self.mainThreadExecutionTimeoutError + ) { + mainBlocked.signal() + _ = releaseMain.wait(timeout: .now() + 5) + return true + } + _ = mainBlocked.wait(timeout: .now() + 2) + let startedAt = Date() + self.invalidateCachedTargetAfterSnapshotFailure() + box.elapsed = Date().timeIntervalSince(startedAt) + box.bundleStillCachedWhileBlocked = self.currentBundleId != nil + self.mainThreadWorkLock.lock() + box.abandonedWhileBlocked = self.abandonedMainThreadWorkCount + self.mainThreadWorkLock.unlock() + releaseMain.signal() + finished.fulfill() + } + + wait(for: [finished], timeout: 8) + let drainDeadline = Date().addingTimeInterval(2) + while hasAbandonedMainThreadWork() || currentBundleId != nil, Date() < drainDeadline { + sleepFor(0.005) + } + + XCTAssertLessThan( + box.elapsed ?? .infinity, + 0.5, + "the failed capture must not wait behind abandoned main-thread work" + ) + XCTAssertEqual( + box.bundleStillCachedWhileBlocked, + true, + "the drop must queue behind the blocked main thread, not run early" + ) + XCTAssertEqual(box.abandonedWhileBlocked, 1, "the deferred drop must not add an abandoned unit") + XCTAssertFalse(hasAbandonedMainThreadWork()) + XCTAssertNil(currentBundleId, "the drop must run once the main thread frees") + } + + /// Routes `command` through the transport's inline and queued paths. The calling test's main + /// thread serves the command's main-thread work while it waits. + func execute(command: Command) throws -> Response { + dispatchPrecondition(condition: .onQueue(.main)) + if let response = inlineResponse(for: command) { + return response + } + final class ResultBox { + var result: Result? + } + let box = ResultBox() + let executed = XCTestExpectation(description: "\(command.command.rawValue) executed off main") + enqueueAccepted(command: command) { result in + box.result = result + executed.fulfill() + } + guard XCTWaiter.wait(for: [executed], timeout: mainThreadExecutionTimeout + 5) == .completed, + let result = box.result + else { + throw NSError( + domain: RunnerErrorDomain.general, + code: RunnerErrorCode.commandReturnedNoResponse, + userInfo: [NSLocalizedDescriptionKey: "command did not finish on the command queue"] + ) + } + return try result.get() + } + + func runnerCommandFixture(_ json: String) throws -> Command { + try JSONDecoder().decode(Command.self, from: Data(json.utf8)) + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandJournalTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandJournalTests.swift new file mode 100644 index 0000000000..44555b0b76 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandJournalTests.swift @@ -0,0 +1,340 @@ +import Foundation +import XCTest +import AgentDeviceSnapshotPresentation + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +extension RunnerTests { + func testUptimeBypassesCommandJournal() throws { + let command = runnerJournalCommand("uptime", id: "uptime-probe") + + let response = try execute(command: command) + let status = commandJournal.status(normalizedCommandId: "uptime-probe") + + XCTAssertEqual(response.ok, true) + XCTAssertNotNil(response.data?.currentUptimeMs) + XCTAssertEqual(status.lifecycleState, RunnerCommandLifecycleState.notAccepted.rawValue) + } + + func testStampingCurrentUptimePreservesPayload() { + let stamped = Response(ok: true, data: DataPayload(message: "recording started")) + .stampingCurrentUptimeMs(123.5) + + XCTAssertEqual(stamped.ok, true) + XCTAssertEqual(stamped.data?.message, "recording started") + XCTAssertEqual(stamped.data?.currentUptimeMs, 123.5) + } + + func testStampingCurrentUptimeCreatesPayloadWhenNil() { + let stamped = Response(ok: true).stampingCurrentUptimeMs(456.0) + + XCTAssertEqual(stamped.ok, true) + XCTAssertEqual(stamped.data?.currentUptimeMs, 456.0) + } + + func testStampingCurrentUptimeSkipsErrorResponses() { + let response = Response(ok: false, error: ErrorPayload(message: "boom")) + let stamped = response.stampingCurrentUptimeMs(789.0) + + XCTAssertEqual(stamped.ok, false) + XCTAssertNil(stamped.data) + XCTAssertEqual(stamped.error?.message, "boom") + } + + func testStampingCurrentMainThreadBusyPreservesPayload() { + let stamped = Response(ok: true, data: DataPayload(nodes: [], truncated: false)) + .stampingCurrentMainThreadBusy(true) + + XCTAssertEqual(stamped.ok, true) + XCTAssertEqual(stamped.data?.runnerMainThreadBusy, true) + } + + func testStampingCurrentMainThreadBusySkipsErrorResponses() { + let response = Response(ok: false, error: ErrorPayload(code: "RUNNER_BUSY", message: "busy")) + let stamped = response.stampingCurrentMainThreadBusy(true) + + XCTAssertEqual(stamped.ok, false) + XCTAssertNil(stamped.data) + XCTAssertEqual(stamped.error?.code, "RUNNER_BUSY") + } + + func testMainThreadBusyStateReportsOccupancy() { + XCTAssertFalse(MainThreadBusyState.idle.reportsMainThreadBusy) + XCTAssertTrue(MainThreadBusyState.busy(abandonedForSeconds: 5).reportsMainThreadBusy) + XCTAssertTrue(MainThreadBusyState.wedged(abandonedForSeconds: 200).reportsMainThreadBusy) + XCTAssertEqual( + Response(ok: true).stampingCurrentMainThreadBusy(false).data?.runnerMainThreadBusy, false) + } + + func testCommandFailedResponseTagsMainThreadTimeoutWithTypedCode() { + let timeout = NSError( + domain: RunnerErrorDomain.general, + code: RunnerErrorCode.mainThreadExecutionTimedOut, + userInfo: [NSLocalizedDescriptionKey: "main thread execution timed out"] + ) + + let response = commandFailedResponse(from: timeout) + + XCTAssertEqual(response.ok, false) + XCTAssertEqual(response.error?.code, RunnerWireErrorCode.mainThreadTimeout) + } + + func testCommandFailedResponseKeepsGenericCodeForOtherErrors() { + let other = NSError(domain: "SomeOtherDomain", code: 99, userInfo: nil) + + let response = commandFailedResponse(from: other) + + XCTAssertEqual(response.error?.code, "COMMAND_FAILED") + } + + func testJournalStoredResponseStaysUnstamped() throws { + let journal = RunnerCommandJournal() + let recordStart = runnerJournalCommand("recordStart", id: "record-start-anchor") + + journal.accept(command: recordStart) + journal.finish( + command: recordStart, + response: Response(ok: true, data: DataPayload(message: "recording started")) + ) + + let status = journal.status(normalizedCommandId: "record-start-anchor") + let responseJson = try XCTUnwrap(status.lifecycleResponseJson) + XCTAssertFalse(responseJson.contains("currentUptimeMs")) + } + + func testCommandJournalRetentionPolicy() throws { + let journal = RunnerCommandJournal() + + let uptime = runnerJournalCommand("uptime", id: "small-scalar") + journal.accept(command: uptime) + journal.finish( + command: uptime, + response: Response(ok: true, data: DataPayload(currentUptimeMs: 12.5)) + ) + + let scalarStatus = journal.status(normalizedCommandId: "small-scalar") + XCTAssertEqual(scalarStatus.lifecycleState, RunnerCommandLifecycleState.completed.rawValue) + XCTAssertEqual(scalarStatus.lifecycleResponseOk, true) + XCTAssertNotNil(scalarStatus.lifecycleResponseJson) + let scalarResponse = try decodeRunnerJournalResponse(scalarStatus.lifecycleResponseJson) + XCTAssertEqual(scalarResponse.data?.currentUptimeMs, 12.5) + + let querySelector = runnerJournalCommand("querySelector", id: "small-object") + journal.accept(command: querySelector) + journal.finish( + command: querySelector, + response: Response(ok: true, data: DataPayload(found: true, nodes: [runnerJournalNode()])) + ) + + let objectStatus = journal.status(normalizedCommandId: "small-object") + XCTAssertNotNil(objectStatus.lifecycleResponseJson) + let objectResponse = try decodeRunnerJournalResponse(objectStatus.lifecycleResponseJson) + XCTAssertEqual(objectResponse.data?.found, true) + XCTAssertEqual(objectResponse.data?.nodes?.count, 1) + + let snapshot = runnerJournalCommand("snapshot", id: "snapshot-tree") + journal.accept(command: snapshot) + journal.finish( + command: snapshot, + response: Response(ok: true, data: DataPayload(nodes: [runnerJournalNode()], truncated: false)) + ) + + let snapshotStatus = journal.status(normalizedCommandId: "snapshot-tree") + XCTAssertEqual(snapshotStatus.lifecycleState, RunnerCommandLifecycleState.completed.rawValue) + XCTAssertEqual(snapshotStatus.lifecycleResponseOk, true) + XCTAssertNil(snapshotStatus.lifecycleResponseJson) + + let screenshot = runnerJournalCommand("screenshot", id: "screenshot-artifact") + journal.accept(command: screenshot) + journal.finish( + command: screenshot, + response: Response(ok: true, data: DataPayload(message: "tmp/screenshot-1.png")) + ) + + let screenshotStatus = journal.status(normalizedCommandId: "screenshot-artifact") + XCTAssertEqual(screenshotStatus.lifecycleState, RunnerCommandLifecycleState.completed.rawValue) + XCTAssertEqual(screenshotStatus.lifecycleResponseOk, true) + XCTAssertNil(screenshotStatus.lifecycleResponseJson) + + let scroll = runnerJournalCommand("scroll", id: "scroll-drag") + journal.accept(command: scroll) + journal.finish( + command: scroll, + response: Response( + ok: true, + data: DataPayload( + message: "scrolled", + gestureStartUptimeMs: 1, + gestureEndUptimeMs: 2, + x: 155, + y: 420, + x2: 155, + y2: 301, + referenceWidth: 300, + referenceHeight: 600 + ) + ) + ) + + let scrollStatus = journal.status(normalizedCommandId: "scroll-drag") + XCTAssertEqual(scrollStatus.lifecycleState, RunnerCommandLifecycleState.completed.rawValue) + XCTAssertEqual(scrollStatus.lifecycleResponseOk, true) + XCTAssertNotNil(scrollStatus.lifecycleResponseJson) + let scrollResponse = try decodeRunnerJournalResponse(scrollStatus.lifecycleResponseJson) + XCTAssertEqual(scrollResponse.data?.x, 155) + XCTAssertEqual(scrollResponse.data?.y, 420) + XCTAssertEqual(scrollResponse.data?.x2, 155) + XCTAssertEqual(scrollResponse.data?.y2, 301) + XCTAssertEqual(scrollResponse.data?.referenceWidth, 300) + XCTAssertEqual(scrollResponse.data?.referenceHeight, 600) + + let largeRead = runnerJournalCommand("readText", id: "large-read") + journal.accept(command: largeRead) + journal.finish( + command: largeRead, + response: Response(ok: true, data: DataPayload(text: String(repeating: "x", count: 17 * 1024))) + ) + + let largeReadStatus = journal.status(normalizedCommandId: "large-read") + XCTAssertEqual(largeReadStatus.lifecycleState, RunnerCommandLifecycleState.completed.rawValue) + XCTAssertEqual(largeReadStatus.lifecycleResponseOk, true) + XCTAssertNil(largeReadStatus.lifecycleResponseJson) + } + + func testCommandJournalKeepsErrorMetadataWhenResponseJsonIsDropped() { + let journal = RunnerCommandJournal() + let snapshot = runnerJournalCommand("snapshot", id: "snapshot-error") + let hint = "Try a smaller read such as snapshot -s -d 8." + + journal.accept(command: snapshot) + journal.finish( + command: snapshot, + response: Response( + ok: false, + error: ErrorPayload( + code: "IOS_AX_SNAPSHOT_FAILED", + message: "iOS XCTest snapshot failed while serializing the accessibility tree.", + hint: hint + ) + ) + ) + + let status = journal.status(normalizedCommandId: "snapshot-error") + XCTAssertEqual(status.lifecycleState, RunnerCommandLifecycleState.failed.rawValue) + XCTAssertEqual(status.lifecycleResponseOk, false) + XCTAssertNil(status.lifecycleResponseJson) + XCTAssertEqual(status.lifecycleErrorCode, "IOS_AX_SNAPSHOT_FAILED") + XCTAssertEqual( + status.lifecycleErrorMessage, + "iOS XCTest snapshot failed while serializing the accessibility tree." + ) + XCTAssertEqual(status.lifecycleErrorHint, hint) + } + + func testCommandJournalRetainsCompletedSequenceResults() throws { + let journal = RunnerCommandJournal() + let sequence = runnerJournalCommand("sequence", id: "sequence-completed") + let results = (0..<20).map { _ in + SequenceStepResult( + ok: true, + kind: "tap", + errorCode: nil, + errorMessage: nil, + gestureStartUptimeMs: 100, + gestureEndUptimeMs: 120 + ) + } + + journal.accept(command: sequence) + journal.finish( + command: sequence, + response: Response( + ok: true, + data: DataPayload( + message: "sequence", + completedSteps: 20, + failedStepIndex: nil, + sequenceResults: results + ) + ) + ) + + let status = journal.status(normalizedCommandId: "sequence-completed") + XCTAssertEqual(status.lifecycleState, RunnerCommandLifecycleState.completed.rawValue) + XCTAssertEqual(status.lifecycleResponseOk, true) + let json = try XCTUnwrap(status.lifecycleResponseJson) + // Worst-case 20-step response must stay under the 16KB journal retention cap. + XCTAssertLessThan(json.utf8.count, 16 * 1024) + let decoded = try decodeRunnerJournalResponse(status.lifecycleResponseJson) + XCTAssertEqual(decoded.data?.completedSteps, 20) + XCTAssertEqual(decoded.data?.sequenceResults?.count, 20) + } + + func testCommandJournalRetainsFailedSequenceResults() throws { + let journal = RunnerCommandJournal() + let sequence = runnerJournalCommand("sequence", id: "sequence-failed") + let longError = String(repeating: "z", count: 200) + let results: [SequenceStepResult] = [ + SequenceStepResult(ok: true, kind: "tap", errorCode: nil, errorMessage: nil, + gestureStartUptimeMs: 100, gestureEndUptimeMs: 120), + SequenceStepResult(ok: true, kind: "tap", errorCode: nil, errorMessage: nil, + gestureStartUptimeMs: 130, gestureEndUptimeMs: 150), + SequenceStepResult(ok: false, kind: "longPress", errorCode: "UNSUPPORTED_OPERATION", + errorMessage: longError, gestureStartUptimeMs: 160, gestureEndUptimeMs: 180), + ] + + journal.accept(command: sequence) + journal.finish( + command: sequence, + response: Response( + ok: true, + data: DataPayload( + message: "sequence", + completedSteps: 2, + failedStepIndex: 2, + sequenceResults: results + ) + ) + ) + + let status = journal.status(normalizedCommandId: "sequence-failed") + XCTAssertEqual(status.lifecycleState, RunnerCommandLifecycleState.completed.rawValue) + let decoded = try decodeRunnerJournalResponse(status.lifecycleResponseJson) + XCTAssertEqual(decoded.data?.completedSteps, 2) + XCTAssertEqual(decoded.data?.failedStepIndex, 2) + XCTAssertEqual(decoded.data?.sequenceResults?.count, 3) + XCTAssertEqual(decoded.data?.sequenceResults?[2].ok, false) + XCTAssertEqual(decoded.data?.sequenceResults?[2].errorCode, "UNSUPPORTED_OPERATION") + } + + private func runnerJournalCommand(_ command: String, id: String) -> Command { + let json = #"{"command":"\#(command)","commandId":"\#(id)"}"# + return try! JSONDecoder().decode(Command.self, from: Data(json.utf8)) + } + + private func runnerJournalNode() -> PresentedNode { + SnapshotPresentation.singleElementRead( + RawAXNode( + index: 0, + type: "button", + label: "Continue", + identifier: "continue", + value: nil, + rect: SnapshotRect(x: 10, y: 20, width: 100, height: 44), + enabled: true, + focused: nil, + selected: nil, + hittable: true, + depth: 0, + parentIndex: nil, + hiddenContentAbove: nil, + hiddenContentBelow: nil + ) + ) + } + + private func decodeRunnerJournalResponse(_ responseJson: String?) throws -> Response { + let responseJson = try XCTUnwrap(responseJson) + return try JSONDecoder().decode(Response.self, from: Data(responseJson.utf8)) + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+InteractionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+InteractionTests.swift new file mode 100644 index 0000000000..659dbd9c8b --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+InteractionTests.swift @@ -0,0 +1,27 @@ +import XCTest +import AgentDeviceSnapshotPresentation + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +extension RunnerTests { + func testDesktopScrollWheelDeltasMapDirections() { + XCTAssertEqual(desktopScrollWheelDeltas(direction: .up, pixels: 120).vertical, 120) + XCTAssertEqual(desktopScrollWheelDeltas(direction: .down, pixels: 120).vertical, -120) + XCTAssertEqual(desktopScrollWheelDeltas(direction: .left, pixels: 120).horizontal, 120) + XCTAssertEqual(desktopScrollWheelDeltas(direction: .right, pixels: 120).horizontal, -120) + } + + func testDesktopScrollWheelDeltaEventsHonorDurationAndPreservePixels() { + let events = desktopScrollWheelDeltaEvents(direction: .down, pixels: 200, durationMs: 50) + XCTAssertEqual(events.count, 4) + XCTAssertEqual(events.map(\.vertical).reduce(0, +), -200) + XCTAssertEqual(events.map(\.horizontal).reduce(0, +), 0) + XCTAssertEqual(desktopScrollEventIntervalSeconds(durationMs: 50, eventCount: events.count), 0.05 / 3.0) + } + + func testDesktopScrollWheelDeltaEventsKeepInstantScrollSingleEvent() { + let events = desktopScrollWheelDeltaEvents(direction: .down, pixels: 200, durationMs: 0) + XCTAssertEqual(events.count, 1) + XCTAssertEqual(events.first?.vertical, -200) + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+KeyboardTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+KeyboardTests.swift new file mode 100644 index 0000000000..8264903617 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+KeyboardTests.swift @@ -0,0 +1,144 @@ +import XCTest +import AgentDeviceSnapshotPresentation + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +extension RunnerTests { + func testKeyboardBandFactReadFailureIsStatedAsUnmeasurableNotAbsence() { + // Absence would let a consumer claim the screen is clear of a keyboard on the strength of a read + // that never answered. + let fact = runnerKeyboardBandFact( + readSucceeded: false, + exists: false, + frame: CGRect(x: 0, y: 198, width: 402, height: 204) + ) + XCTAssertEqual(fact, .unmeasurable(RunnerKeyboardBandReason.queryFailed)) + XCTAssertEqual(fact.payload.kind, "unmeasurable") + XCTAssertEqual(fact.payload.reason, RunnerKeyboardBandReason.queryFailed) + XCTAssertNil(fact.payload.frame) + } + + func testKeyboardBandFactPublishesAbsenceWhenTheQueryFindsNoKeyboard() { + let fact = runnerKeyboardBandFact(readSucceeded: true, exists: false, frame: .zero) + XCTAssertEqual(fact, .absent) + XCTAssertEqual(fact.payload.kind, "absent") + XCTAssertNil(fact.payload.frame) + XCTAssertNil(fact.payload.reason) + } + + func testKeyboardBandFactPublishesTheMeasuredBandInAppOrientationSpace() { + // The landscape band measured on iPhone 17 Pro (iOS 26.2) after #2653: full width across the + // bottom of a 402 pt-tall app, which is what the tree reported as a strip down the left edge. + let frame = CGRect(x: 0, y: 198, width: 874, height: 204) + let fact = runnerKeyboardBandFact(readSucceeded: true, exists: true, frame: frame) + XCTAssertEqual(fact, .visible(frame)) + let payload = fact.payload + XCTAssertEqual(payload.kind, "visible") + XCTAssertEqual(payload.frame, SnapshotRect(x: 0, y: 198, width: 874, height: 204)) + XCTAssertNil(payload.reason) + } + + func testKeyboardBandFactRefusesUnusableGeometryInsteadOfClaimingAbsence() { + let nan = CGFloat(Double.nan) + let infinite = CGFloat.infinity + let unusable: [CGRect] = [ + .zero, + CGRect(x: 0, y: 198, width: 0, height: 204), + CGRect(x: 0, y: 198, width: 874, height: -1), + CGRect(x: 0, y: 198, width: -874, height: 204), + CGRect(x: 0, y: nan, width: 874, height: 204), + CGRect(x: 0, y: 198, width: nan, height: 204), + CGRect(x: infinite, y: 198, width: 874, height: 204) + ] + for frame in unusable { + let fact = runnerKeyboardBandFact(readSucceeded: true, exists: true, frame: frame) + XCTAssertEqual( + fact, + .unmeasurable(RunnerKeyboardBandReason.unusableFrame), + "expected \(frame) to be refused as a band" + ) + } + } + + func testKeyboardBandFactPayloadRoundTripsThroughTheWireShape() throws { + let cases: [RunnerKeyboardBandFact] = [ + .visible(CGRect(x: 0, y: 583, width: 402, height: 291)), + .absent, + .unmeasurable(RunnerKeyboardBandReason.queryTimeout) + ] + for fact in cases { + let data = try JSONEncoder().encode(fact.payload) + // `encodeIfPresent` for the two optional fields: a fact carries its own evidence and nothing + // else, so the daemon never has to distinguish a null from an absent key. + let object = try XCTUnwrap(JSONSerialization.jsonObject(with: data) as? [String: Any]) + switch fact { + case .visible: + XCTAssertNil(object["reason"]) + XCTAssertNotNil(object["frame"]) + case .absent: + XCTAssertNil(object["frame"]) + XCTAssertNil(object["reason"]) + case .unmeasurable: + XCTAssertNil(object["frame"]) + XCTAssertNotNil(object["reason"]) + } + XCTAssertEqual(try JSONDecoder().decode(KeyboardBandFactPayload.self, from: data), fact.payload) + } + } + + func testRunnerScreenshotStabilitySettledNeedsEnoughSamples() { + XCTAssertFalse(runnerScreenshotStabilitySettled([], requiredConsecutiveMatches: 3)) + XCTAssertFalse(runnerScreenshotStabilitySettled([Data([1])], requiredConsecutiveMatches: 3)) + let frame = Data([1, 2, 3]) + XCTAssertFalse( + runnerScreenshotStabilitySettled([frame, frame], requiredConsecutiveMatches: 3) + ) + } + + func testRunnerScreenshotStabilitySettledTrueWhenWindowMatches() { + let frame = Data([1, 2, 3]) + XCTAssertTrue( + runnerScreenshotStabilitySettled([Data([9]), frame, frame, frame], requiredConsecutiveMatches: 3) + ) + } + + func testRunnerScreenshotStabilitySettledFalseOnMidWindowMismatch() { + // A momentary pause (two matching samples) followed by resumed movement + // must not read as settled: the 3-sample window still spans the mismatch. + let frame = Data([1, 2, 3]) + let moved = Data([4, 5, 6]) + XCTAssertFalse( + runnerScreenshotStabilitySettled([frame, frame, moved], requiredConsecutiveMatches: 3) + ) + } + + func testRunnerScreenshotStabilitySettledFalseOnFailedCapture() { + // A nil sample (failed screenshot) never counts as a match, even against + // other nils — an unverifiable run must not look "stable". + XCTAssertFalse(runnerScreenshotStabilitySettled([nil, nil, nil], requiredConsecutiveMatches: 3)) + let frame = Data([1, 2, 3]) + XCTAssertFalse( + runnerScreenshotStabilitySettled([frame, frame, nil], requiredConsecutiveMatches: 3) + ) + } + + func testRunnerScreenshotStabilitySettledOnlyLooksAtTheTrailingWindow() { + // An older mismatch before the trailing window must not block settlement + // once the required run of most-recent samples agrees. + let frame = Data([9]) + XCTAssertTrue( + runnerScreenshotStabilitySettled( + [Data([1]), Data([2]), frame, frame, frame], + requiredConsecutiveMatches: 3 + ) + ) + } + + func testRunnerScreenshotStabilitySettledRejectsDegenerateRequirement() { + // Fewer than 2 required matches would make any single sample "settled" — + // guard against a misconfigured caller rather than silently no-op the wait. + XCTAssertFalse( + runnerScreenshotStabilitySettled([Data([1])], requiredConsecutiveMatches: 1) + ) + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LifecycleTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LifecycleTests.swift index cba532f03b..559fa9021b 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LifecycleTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LifecycleTests.swift @@ -104,3 +104,33 @@ extension RunnerTests { } } #endif + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +extension RunnerTests { + func testResettingTargetBoundStateForgetsTheLastWrittenMarkers() { + defer { invalidateCachedTarget(reason: "unit_test_cleanup") } + lastLoggedFastAppGuardLine = "AGENT_DEVICE_RUNNER_FAST_APP_GUARD bundle=app state=4" + lastLoggedGesturePolicyLines[.scroll] = "AGENT_DEVICE_RUNNER_SYNTHESIZED_GESTURE_POLICY kind=scroll" + resetTargetBoundState() + XCTAssertNil(lastLoggedFastAppGuardLine, "a rebind must state the guard once more") + XCTAssertTrue(lastLoggedGesturePolicyLines.isEmpty, "a rebind must state the policy once more") + } + + func testFastAppGuardMarkerWritesOnceUntilTheFactChanges() { + var written: [String] = [] + runnerMarkerWriter = { written.append($0) } + defer { + runnerMarkerWriter = { NSLog("%@", $0) } + invalidateCachedTarget(reason: "unit_test_cleanup") + } + writeFastAppGuardMarker(bundleId: "com.example.app", state: .runningForeground) + writeFastAppGuardMarker(bundleId: "com.example.app", state: .runningForeground) + XCTAssertEqual(written.count, 1, "a repeated fact writes no second line") + writeFastAppGuardMarker(bundleId: "com.example.other", state: .runningForeground) + XCTAssertEqual(written.count, 2, "a changed fact writes a new line") + resetTargetBoundState() + writeFastAppGuardMarker(bundleId: "com.example.other", state: .runningForeground) + XCTAssertEqual(written.count, 3, "a rebind states the same fact once more") + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+MainThreadWorkTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+MainThreadWorkTests.swift new file mode 100644 index 0000000000..f98e37e7a8 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+MainThreadWorkTests.swift @@ -0,0 +1,93 @@ +import XCTest + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +extension RunnerTests { + func testRunMainThreadWorkExecutesOffMainCallerOnMainThread() { + final class ResultBox { + var observedMainThread: Bool? + var error: Error? + } + let box = ResultBox() + let finished = expectation(description: "off-main caller finished") + + DispatchQueue(label: "agent-device.runner.tests.off-main").async { + do { + box.observedMainThread = try self.runMainThreadWork( + "command_execution", + timeout: 1, + timeoutError: self.mainThreadExecutionTimeoutError + ) { + Thread.isMainThread + } + } catch { + box.error = error + } + finished.fulfill() + } + + wait(for: [finished], timeout: 2) + XCTAssertNil(box.error) + XCTAssertEqual(box.observedMainThread, true) + } + + func testRunMainThreadWorkTimeoutMarksAbandonedUntilDrained() { + final class ResultBox { + var error: Error? + var abandonedCount: Int? + var abandonedSinceSet: Bool? + var busyWhileAbandoned = false + } + let box = ResultBox() + let releaseWork = DispatchSemaphore(value: 0) + let observedAbandoned = DispatchSemaphore(value: 0) + let timedOut = expectation(description: "off-main caller timed out") + + DispatchQueue(label: "agent-device.runner.tests.timeout").async { + do { + _ = try self.runMainThreadWork( + "command_execution", + timeout: 0, + timeoutError: self.mainThreadExecutionTimeoutError + ) { + _ = releaseWork.wait(timeout: .now() + 2) + return true + } + } catch { + box.error = error + } + self.mainThreadWorkLock.lock() + box.abandonedCount = self.abandonedMainThreadWorkCount + box.abandonedSinceSet = self.abandonedMainThreadWorkSince != nil + self.mainThreadWorkLock.unlock() + if case .busy = self.currentMainThreadBusyState() { + box.busyWhileAbandoned = true + } + observedAbandoned.signal() + timedOut.fulfill() + } + DispatchQueue(label: "agent-device.runner.tests.release-timeout").async { + _ = observedAbandoned.wait(timeout: .now() + 2) + releaseWork.signal() + } + + wait(for: [timedOut], timeout: 3) + let drainDeadline = Date().addingTimeInterval(2) + while hasAbandonedMainThreadWork(), Date() < drainDeadline { + sleepFor(0.005) + } + + XCTAssertEqual((box.error as NSError?)?.code, RunnerErrorCode.mainThreadExecutionTimedOut) + XCTAssertEqual(box.abandonedCount, 1) + XCTAssertEqual(box.abandonedSinceSet, true) + XCTAssertTrue(box.busyWhileAbandoned) + XCTAssertFalse(hasAbandonedMainThreadWork(), "drained work must release the main thread") + mainThreadWorkLock.lock() + let sinceCleared = abandonedMainThreadWorkSince == nil + mainThreadWorkLock.unlock() + XCTAssertTrue(sinceCleared) + guard case .idle = currentMainThreadBusyState() else { + return XCTFail("expected the runner idle once the abandoned work drained") + } + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift new file mode 100644 index 0000000000..e18e91241f --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift @@ -0,0 +1,169 @@ +import XCTest + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +extension RunnerTests { + func testTopLeadingNavigationFallbackPointTargetsHeaderControlBand() throws { + let point = try XCTUnwrap( + Self.topLeadingNavigationFallbackPoint( + in: CGRect(x: 0, y: 0, width: 430, height: 932) + ) + ) + + XCTAssertEqual(point.x, 34.4, accuracy: 0.01) + XCTAssertEqual(point.y, 132, accuracy: 0.01) + } + + func testTopLeadingNavigationFallbackPointRejectsInvalidFrame() { + XCTAssertNil(Self.topLeadingNavigationFallbackPoint(in: .infinite)) + XCTAssertNil(Self.topLeadingNavigationFallbackPoint(in: .zero)) + } + + func testNavigationBackControlRankPrefersBackThenCloseThenCancel() { + XCTAssertEqual(Self.navigationBackControlRank(label: "Back", identifier: ""), 0) + XCTAssertEqual(Self.navigationBackControlRank(label: "Close", identifier: ""), 1) + XCTAssertEqual(Self.navigationBackControlRank(label: "Cancel search", identifier: ""), 2) + XCTAssertNil(Self.navigationBackControlRank(label: "Search for more feeds", identifier: "")) + } + + func testNavigationBackPredicateUsesTheSharedKeywordTable() { + let predicate = Self.navigationBackPredicate() + + XCTAssertTrue(predicate.evaluate(with: ["label": "Back", "identifier": ""])) + XCTAssertTrue(predicate.evaluate(with: ["label": "", "identifier": "close-button"])) + XCTAssertFalse(predicate.evaluate(with: ["label": "Search for more feeds", "identifier": ""])) + } + + func testTopNavigationControlFrameAcceptsOnlyHeaderBand() { + let window = CGRect(x: 0, y: 0, width: 430, height: 932) + + XCTAssertTrue( + Self.isTopNavigationControlFrame( + CGRect(x: 340, y: 84, width: 72, height: 44), + in: window + ) + ) + XCTAssertFalse( + Self.isTopNavigationControlFrame( + CGRect(x: 20, y: 760, width: 72, height: 44), + in: window + ) + ) + XCTAssertFalse(Self.isTopNavigationControlFrame(.infinite, in: window)) + } + + func testNavigationVisualVerificationSeparatesNoChangeFromNoSample() { + XCTAssertEqual( + Self.navigationVisualObservation(before: Data([1, 2, 3]), after: Data([1, 2, 4])), + .changed + ) + XCTAssertEqual( + Self.navigationVisualObservation(before: Data([1, 2, 3]), after: Data([1, 2, 3])), + .unchanged + ) + // A missing sample is neither a change nor a no-change; treating it as "unchanged" would let a + // capture that refused become the reason the `back` command claims no control exists (#2728). + XCTAssertEqual(Self.navigationVisualObservation(before: nil, after: Data([1])), .unobserved) + XCTAssertEqual(Self.navigationVisualObservation(before: Data([1]), after: nil), .unobserved) + XCTAssertEqual(Self.navigationVisualObservation(before: nil, after: nil), .unobserved) + } + + func testNavigationFallbackReportsTheRefusalItHitNotADefaultCode() { + // The refusal from the most recent sample wins, so the code names what the fallback last looked at + // before giving up; an earlier refusal is reported only when the later sample carried none (#2728). + let after = NavigationVisualSample( + data: nil, + refusalCode: "APP_SCREEN_WINDOW_UNRESOLVED", + refusalHint: "after hint" + ) + let before = NavigationVisualSample( + data: nil, + refusalCode: "APP_SCREEN_UNRESOLVED", + refusalHint: "before hint" + ) + let laterWins = Self.navigationFallbackErrorPayload(after: after, before: before) + XCTAssertEqual(laterWins.code, "APP_SCREEN_WINDOW_UNRESOLVED") + XCTAssertEqual(laterWins.hint, "after hint") + + let onlyBefore = Self.navigationFallbackErrorPayload( + after: NavigationVisualSample(data: nil), + before: before + ) + XCTAssertEqual(onlyBefore.code, "APP_SCREEN_UNRESOLVED") + XCTAssertEqual(onlyBefore.hint, "before hint") + + // Neither side named a reason (unreachable on iOS): a real capture code, never a bare failure. + let unnamed = Self.navigationFallbackErrorPayload( + after: NavigationVisualSample(data: nil), + before: NavigationVisualSample(data: nil) + ) + XCTAssertEqual(unnamed.code, "APP_SCREEN_UNRESOLVED") + XCTAssertTrue(unnamed.message.contains("unknown outcome")) + } + + func testVerifyNavigationFallbackOutcomeReportsUnresolvedWindowWithoutSystemSurface() { + // The in-app `back` fallback ran its tap but the app resolved no window. It must name that refusal + // as an unknown outcome AND must not have sampled the system surface: capturing SpringBoard's home + // screen twice reads as "unchanged" and launders a wrong-process frame into "no back control + // exists" (#2728). This drives the SAME `navigationFallbackSample` production calls, handing it a + // system surface that fails the test if consulted — so reverting the fallback to sample SpringBoard + // (or dropping the refusal code) turns this red, which an inline `.never` re-creation could not. + var askedSystemSurface = false + let sample = Self.navigationFallbackSample( + resolvingApp: { .failure(.unresolvedWindow) }, + systemSurface: { + askedSystemSurface = true + return .failure(.unresolvedWindow) + }, + encoding: { _ in Data([1, 2, 3]) } + ) + XCTAssertFalse(askedSystemSurface, "the in-app fallback samples the app only, never SpringBoard") + + XCTAssertNil(sample.data) + XCTAssertEqual(sample.refusalCode, "APP_SCREEN_WINDOW_UNRESOLVED") + + let observation = Self.navigationVisualObservation(before: sample.data, after: sample.data) + XCTAssertEqual(observation, .unobserved) + + switch Self.inAppBackOutcome(observation: observation, before: sample, after: sample) { + case .unverified(let payload): + XCTAssertEqual(payload.code, "APP_SCREEN_WINDOW_UNRESOLVED") + case .performed, .unavailable: + XCTFail("a refused capture must report an unknown outcome, not 'no back control'") + } + } + + func testNavigationVisualSampleDistinguishesEncodedFrameFromRefusal() { + // The same capture entry point yields three different samples, and only the refusal ones may carry + // a code: an encoded frame is evidence, a resolved-but-unencodable image and a refusal are not + // (#2728). Reverting the mapping to a plain no-sample loses the reason a host keys on. + let captured = CapturedAppScreen( + image: RunnerImage(), + displayID: 3, + pixelWidth: 12, + pixelHeight: 24, + pixelsPerPoint: 3 + ) + + let encoded = Self.navigationVisualSample( + from: .success(captured), + encoding: { _ in Data([7, 7]) } + ) + XCTAssertEqual(encoded.data, Data([7, 7])) + XCTAssertNil(encoded.refusalCode) + + let unencodable = Self.navigationVisualSample( + from: .success(captured), + encoding: { _ in nil } + ) + XCTAssertNil(unencodable.data) + XCTAssertEqual(unencodable.refusalCode, "APP_SCREEN_CAPTURE_UNRENDERABLE") + + let refused = Self.navigationVisualSample( + from: .failure(.unresolvedScreen), + encoding: { _ in Data([7, 7]) } + ) + XCTAssertNil(refused.data) + XCTAssertEqual(refused.refusalCode, "APP_SCREEN_UNRESOLVED") + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+PrivateAXPresentationTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+PrivateAXPresentationTests.swift new file mode 100644 index 0000000000..23ee5db993 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+PrivateAXPresentationTests.swift @@ -0,0 +1,211 @@ +import XCTest +import AgentDeviceSnapshotPresentation + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +extension RunnerTests { + fileprivate static func privateAXFrame( + _ x: Double, _ y: Double, _ width: Double, _ height: Double + ) -> [String: Any] { + ["x": x, "y": y, "width": width, "height": height] + } + + /// A scroll container whose second row is scrolled out of the viewport, plus an unlabeled + /// decoration: the shapes the regular projection folds away and the raw projection must keep. + fileprivate static var privateAXScrolledFixture: [String: Any] { + let frame = privateAXFrame + return ["type": Int(XCUIElement.ElementType.application.rawValue), + "label": "Element", "frame": frame(0, 0, 402, 874), "children": [[ + "type": Int(XCUIElement.ElementType.scrollView.rawValue), "frame": frame(0, 96, 402, 700), + "actions": ["Scroll down"], + "children": [ + ["type": Int(XCUIElement.ElementType.button.rawValue), "label": "Profile picture", + "frame": frame(16, 120, 44, 44), + "children": [["type": Int(XCUIElement.ElementType.image.rawValue), + "frame": frame(16, 120, 1, 1)]]], + ["type": Int(XCUIElement.ElementType.button.rawValue), "label": "Theme", + "frame": frame(16, 900, 360, 44)]]]]] + } + + /// Acquire with the private-AX serializer, run the one normalization pass the production capture + /// plan runs (`captureWithBackend`), then present through the shared regular fold -- the production + /// route for this backend since the fold moved into presentation (#1797, #2661). + fileprivate func privateAXRegularPresentation( + rawRoot: [String: Any], + viewport: CGRect, + interactiveOnly: Bool = false + ) throws -> [PresentedNode] { + let hint = CaptureHint( + projection: .regular, depth: nil, regularPresentedDepth: nil, + interactiveOnly: interactiveOnly, customActions: false) + let nodes = privateAXNormalizedAcquisition( + rawRoot: rawRoot, hint: hint, viewport: viewport, + interfaceOrientation: RunnerInterfaceOrientation.portrait) + return try SnapshotPresentation.presentRegular( + SnapshotAcquisition( + hint: hint, nodes: nodes, truncated: false, effectiveDepth: nil, viewport: viewport, + interfaceOrientation: RunnerInterfaceOrientation.portrait), + options: PresentationOptions( + interactiveOnly: interactiveOnly, depth: nil, scope: nil, raw: false), + policy: .cursorProjected + ).nodes + } + + /// Acquire, then normalize once -- the exact pair `captureWithBackend` runs for this backend. + fileprivate func privateAXNormalizedAcquisition( + rawRoot: [String: Any], + hint: CaptureHint, + viewport: CGRect, + interfaceOrientation: Int + ) -> [RawAXNode] { + SnapshotGeometrySpace.normalized( + nodes: privateAXAcquisition(rawRoot: rawRoot, hint: hint), + viewport: viewport, + interfaceOrientation: interfaceOrientation + ) + } + + /// The one normalization pass has to reach the node it publishes: this asserts the rotated rect of + /// a key under a turned surface host, and an untouched sibling under the app's own window, so a + /// pass that drops the space a subtree declared fails here. + func testPrivateAXAcquisitionPublishesATurnedSurfaceHostInAppOrientationSpace() { + let frame = Self.privateAXFrame + let appWindow: [String: Any] = [ + "type": Int(XCUIElement.ElementType.window.rawValue), + "frame": frame(0, 0, 874, 402), + "children": [ + ["type": Int(XCUIElement.ElementType.button.rawValue), "label": "Home", + "frame": frame(204, 323, 91, 55), "children": []] + ] + ] + let keyboardWindow: [String: Any] = [ + "type": Int(XCUIElement.ElementType.window.rawValue), + "frame": frame(0, 0, 874, 402), + "children": [ + ["type": Int(XCUIElement.ElementType.other.rawValue), + "frame": frame(0, 0, 402, 874), + "children": [ + ["type": Int(XCUIElement.ElementType.key.rawValue), "label": "q", + "frame": frame(154, 77, 45, 72), "children": []] + ]] + ] + ] + let hint = CaptureHint( + projection: .raw, depth: nil, regularPresentedDepth: nil, + interactiveOnly: false, customActions: false) + let nodes = privateAXNormalizedAcquisition( + rawRoot: [ + "type": Int(XCUIElement.ElementType.application.rawValue), + "label": "Element", "frame": frame(0, 0, 874, 402), + "children": [appWindow, keyboardWindow] + ], + hint: hint, + viewport: CGRect(x: 0, y: 0, width: 874, height: 402), + interfaceOrientation: RunnerInterfaceOrientation.landscapeRight + ) + + // Measured on iPhone 17 Pro (26.2): the key plane's left column arrives 154 pt along the device's + // long axis and comes back 203 pt down the app's short one. + XCTAssertEqual( + nodes.first { $0.label == "q" }?.rect, + SnapshotRect(x: 77, y: 203, width: 72, height: 45) + ) + XCTAssertEqual( + nodes.first { $0.label == "Home" }?.rect, + SnapshotRect(x: 204, y: 323, width: 91, height: 55) + ) + } + + func testPrivateAXRegularPresentationProjectsToViewportAndKeepsScrollHint() throws { + let nodes = try privateAXRegularPresentation( + rawRoot: Self.privateAXScrolledFixture, + viewport: CGRect(x: 0, y: 0, width: 402, height: 874)) + XCTAssertEqual(nodes.compactMap(\.label), ["Element", "Profile picture"]) + let scrollView = nodes.first { $0.type == "ScrollView" } + XCTAssertEqual(scrollView?.hiddenContentBelow, true) + XCTAssertEqual(scrollView?.actions, ["Scroll down"]) + } + + /// #1797 D4: the raw projection is the acquired tree. The offscreen row and the sub-pixel + /// decoration the regular projection folds away are both present, at traversal depth, and every + /// regular node still appears -- `regular ⊆ raw` on the same capture. + func testPrivateAXRawProjectionKeepsEveryAcquiredNode() throws { + let viewport = CGRect(x: 0, y: 0, width: 402, height: 874) + let root = Self.privateAXScrolledFixture + let regular = try privateAXRegularPresentation(rawRoot: root, viewport: viewport, + interactiveOnly: true) + let raw = privateAXNormalizedAcquisition(rawRoot: root, + hint: CaptureHint( + projection: .raw, depth: nil, regularPresentedDepth: nil, + interactiveOnly: false, customActions: false), + viewport: viewport, + interfaceOrientation: RunnerInterfaceOrientation.portrait) + + XCTAssertEqual(raw.map(\.type), ["Application", "ScrollView", "Button", "Image", "Button"]) + XCTAssertEqual(raw.map(\.depth), [0, 1, 2, 3, 2]) + XCTAssertEqual(raw.map(\.parentIndex), [nil, 0, 1, 2, 1]) + XCTAssertEqual(raw.compactMap(\.label), ["Element", "Profile picture", "Theme"]) + // The offscreen row is a reported fact in raw, and reported facts do not become hittable + // just because the projection kept them. + XCTAssertEqual(raw.last?.hittable, false) + XCTAssertTrue(raw.allSatisfy { $0.hiddenContentAbove == nil && $0.hiddenContentBelow == nil }) + + let rawKeys = Set(raw.map { "\($0.type)-\($0.label ?? "")-\($0.rect.y)" }) + for node in regular { + XCTAssertTrue( + rawKeys.contains("\(node.type)-\(node.label ?? "")-\(node.rect.y)"), + "regular node \(node.type)/\(node.label ?? "") is missing from the raw projection" + ) + } + XCTAssertGreaterThan(raw.count, regular.count) + } + + /// Raw depth is traversal depth, so a raw `--depth` request is the one narrowing this backend + /// can prove complete. + func testPrivateAXRawProjectionAppliesRequestedTraversalDepth() { + let raw = privateAXNormalizedAcquisition(rawRoot: Self.privateAXScrolledFixture, + hint: CaptureHint( + projection: .raw, depth: 2, regularPresentedDepth: nil, + interactiveOnly: false, customActions: false), + viewport: CGRect(x: 0, y: 0, width: 402, height: 874), + interfaceOrientation: RunnerInterfaceOrientation.portrait) + XCTAssertEqual(raw.map(\.type), ["Application", "ScrollView", "Button", "Button"]) + XCTAssertEqual(raw.map(\.depth), [0, 1, 2, 2]) + } + + func testPrivateAXPresentationKeepsOffscreenSubtreeExcludedWhenChildFramesAreClamped() throws { + let frame = Self.privateAXFrame + let root: [String: Any] = ["type": Int(XCUIElement.ElementType.application.rawValue), + "label": "Element", "frame": frame(0, 0, 402, 874), "children": [[ + "type": Int(XCUIElement.ElementType.table.rawValue), "frame": frame(0, 96, 402, 700), + "children": [["type": Int(XCUIElement.ElementType.cell.rawValue), + "label": "Theme", "frame": frame(0, 900, 402, 44), "children": [[ + "type": Int(XCUIElement.ElementType.staticText.rawValue), + "label": "Theme", "frame": frame(16, 96, 120, 44)], + ["type": Int(XCUIElement.ElementType.switch.rawValue), + "label": "Theme", "frame": frame(340, 96, 46, 44)]]]]]]] + + let nodes = try privateAXRegularPresentation( + rawRoot: root, viewport: CGRect(x: 0, y: 0, width: 402, height: 874)) + + XCTAssertEqual(nodes.compactMap(\.label), ["Element"]) + XCTAssertEqual(nodes.first { $0.type == "Table" }?.hiddenContentBelow, true) + } + + func testPrivateAXGeometrylessSemanticsAreNeverActionableOrScrollContexts() throws { + let zero = ["x": 0, "y": 0, "width": 0, "height": 0] + let root: [String: Any] = ["type": Int(XCUIElement.ElementType.application.rawValue), + "label": "Element", "frame": ["x": 0, "y": 0, "width": 402, "height": 874], + "children": [["type": Int(XCUIElement.ElementType.scrollView.rawValue), + "label": "Settings semantics", "frame": zero, "children": [[ + "type": Int(XCUIElement.ElementType.button.rawValue), "label": "Theme", "frame": zero], + ["type": Int(XCUIElement.ElementType.other.rawValue), "frame": zero]]]]] + let nodes = try privateAXRegularPresentation( + rawRoot: root, viewport: CGRect(x: 0, y: 0, width: 402, height: 874), + interactiveOnly: true) + XCTAssertEqual(nodes.compactMap(\.label), ["Element", "Settings semantics", "Theme"]) + XCTAssertEqual(nodes.filter { $0.index != 0 }.map(\.hittable), [false, false]) + XCTAssertFalse(nodes.contains { $0.type == "Other" }) + XCTAssertTrue(nodes.allSatisfy { $0.hiddenContentAbove == nil && $0.hiddenContentBelow == nil }) + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScrollGestureTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScrollGestureTests.swift new file mode 100644 index 0000000000..9e9b9d26fc --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScrollGestureTests.swift @@ -0,0 +1,197 @@ +import XCTest + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +private struct ScrollGestureFixture: Decodable { + struct Constants: Decodable { + let defaultIosScrollAmount: Double + let defaultMobileScrollDurationMs: Double + let defaultIosScrollDurationMs: Double + let defaultScrollAmount: Double + let defaultEdgePaddingFraction: Double + let ordinaryScrollReleaseBehavior: String + let edgeScrollReleaseBehavior: String + } + struct Expected: Decodable { + let x1: Double + let y1: Double + let x2: Double + let y2: Double + let pixels: Double + } + struct Case: Decodable { + let name: String + let direction: String + let amount: Double? + let pixels: Double? + let referenceWidth: Double + let referenceHeight: Double + let expected: Expected + } + + let constants: Constants + let cases: [Case] +} + +extension RunnerTests { + // Cross-language parity table: every case in contracts/fixtures/scroll-gesture.json must agree + // with the vitest twin (packages/contracts/src/scroll-gesture.test.ts). Add vectors there, + // never fork the math. + private func loadScrollGestureFixture() throws -> ScrollGestureFixture { + let fixtureURL = URL(fileURLWithPath: #filePath) + .deletingLastPathComponent() // UnitTests + .deletingLastPathComponent() // AgentDeviceRunnerUITests + .deletingLastPathComponent() // AgentDeviceRunner + .deletingLastPathComponent() // runner + .deletingLastPathComponent() // apple + .deletingLastPathComponent() // repo root + .appendingPathComponent("contracts") + .appendingPathComponent("fixtures") + .appendingPathComponent("scroll-gesture.json") + return try JSONDecoder().decode(ScrollGestureFixture.self, from: Data(contentsOf: fixtureURL)) + } + + func testRunnerScrollGesturePlanMatchesParityTable() throws { + let fixture = try loadScrollGestureFixture() + XCTAssertFalse(fixture.cases.isEmpty, "parity table must not be empty") + for testCase in fixture.cases { + let plan = try XCTUnwrap( + runnerScrollGesturePlan( + direction: try XCTUnwrap(RunnerScrollDirection(rawValue: testCase.direction)), + amount: testCase.amount, + pixels: testCase.pixels, + referenceWidth: testCase.referenceWidth, + referenceHeight: testCase.referenceHeight + ), + testCase.name + ) + XCTAssertEqual(plan.x1, testCase.expected.x1, testCase.name) + XCTAssertEqual(plan.y1, testCase.expected.y1, testCase.name) + XCTAssertEqual(plan.x2, testCase.expected.x2, testCase.name) + XCTAssertEqual(plan.y2, testCase.expected.y2, testCase.name) + XCTAssertEqual(plan.travelPixels, testCase.expected.pixels, testCase.name) + } + } + + // The planner constants are private on both sides; the table pins them behaviourally on a + // 1000px axis where every rounding step is exact. + func testRunnerScrollGesturePlanUsesParityTableConstants() throws { + let constants = try loadScrollGestureFixture().constants + let defaultScroll = try JSONDecoder().decode( + Command.self, from: Data(#"{"command":"scroll"}"#.utf8)) + let defaults = runnerDragCommandDefaults(defaultScroll) + XCTAssertEqual(defaults.durationMs, constants.defaultIosScrollDurationMs) + XCTAssertEqual(defaults.scrollAmount, constants.defaultIosScrollAmount) + let defaulted = try XCTUnwrap( + runnerScrollGesturePlan( + direction: .down, amount: nil, pixels: nil, referenceWidth: 1000, referenceHeight: 1000 + ) + ) + XCTAssertEqual(defaulted.travelPixels, 1000 * constants.defaultScrollAmount) + let saturated = try XCTUnwrap( + runnerScrollGesturePlan( + direction: .down, amount: 10, pixels: nil, referenceWidth: 1000, referenceHeight: 1000 + ) + ) + XCTAssertEqual( + saturated.travelPixels, 1000 - 2 * 1000 * constants.defaultEdgePaddingFraction) + } + + func testRunnerScrollAndDragCommandDefaultsStayDistinct() throws { + func command(_ json: String) throws -> Command { + try JSONDecoder().decode(Command.self, from: Data(json.utf8)) + } + + let pixelScroll = runnerDragCommandDefaults( + try command(#"{"command":"scroll","pixels":120}"#)) + XCTAssertNil(pixelScroll.scrollAmount) + XCTAssertEqual(pixelScroll.durationMs, 400) + + let drag = runnerDragCommandDefaults(try command(#"{"command":"drag"}"#)) + XCTAssertNil(drag.scrollAmount) + XCTAssertEqual(drag.durationMs, 250) + + let explicitScroll = runnerDragCommandDefaults( + try command(#"{"command":"scroll","amount":0.5,"durationMs":125}"#)) + XCTAssertEqual(explicitScroll.scrollAmount, 0.5) + XCTAssertEqual(explicitScroll.durationMs, 125) + + let explicitDrag = runnerDragCommandDefaults( + try command(#"{"command":"drag","durationMs":125}"#)) + XCTAssertEqual(explicitDrag.durationMs, 125) + } + + func testRunnerScrollReleaseBehaviorSelectsTheDragProfile() throws { + let constants = try loadScrollGestureFixture().constants + let controlled = try XCTUnwrap(ScrollReleaseBehavior(rawValue: constants.ordinaryScrollReleaseBehavior)) + let inertial = try XCTUnwrap(ScrollReleaseBehavior(rawValue: constants.edgeScrollReleaseBehavior)) + XCTAssertEqual( + scrollDragProfile(releaseBehavior: nil), + .controlledScroll + ) + XCTAssertEqual( + scrollDragProfile(releaseBehavior: controlled), + .controlledScroll + ) + XCTAssertEqual( + scrollDragProfile(releaseBehavior: inertial), + .fastSwipe + ) + } + + func testControlledScrollProfileUsesReliableCadenceAndMonotonicDeceleration() { + XCTAssertEqual(RunnerControlledScrollFrameCount(350), 21) + XCTAssertEqual(RunnerControlledScrollFrameCount(400), 24) + XCTAssertEqual(RunnerControlledScrollFrameCount(500), 30) + XCTAssertEqual(RunnerControlledScrollFrameCount(1_000), 30) + XCTAssertEqual(RunnerControlledScrollFrameCount(10_000), 30) + + let frameCount = RunnerControlledScrollFrameCount(400) + let progress = (0...frameCount).map { + RunnerControlledScrollProgress(Double($0) / Double(frameCount)) + } + let deltas = zip(progress.dropFirst(), progress).map { $0.0 - $0.1 } + XCTAssertEqual(progress.first, 0) + XCTAssertEqual(progress.last, 1) + XCTAssertTrue(zip(deltas, deltas.dropFirst()).allSatisfy { $0.1 <= $0.0 }) + + let iPhoneViewportPoints = 874.0 + let defaultFingerTravel = iPhoneViewportPoints * 0.65 + let finalSampleTravel = (1 - progress[progress.count - 2]) * defaultFingerTravel + XCTAssertLessThan(finalSampleTravel, 0.1) + } + + func testRunnerScrollGesturePlanRejectsUnknownDirection() { + XCTAssertNil(RunnerScrollDirection(rawValue: "sideways")) + } + + func testRunnerScrollGesturePlanRejectsInvalidAmountAndPixels() { + XCTAssertNil( + runnerScrollGesturePlan( + direction: .down, + amount: 0, + pixels: nil, + referenceWidth: 300, + referenceHeight: 600 + ) + ) + XCTAssertNil( + runnerScrollGesturePlan( + direction: .down, + amount: nil, + pixels: -10, + referenceWidth: 300, + referenceHeight: 600 + ) + ) + XCTAssertNil( + runnerScrollGesturePlan( + direction: .down, + amount: .infinity, + pixels: nil, + referenceWidth: 300, + referenceHeight: 600 + ) + ) + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScrollViewportPolicyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScrollViewportPolicyTests.swift new file mode 100644 index 0000000000..208397cc95 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScrollViewportPolicyTests.swift @@ -0,0 +1,165 @@ +import XCTest +import AgentDeviceSnapshotPresentation + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +private struct ScrollViewportPolicyFixture: Decodable { + struct Frame: Decodable { + let x: Double + let y: Double + let width: Double + let height: Double + + var cgRect: CGRect { + CGRect(x: x, y: y, width: width, height: height) + } + } + + struct Constants: Decodable { + let minVisibleFraction: Double + let accessoryAllowance: Double + } + + struct Expected: Decodable { + let kind: String + let viewport: Frame? + let keyboardMinY: Double? + let visibleHeight: Double? + } + + struct TestCase: Decodable { + let name: String + let viewport: Frame + let keyboard: Frame + let expected: Expected + } + + let constants: Constants + let cases: [TestCase] +} + +extension RunnerTests { + /// Golden parity table (#2500): every case in contracts/fixtures/scroll-keyboard-policy.json must + /// agree with the vitest twin. Add cases there, never fork the rule. + func testScrollViewportKeyboardClipMatchesGoldenParityTable() throws { + let fixture = try loadScrollViewportPolicyFixture() + XCTAssertFalse(fixture.cases.isEmpty, "parity table must not be empty") + for testCase in fixture.cases { + let clip = ScrollViewportPolicy.clip( + viewport: testCase.viewport.cgRect, + keyboard: testCase.keyboard.cgRect + ) + switch testCase.expected.kind { + case "unobstructed": + XCTAssertEqual(clip, .unobstructed, testCase.name) + case "avoided": + let expectedFrame = try XCTUnwrap(testCase.expected.viewport, testCase.name).cgRect + let expectedMinY = try XCTUnwrap(testCase.expected.keyboardMinY, testCase.name) + XCTAssertEqual( + clip, + .avoided(frame: expectedFrame, keyboardMinY: expectedMinY), + testCase.name + ) + case "occluded": + let expectedMinY = try XCTUnwrap(testCase.expected.keyboardMinY, testCase.name) + let expectedVisibleHeight = try XCTUnwrap(testCase.expected.visibleHeight, testCase.name) + XCTAssertEqual( + clip, + .occluded(keyboardMinY: expectedMinY, visibleHeight: expectedVisibleHeight), + testCase.name + ) + default: + XCTFail("unknown expected kind `\(testCase.expected.kind)` in \(testCase.name)") + } + } + } + + /// The thresholds are the table's, not RunnerScrollViewportPolicy.swift's. The refusal reason + /// and the runner code are each one side's own vocabulary: the reason is what the host publishes, + /// the code is what this runner answers with, and neither is a shared clip constant. + func testScrollViewportPolicyUsesParityTableConstants() throws { + let constants = try loadScrollViewportPolicyFixture().constants + XCTAssertEqual(constants.minVisibleFraction, ScrollViewportPolicy.minVisibleFraction) + XCTAssertEqual(constants.accessoryAllowance, ScrollViewportPolicy.accessoryAllowance) + } + + /// A clipped landscape band shortens the frame, and `CoordinateSpaceRotation.native(point:)` derives a + /// `landscapeRight` native x from the frame's HEIGHT. Rotating inside the band therefore moves the + /// dispatched path sideways by exactly what the keyboard took, off the lane the plan was built for, + /// so the plan band and the coordinate basis stay separate values through dispatch (#2500). + func testScrollViewportDispatchKeepsTheUnclippedFrameAsItsCoordinateRotationBasis() throws { + let viewport = CGRect(x: 0, y: 0, width: 1210, height: 834) + let keyboard = CGRect(x: 0, y: 588, width: 1210, height: 246) + let clip = ScrollViewportPolicy.clip(viewport: viewport, keyboard: keyboard) + guard case .avoided(let band, let keyboardMinY) = clip else { + return XCTFail("expected a landscape keyboard to be avoided, got \(clip)") + } + XCTAssertEqual(band.height, 576) + + guard case .gesture(let gesture) = ScrollViewportPolicy.frames( + referenceFrame: viewport, + clip: clip + ).gestureDispatch(direction: .up, amount: nil, pixels: nil) else { + return XCTFail("expected a gesture inside the clipped band") + } + XCTAssertEqual(gesture.planFrame, band) + XCTAssertEqual(gesture.keyboardMinY, keyboardMinY) + XCTAssertEqual(gesture.coordinateFrame, viewport, "the rotation basis must survive the clip") + XCTAssertLessThanOrEqual( + max(gesture.plan.y1, gesture.plan.y2), + keyboard.minY - ScrollViewportPolicy.accessoryAllowance, + "a landscape swipe must stay clear of the keys" + ) + + let reported = gesture.attachingEvidence( + to: Response( + ok: true, + data: DataPayload(referenceWidth: viewport.width, referenceHeight: viewport.height), + error: nil + ) + ) + XCTAssertEqual( + reported.data?.referenceHeight, + band.height, + "the payload names the band the plan ran inside, not the synthesis frame" + ) + XCTAssertEqual(reported.data?.referenceWidth, viewport.width) + XCTAssertEqual(reported.data?.keyboardMinY, keyboardMinY) + XCTAssertEqual(reported.data?.keyboardAvoided, true) + + let orientedStartY = gesture.planFrame.minY + gesture.plan.y1 + let dispatchedFromViewport = CoordinateSpaceRotation.native( + point: CGPoint(x: gesture.planFrame.minX + gesture.plan.x1, y: orientedStartY), + in: gesture.coordinateFrame, + interfaceOrientation: RunnerInterfaceOrientation.landscapeRight + ) + let dispatchedFromBand = CoordinateSpaceRotation.native( + point: CGPoint(x: gesture.planFrame.minX + gesture.plan.x1, y: orientedStartY), + in: gesture.planFrame, + interfaceOrientation: RunnerInterfaceOrientation.landscapeRight + ) + XCTAssertEqual( + dispatchedFromViewport.x - dispatchedFromBand.x, + viewport.height - band.height, + accuracy: 0.001, + "rotating inside the clipped band would shift native x by what the keyboard took" + ) + } + + private func loadScrollViewportPolicyFixture() throws -> ScrollViewportPolicyFixture { + let fixtureURL = URL(fileURLWithPath: #filePath) + .deletingLastPathComponent() // UnitTests + .deletingLastPathComponent() // AgentDeviceRunnerUITests + .deletingLastPathComponent() // AgentDeviceRunner + .deletingLastPathComponent() // runner + .deletingLastPathComponent() // apple + .deletingLastPathComponent() // repo root + .appendingPathComponent("contracts") + .appendingPathComponent("fixtures") + .appendingPathComponent("scroll-keyboard-policy.json") + return try JSONDecoder().decode( + ScrollViewportPolicyFixture.self, + from: Data(contentsOf: fixtureURL) + ) + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SequenceExecutionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SequenceExecutionTests.swift new file mode 100644 index 0000000000..a2e0327df6 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SequenceExecutionTests.swift @@ -0,0 +1,186 @@ +import XCTest + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +// MARK: - In-bundle unit tests (device-free) + +extension RunnerTests { + func testSequenceDecodesStepsFromWire() throws { + let json = """ + {"command":"sequence","commandId":"seq-1","steps":[ + {"kind":"tap","x":100,"y":200}, + {"kind":"doubleTap","x":101,"y":200}, + {"kind":"longPress","x":102,"y":200,"durationMs":300,"pauseMs":50} + ]} + """ + let command = try JSONDecoder().decode(Command.self, from: Data(json.utf8)) + XCTAssertEqual(command.command, .sequence) + XCTAssertEqual(command.steps?.count, 3) + XCTAssertEqual(command.steps?[0].kind, "tap") + XCTAssertEqual(command.steps?[1].kind, "doubleTap") + XCTAssertEqual(command.steps?[2].pauseMs, 50) + } + + func testSequenceAcceptsDoubleTapKind() { + // A doubleTap step missing coords must fail on the coords check, not the kind allowlist — + // proving "doubleTap" passes validateSequenceStep without needing a device to execute on. + let response = executeSequenceForTest(steps: [ + sequenceStep(kind: "doubleTap", x: nil) + ]) + XCTAssertEqual(response.ok, false) + XCTAssertEqual(response.error?.code, "INVALID_ARGS") + XCTAssertTrue(response.error?.message.contains("requires finite x and y") ?? false) + XCTAssertFalse(response.error?.message.contains("unsupported kind") ?? true) + } + + func testSequenceRejectsUnknownKind() throws { + let response = executeSequenceForTest(steps: [ + sequenceStep(kind: "tap", x: 1, y: 2), + sequenceStep(kind: "pinch", x: 3, y: 4), + ]) + XCTAssertEqual(response.ok, false) + XCTAssertEqual(response.error?.code, "INVALID_ARGS") + XCTAssertTrue(response.error?.message.contains("step 1") ?? false) + XCTAssertTrue(response.error?.message.contains("pinch") ?? false) + } + + func testSequenceRejectsEmpty() { + let response = executeSequenceForTest(steps: []) + XCTAssertEqual(response.ok, false) + XCTAssertEqual(response.error?.code, "INVALID_ARGS") + } + + func testSequenceRejectsTooManySteps() { + let steps = (0..<21).map { _ in sequenceStep(kind: "tap", x: 1, y: 2) } + let response = executeSequenceForTest(steps: steps) + XCTAssertEqual(response.ok, false) + XCTAssertEqual(response.error?.code, "INVALID_ARGS") + XCTAssertTrue(response.error?.message.contains("at most 20") ?? false) + } + + func testSequenceHasSynthesizedCoordinateStep() { + XCTAssertTrue( + sequenceHasSynthesizedCoordinateStep([ + sequenceStep(kind: "tap", x: 1, y: 2, synthesized: true), + ]) + ) + XCTAssertFalse( + sequenceHasSynthesizedCoordinateStep([ + sequenceStep(kind: "tap", x: 1, y: 2), + sequenceStep(kind: "doubleTap", x: 1, y: 2, synthesized: true), + ]) + ) + } + + func testAssembleSequencePreservesOrderOnSuccess() { + let steps = [ + sequenceStep(kind: "tap", x: 1, y: 1), + sequenceStep(kind: "longPress", x: 2, y: 2), + sequenceStep(kind: "tap", x: 3, y: 3), + ] + var calls: [Int] = [] + let execution = assembleSequenceExecution(steps: steps) { index, _ in + calls.append(index) + return SequenceStepOutcome( + outcome: .performed, + gestureStartUptimeMs: Double(index * 10), + gestureEndUptimeMs: Double(index * 10 + 5) + ) + } + XCTAssertEqual(calls, [0, 1, 2]) + XCTAssertEqual(execution.completedSteps, 3) + XCTAssertNil(execution.failedStepIndex) + XCTAssertEqual(execution.results.map { $0.kind }, ["tap", "longPress", "tap"]) + XCTAssertEqual(execution.gestureStartUptimeMs, 0) + XCTAssertEqual(execution.gestureEndUptimeMs, 25) + } + + func testAssembleSequenceStopsAtFirstFailure() { + let steps = [ + sequenceStep(kind: "tap", x: 1, y: 1), + sequenceStep(kind: "longPress", x: 2, y: 2), + sequenceStep(kind: "tap", x: 3, y: 3), + ] + var calls: [Int] = [] + let execution = assembleSequenceExecution(steps: steps) { index, _ in + calls.append(index) + if index == 1 { + return SequenceStepOutcome( + outcome: .unsupported(message: "long press unsupported", hint: nil), + gestureStartUptimeMs: 10, + gestureEndUptimeMs: 15 + ) + } + return SequenceStepOutcome(outcome: .performed, gestureStartUptimeMs: 0, gestureEndUptimeMs: 5) + } + // Step 2 is never invoked. + XCTAssertEqual(calls, [0, 1]) + XCTAssertEqual(execution.completedSteps, 1) + XCTAssertEqual(execution.failedStepIndex, 1) + // results.count == completedSteps + 1 (the failed step). + XCTAssertEqual(execution.results.count, 2) + XCTAssertEqual(execution.results[1].ok, false) + XCTAssertEqual(execution.results[1].errorCode, "UNSUPPORTED_OPERATION") + XCTAssertEqual(execution.results[1].errorMessage, "long press unsupported") + } + + func testSequenceWorstCaseResponseStaysUnderJournalCap() throws { + let longMessage = String(repeating: "e", count: 200) + let results = (0..<20).map { index in + SequenceStepResult( + ok: index < 19, + kind: "longPress", + errorCode: index < 19 ? nil : "UNSUPPORTED_OPERATION", + errorMessage: index < 19 ? nil : longMessage, + gestureStartUptimeMs: 123456.789, + gestureEndUptimeMs: 123466.789 + ) + } + let response = Response( + ok: true, + data: DataPayload( + message: "sequence", + completedSteps: 19, + failedStepIndex: 19, + sequenceResults: results + ) + ) + let encoded = try JSONEncoder().encode(response) + XCTAssertLessThan(encoded.count, 16 * 1024) + } + + private func sequenceStep( + kind: String, + x: Double?, + y: Double? = nil, + synthesized: Bool? = nil + ) -> SequenceStep { + SequenceStep( + kind: kind, + x: x, + y: y, + durationMs: nil, + pauseMs: nil, + synthesized: synthesized + ) + } + + /// Validation runs before any executor call, so the INVALID_ARGS paths are exercised without + /// reaching the device executor (which is never invoked when validation rejects). + private func executeSequenceForTest(steps: [SequenceStep]) -> Response { + let command = makeSequenceCommand(steps: steps) + return executeSequence(command: command, activeApp: app) + } + + /// Build a sequence Command via JSON so the test does not depend on the memberwise init's + /// parameter order. + private func makeSequenceCommand(steps: [SequenceStep]) -> Command { + struct SequenceCommandFixture: Encodable { + let command = "sequence" + let commandId = "seq-test" + let steps: [SequenceStep] + } + let data = try! JSONEncoder().encode(SequenceCommandFixture(steps: steps)) + return try! JSONDecoder().decode(Command.self, from: data) + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotBackendCapabilitiesTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotBackendCapabilitiesTests.swift new file mode 100644 index 0000000000..a56627eed9 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotBackendCapabilitiesTests.swift @@ -0,0 +1,80 @@ +import XCTest + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +private struct SnapshotBackendParityFixture: Decodable { + struct Availability: Decodable { + let simulator: Bool + let physicalDevice: Bool + } + + struct Backend: Decodable { + let name: String + let forceable: Bool + let supportsRawProjection: Bool + let regularDepth: String + let hittable: String + let availability: Availability + } + + let backends: [Backend] +} + +extension RunnerTests { + private func loadSnapshotBackendParityFixture() throws -> SnapshotBackendParityFixture { + let fixtureURL = URL(fileURLWithPath: #filePath) + .deletingLastPathComponent() // UnitTests + .deletingLastPathComponent() // AgentDeviceRunnerUITests + .deletingLastPathComponent() // AgentDeviceRunner + .deletingLastPathComponent() // runner + .deletingLastPathComponent() // apple + .deletingLastPathComponent() // repo root + .appendingPathComponent("contracts") + .appendingPathComponent("fixtures") + .appendingPathComponent("ios-snapshot-backends.json") + return try JSONDecoder().decode( + SnapshotBackendParityFixture.self, + from: Data(contentsOf: fixtureURL) + ) + } + + /// The JSON table is the cross-runtime declaration used by the TypeScript capability registry + /// and this runner. A backend case, forceability branch, projection/depth claim, or availability + /// change that is not classified in both implementations fails before an iOS smoke can drift. + func testSnapshotBackendDeclarationsMatchCapabilityFixture() throws { + let fixture = try loadSnapshotBackendParityFixture() + XCTAssertEqual( + fixture.backends.map(\.name), + SnapshotBackendKind.allCases.map(\.rawValue) + ) + + for expected in fixture.backends { + guard let backend = SnapshotBackendKind(rawValue: expected.name) else { + XCTFail("fixture contains an unknown snapshot backend: \(expected.name)") + continue + } + XCTAssertEqual(backend.isForceable, expected.forceable, expected.name) + XCTAssertEqual(backend.supportsRawProjection, expected.supportsRawProjection, expected.name) + XCTAssertEqual( + backend.regularDepthCapability.rawValue, + expected.regularDepth, + "regular depth capability: \(expected.name)" + ) + XCTAssertEqual( + backend.hittableSemantics, + expected.hittable, + "hittable semantics: \(expected.name)" + ) + XCTAssertEqual( + backend.isAvailable(on: .simulator), + expected.availability.simulator, + "simulator availability: \(expected.name)" + ) + XCTAssertEqual( + backend.isAvailable(on: .physicalDevice), + expected.availability.physicalDevice, + "physical-device availability: \(expected.name)" + ) + } + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanTests.swift new file mode 100644 index 0000000000..f0eae25e74 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanTests.swift @@ -0,0 +1,492 @@ +import XCTest +import AgentDeviceSnapshotPresentation + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +// MARK: - In-bundle unit tests + +extension RunnerTests { + private func planTestNode( + index: Int, + type: String, + label: String? = nil, + identifier: String? = nil, + hittable: Bool = false, + parentIndex: Int? = nil + ) -> PresentedNode { + SnapshotPresentation.singleElementRead( + RawAXNode( + index: index, + type: type, + label: label, + identifier: identifier, + value: nil, + rect: SnapshotRect(.zero), + enabled: true, + focused: nil, + selected: nil, + hittable: hittable, + depth: parentIndex == nil ? 0 : 1, + parentIndex: parentIndex, + hiddenContentAbove: nil, + hiddenContentBelow: nil + ) + ) + } + + func testSparsePayloadReasonMatrix() { + let root = planTestNode(index: 0, type: "Application", label: "Example App", hittable: true) + let window = planTestNode(index: 1, type: "Window", parentIndex: 0) + let button = planTestNode(index: 1, type: "Button", label: "Ok", hittable: true, parentIndex: 0) + let shell = planTestNode( + index: 1, + type: "Other", + identifier: "appShell", + parentIndex: 0 + ) + let serializationPlaceholder = planTestNode( + index: 2, + type: "Other", + label: "[object Object]", + parentIndex: 1 + ) + + // Labeled, hittable root over a bare window is still sparse. + XCTAssertNotNil(Self.sparsePayloadReason(DataPayload(nodes: [root, window], truncated: false))) + // Deadline-truncated near-empty sweep needs recovery even with one real control. + XCTAssertNotNil(Self.sparsePayloadReason(DataPayload(nodes: [root, button], truncated: true))) + // The same tiny tree from a completed sweep is a legitimately minimal screen. + XCTAssertNil(Self.sparsePayloadReason(DataPayload(nodes: [root, button], truncated: false))) + // Container metadata plus a stringified serialization placeholder is not readable UI. + XCTAssertNotNil( + Self.sparsePayloadReason( + DataPayload(nodes: [root, shell, serializationPlaceholder], truncated: false) + ) + ) + let actionableShell = planTestNode( + index: 1, + type: "Other", + identifier: "checkout", + hittable: true, + parentIndex: 0 + ) + XCTAssertNil( + Self.sparsePayloadReason(DataPayload(nodes: [root, actionableShell], truncated: false)) + ) + // Empty payloads are degraded. + XCTAssertNotNil(Self.sparsePayloadReason(DataPayload(nodes: [], truncated: false))) + } + + func testCollapsedLeafIndexesFlagsMergedContainersOnly() { + let root = planTestNode(index: 0, type: "Application", label: "App") + let merged = planTestNode( + index: 1, + type: "Other", + label: (0...30).map { "Row \($0), Tap" }.joined(separator: ", "), + parentIndex: 0 + ) + let prose = planTestNode( + index: 2, + type: "StaticText", + label: (0...30).map { "clause \($0)" }.joined(separator: ", "), + parentIndex: 0 + ) + XCTAssertEqual(Self.collapsedLeafIndexes([root, merged, prose]), [1]) + XCTAssertNil(Self.collapsedLeafIndexes([root, prose])) + } + + func testTerminalFailsClosedOnInteractiveAxFailureRegardlessOfSparseBest() { + // Interactive AX failure must invalidate + fail closed; a later tier's sparse synthetic-root + // "best" must never downgrade this to a returned-sparse payload (regression: best == nil guard). + XCTAssertEqual( + Self.resolveSnapshotPlanTerminal(terminal: .sparseWithFatalOnAXFailure, interactiveOnly: true), + .failClosed + ) + XCTAssertEqual( + Self.resolveSnapshotPlanTerminal(terminal: .sparseWithFatalOnAXFailure, interactiveOnly: false), + .sparseBest + ) + XCTAssertEqual( + Self.resolveSnapshotPlanTerminal(terminal: .throwOnAXFailure, interactiveOnly: true), + .throwAxFailure + ) + } + + func testXCTestChannelStateFirstFailureStampsDeferredCodeOnlyForDeferral() { + XCTAssertNil(Self.xcTestChannelStateFirstFailure(.normal)) + XCTAssertEqual(Self.xcTestChannelStateFirstFailure(.deferredToIndependentBackend)?.code, "deferred") + XCTAssertEqual(Self.xcTestChannelStateFirstFailure(.boundedXCTestProbe)?.code, "budget") + } + + func testSnapshotQualityCarriesPhaseTimingAtResponseLevel() { + let timing = SnapshotCaptureTiming(acquisitionMs: 12, presentationMs: 34) + let capture = SnapshotBackendCapture( + payload: DataPayload( + nodes: [planTestNode(index: 0, type: "Application", label: "App")], + truncated: false + ), + effectiveDepth: nil, + timing: timing + ) + + let payload = stampedSnapshotPayload( + capture, + backend: .recursiveTree, + state: "healthy", + reason: nil + ) + + XCTAssertEqual(payload.snapshotQuality?.timing, timing) + XCTAssertEqual(payload.nodes?.count, 1) + } + + func testStampedPayloadCarriesDisclosuresOnlyInTheVerdict() { + let root = planTestNode(index: 0, type: "Application", label: "App") + let merged = planTestNode( + index: 1, + type: "Other", + label: (0...30).map { "Tab \($0)" }.joined(separator: ", "), + parentIndex: 0 + ) + let coverage = SnapshotCustomActionCoverage( + read: 12, candidates: 19, truncated: 0, blocked: false) + let silent = stampedSnapshotPayload( + SnapshotBackendCapture( + payload: DataPayload(nodes: [root, merged], truncated: false), + effectiveDepth: nil, + customActions: coverage + ), + backend: .recursiveTree, + state: "healthy", + reason: nil + ) + XCTAssertNil(silent.message) + XCTAssertEqual(silent.snapshotQuality?.customActions, coverage) + XCTAssertEqual(silent.snapshotQuality?.collapsedLeafIndexes, [1]) + + let underlying = stampedSnapshotPayload( + SnapshotBackendCapture( + payload: DataPayload(message: "underlying", nodes: [root], truncated: false), + effectiveDepth: 4 + ), + backend: .privateAX, + state: "recovered", + reason: (reason: "tree capture timed out", code: "budget") + ) + XCTAssertEqual(underlying.message, "underlying") + } + + func testStampedPayloadTruncationTracksCompletenessNotRecoveryProvenance() { + let complete = SnapshotBackendCapture( + payload: DataPayload( + nodes: [ + planTestNode(index: 0, type: "Application", label: "App"), + planTestNode(index: 1, type: "Button", label: "Open", parentIndex: 0), + ], + truncated: false + ), + effectiveDepth: nil + ) + let deferred: (reason: String, code: String) = ( + "XCTest-backed snapshot tiers were deferred after recent slow accessibility work", "deferred" + ) + + // The CI signature behind `is absent ... capture was truncated`: a complete private AX + // tree selected while the XCTest channel is penalized is whole, and must say so. + let recovered = stampedSnapshotPayload( + complete, backend: .privateAX, state: "recovered", reason: deferred) + XCTAssertEqual(recovered.snapshotQuality?.state, "recovered") + XCTAssertEqual(recovered.truncated, false) + + let depthLimited = stampedSnapshotPayload( + SnapshotBackendCapture(payload: complete.payload, effectiveDepth: 56), + backend: .privateAX, state: "recovered", reason: deferred) + XCTAssertEqual(depthLimited.truncated, true) + + let cappedPayload = stampedSnapshotPayload( + SnapshotBackendCapture( + payload: DataPayload(nodes: complete.payload.nodes ?? [], truncated: true), + effectiveDepth: nil), + backend: .recursiveTree, state: "healthy", reason: nil) + XCTAssertEqual(cappedPayload.truncated, true) + + let sparse = stampedSnapshotPayload( + complete, backend: .querySweep, state: "sparse", + reason: ("snapshot returned no semantic controls or content", "sparse-tree")) + XCTAssertEqual(sparse.truncated, true) + } + + func testSnapshotQualityCarriesUnscopedQualityPayload() { + let quality = DataPayload( + nodes: [planTestNode(index: 0, type: "Application", label: "App")], + truncated: false + ) + let capture = SnapshotBackendCapture( + payload: quality, + effectiveDepth: nil, + qualityPayload: quality + ) + + let payload = stampedSnapshotPayload( + capture, + backend: .recursiveTree, + state: "healthy", + reason: nil + ) + + XCTAssertEqual(payload.qualityPayload?.nodes.count, 1) + XCTAssertEqual(payload.qualityPayload?.truncated, false) + XCTAssertNil(payload.qualityPayload?.scope) + } + + func testDirectPresentationDoesNotClaimPlanTiming() { + let options = PresentationOptions( + interactiveOnly: false, + depth: nil, + scope: nil, + raw: true + ) + let result = SnapshotPresentation.presentRaw( + SnapshotAcquisition( + hint: SnapshotPresentation.captureHint(for: options), + nodes: [], + truncated: false, + effectiveDepth: nil, + viewport: .infinite + ), + options: options + ) + let capture = Self.makeSnapshotBackendCapture(from: result) + + let payload = stampedSnapshotPayload( + capture, + backend: .recursiveTree, + state: "healthy", + reason: nil + ) + + XCTAssertNil(payload.snapshotQuality?.timing) + } + + /// The raw plan is derived from what each backend can actually serve, not from a second + /// hand-maintained list. Non-vacuity: flipping `querySweep.supportsRawProjection` to true adds it + /// to the plan and fails the first two assertions — which is exactly the shape of #1797 D4, a + /// `--raw` request answered by a backend that has no hierarchy to return. + func testRawDiagnosticPlanCarriesOnlyBackendsThatCanServeRaw() { + XCTAssertEqual(Self.rawDiagnosticPlan, [.recursiveTree, .privateAX]) + XCTAssertEqual( + SnapshotBackendKind.allCases.filter { !$0.supportsRawProjection }, [.querySweep]) + XCTAssertTrue(Self.rawDiagnosticPlan.allSatisfy(\.supportsRawProjection)) + // Tree-first error propagation is the raw plan's other contract (ADR 0004). + XCTAssertEqual(Self.rawDiagnosticPlan.first, .recursiveTree) + } + + /// A projection mismatch is a runner bug, not an accessibility failure: it must not take the + /// AX-failure terminal route (rethrow / fail-closed), just drop its tier with a named reason. + func testProjectionMismatchFailureIsStructuredAndNotAnAxFailure() { + let failure = Self.snapshotProjectionMismatchFailure( + .querySweep, requested: .raw, acquired: .regular) + XCTAssertEqual(failure.code, "IOS_SNAPSHOT_PROJECTION_MISMATCH") + XCTAssertTrue(failure.message.contains("queries")) + XCTAssertTrue(failure.message.contains("raw")) + XCTAssertFalse(Self.isAxSnapshotFailure(failure)) + } + + /// #1634 P2: the decoded wire field must reach capture options and its + /// applicable plan. A pinned REGULAR capture defers to privateAX-first; the + /// RAW diagnostic plan is never rerouted by the pin — raw keeps tree-first + /// error propagation, which is exactly why raw baselines are excluded from + /// corroboration daemon-side. + func testDecodedPreferredBackendReachesOptionsAndApplicablePlan() throws { + let json = #"{"command":"snapshot","preferredBackend":"private-ax"}"# + let command = try JSONDecoder().decode(Command.self, from: Data(json.utf8)) + let options = Self.presentationOptions(from: command) + XCTAssertEqual(options.preferredBackend, "private-ax") + XCTAssertFalse(options.raw) + + let treated = Self.snapshotXCTestChannelTreatedAsPenalized( + penalized: false, preferredBackend: options.preferredBackend) + let pinned = Self.effectiveSnapshotCapturePlan( + Self.regularVisiblePlan, + xCTestChannelPenalized: treated, + preferredBackend: options.preferredBackend + ) + XCTAssertEqual(pinned.plan, [.privateAX]) + XCTAssertEqual(pinned.xCTestChannelState, .deferredToIndependentBackend) + + let raw = Self.effectiveSnapshotCapturePlan( + Self.rawDiagnosticPlan, + xCTestChannelPenalized: treated, + preferredBackend: options.preferredBackend + ) + XCTAssertEqual(raw.plan, Self.rawDiagnosticPlan) + + // A command without the field decodes to no pin and a normal plan. + let bare = try JSONDecoder().decode( + Command.self, from: Data(#"{"command":"snapshot"}"#.utf8)) + XCTAssertNil(Self.presentationOptions(from: bare).preferredBackend) + } + + /// #1635: the force seam must select the recursive tree even when the XCTest + /// channel is currently penalized. Without the preferred-backend argument, + /// this call returns the independent private-AX recovery plan instead. + func testPreferredTreeBackendPinsRegularPlanAndLeavesStructuredEvidence() { + let forced = Self.effectiveSnapshotCapturePlan( + Self.regularVisiblePlan, + xCTestChannelPenalized: true, + preferredBackend: SnapshotBackendKind.recursiveTree.rawValue + ) + XCTAssertEqual(forced.plan, [.recursiveTree]) + XCTAssertEqual(forced.xCTestChannelState, .normal) + XCTAssertEqual( + Self.xcTestChannelStateFirstFailure( + forced.xCTestChannelState, + preferredBackend: forced.preferredBackend?.rawValue + )?.code, + "requested-backend" + ) + } + + /// Same-backend evidence probes: a daemon-pinned private-AX capture takes the + /// penalized route even with a healthy channel, so tap-outcome corroboration + /// baselines and probes are always captured by the same backend (backends are + /// never comparable views of a screen). Composed with the plan rule, the pin + /// yields the privateAX-first deferred plan. + func testPreferredPrivateAXBackendPlansAsPenalized() { + XCTAssertTrue( + Self.snapshotXCTestChannelTreatedAsPenalized(penalized: false, preferredBackend: "private-ax")) + XCTAssertTrue( + Self.snapshotXCTestChannelTreatedAsPenalized(penalized: true, preferredBackend: nil)) + XCTAssertFalse( + Self.snapshotXCTestChannelTreatedAsPenalized(penalized: false, preferredBackend: nil)) + XCTAssertFalse( + Self.snapshotXCTestChannelTreatedAsPenalized(penalized: false, preferredBackend: "tree")) + + let pinned = Self.effectiveSnapshotCapturePlan( + Self.regularVisiblePlan, + xCTestChannelPenalized: Self.snapshotXCTestChannelTreatedAsPenalized( + penalized: false, preferredBackend: "private-ax" + ), + preferredBackend: "private-ax" + ) + XCTAssertEqual(pinned.plan, [.privateAX]) + XCTAssertEqual(pinned.xCTestChannelState, .deferredToIndependentBackend) + } + + func testEffectiveSnapshotCapturePlanDefersXCTestBackedTiersOnlyWhenPenalizedRegularPlan() { + let regular = Self.effectiveSnapshotCapturePlan( + Self.regularVisiblePlan, + xCTestChannelPenalized: true + ) + XCTAssertEqual(regular.plan, [.privateAX]) + XCTAssertEqual(regular.xCTestChannelState, .deferredToIndependentBackend) + XCTAssertNil(regular.treeCaptureSliceBudgetOverride) + + let unpenalized = Self.effectiveSnapshotCapturePlan( + Self.regularVisiblePlan, + xCTestChannelPenalized: false + ) + XCTAssertEqual(unpenalized.plan, Self.regularVisiblePlan) + XCTAssertEqual(unpenalized.xCTestChannelState, .normal) + XCTAssertNil(unpenalized.treeCaptureSliceBudgetOverride) + + // The raw diagnostic plan preserves tree-first error propagation even under penalty. + let raw = Self.effectiveSnapshotCapturePlan( + Self.rawDiagnosticPlan, + xCTestChannelPenalized: true + ) + XCTAssertEqual(raw.plan, Self.rawDiagnosticPlan) + XCTAssertEqual(raw.xCTestChannelState, .normal) + XCTAssertNil(raw.treeCaptureSliceBudgetOverride) + } + + func testEffectiveSnapshotCapturePlanUsesBoundedXCTestProbeWhenNoIndependentBackendRuns() { + let physicalDevicePlan = Self.effectiveSnapshotCapturePlan( + Self.regularVisiblePlan, + xCTestChannelPenalized: true, + availableBackends: [.recursiveTree, .querySweep] + ) + + XCTAssertEqual(physicalDevicePlan.plan, [.recursiveTree, .querySweep]) + XCTAssertEqual(physicalDevicePlan.xCTestChannelState, .boundedXCTestProbe) + XCTAssertEqual( + physicalDevicePlan.treeCaptureSliceBudgetOverride, + Self.penalizedXCTestProbeTreeSliceBudget + ) + } + + func testSnapshotXCTestChannelPenaltyMatchesBundleAndExpires() { + defer { + snapshotXCTestChannelPenaltyBundleId = nil + snapshotXCTestChannelPenaltyUntil = .distantPast + } + + penalizeSnapshotXCTestChannel(bundleId: "xyz.blueskyweb.app", reason: "test") + XCTAssertTrue(isSnapshotXCTestChannelPenalized(bundleId: "xyz.blueskyweb.app")) + XCTAssertFalse(isSnapshotXCTestChannelPenalized(bundleId: "com.other.app")) + + // A penalty recorded without a bundle applies to any current target. + penalizeSnapshotXCTestChannel(bundleId: nil, reason: "test") + XCTAssertTrue(isSnapshotXCTestChannelPenalized(bundleId: "com.other.app")) + + // Expired penalties stop applying. + snapshotXCTestChannelPenaltyUntil = Date(timeIntervalSinceNow: -1) + XCTAssertFalse(isSnapshotXCTestChannelPenalized(bundleId: "com.other.app")) + } + + func testAbandonedMainThreadWorkSkipsOnlyXCTestBackedSnapshotTiers() { + abandonedMainThreadWorkCount = 1 + defer { abandonedMainThreadWorkCount = 0 } + + XCTAssertTrue(shouldSkipSnapshotBackendForAbandonedMainThreadWork(.recursiveTree)) + XCTAssertTrue(shouldSkipSnapshotBackendForAbandonedMainThreadWork(.querySweep)) + XCTAssertFalse(shouldSkipSnapshotBackendForAbandonedMainThreadWork(.privateAX)) + } + +#if os(iOS) + /// #2403: a plan pinned to private AX serves a regular `--depth` request through acquisition + /// and presentation. With a backend depth gate in `captureWithBackend`, private AX returns no + /// capture, the plan falls through to the synthetic sparse root, and the daemon rejects that + /// zero-rect root as a missing viewport. + func testPrivateAXPinnedRegularDepthReachesAcquisitionAndPresentation() throws { + app.launchArguments = ["--agent-device-selector-read-regression"] + app.launch() + currentApp = app + currentBundleId = nil + defer { + currentApp = nil + clearPrivateAXAcceptedDepth(reason: "test-cleanup") + app.terminate() + } + func capture(depth: Int?) throws -> DataPayload { + try runSnapshotCapturePlan( + Self.regularVisiblePlan, + app: app, + options: PresentationOptions( + interactiveOnly: false, + depth: depth, + scope: nil, + raw: false, + preferredBackend: SnapshotBackendKind.privateAX.rawValue + ), + terminal: .sparseWithFatalOnAXFailure + ) + } + + let capped = try capture(depth: 1) + + let quality = try XCTUnwrap(capped.snapshotQuality) + XCTAssertEqual(quality.backend, SnapshotBackendKind.privateAX.rawValue) + XCTAssertNotEqual(quality.state, "sparse") + let nodes = try XCTUnwrap(capped.nodes) + XCTAssertGreaterThan(nodes.count, 1) + XCTAssertEqual(nodes.map(\.depth).max(), 1) + XCTAssertNotEqual(nodes[0].rect, SnapshotRect(x: 0, y: 0, width: 0, height: 0)) + XCTAssertTrue(nodes.contains { $0.label == "Readable target" }) + + // The presented cut only ever narrows the unscoped capture from the same backend. + let unscoped = try XCTUnwrap(try capture(depth: nil).nodes) + XCTAssertLessThanOrEqual(nodes.count, unscoped.count) + } +#endif +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotTests.swift new file mode 100644 index 0000000000..527da828fb --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotTests.swift @@ -0,0 +1,245 @@ +import XCTest +import AgentDeviceSnapshotPresentation + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +extension RunnerTests { + func testSnapshotAccessibilityUnavailableMarksSparseSnapshotRunnerFatal() { + currentApp = app + currentBundleId = "com.example.app" + + let payload = snapshotAccessibilityUnavailable( + failure: SnapshotCaptureFailure( + code: Self.axSnapshotErrorCode, + message: Self.axSnapshotFailureMessage, + hint: Self.axSnapshotHint + ) + ) + + XCTAssertEqual(payload.message, "\(Self.axSnapshotFailureMessage) Hint: \(Self.axSnapshotHint)") + XCTAssertEqual(payload.nodes?.count, 1) + XCTAssertEqual(payload.nodes?.first?.type, "Application") + XCTAssertEqual(payload.truncated, true) + XCTAssertEqual(payload.runnerFatal, true) + XCTAssertEqual(payload.runnerFatalReason, Self.axSnapshotUnavailableReason) + // The planned terminal result carries the structured verdict like every other planned + // snapshot — downstream sparse handling keys off it, not off node shapes. + XCTAssertEqual(payload.snapshotQuality?.state, "sparse") + XCTAssertEqual(payload.snapshotQuality?.reasonCode, "ax-rejected") + XCTAssertEqual(payload.snapshotQuality?.reason, Self.axSnapshotFailureMessage) + XCTAssertNil(currentApp) + XCTAssertNil(currentBundleId) + } + + func testRecoveredSnapshotMessagePreservesHint() { + let message = recoveredSnapshotMessage( + SnapshotCaptureFailure( + code: Self.axSnapshotErrorCode, + message: Self.axSnapshotFailureMessage, + hint: Self.axSnapshotHint + ) + ) + + XCTAssertTrue(message.contains(Self.axSnapshotFailureMessage)) + XCTAssertTrue(message.contains(Self.axSnapshotHint)) + } + + func testRawSnapshotTooLargeFailureIsStructured() { + let failure = rawSnapshotTooLargeFailure(nodeCount: Self.rawSnapshotMaxNodes + 1) + + XCTAssertEqual(failure.code, Self.rawSnapshotTooLargeCode) + XCTAssertTrue(failure.message.contains("\(Self.rawSnapshotMaxNodes) nodes")) + XCTAssertEqual(failure.hint, Self.rawSnapshotTooLargeHint) + } + + func testSystemModalProbeSliceSharesAndClampsToPlanDeadline() { + // Fresh plan deadline: the probe gets its full dedicated budget. + XCTAssertEqual(Self.systemModalProbeSlice(budget: 4, deadlineRemaining: 20), 4) + // Nearly-spent plan deadline: the probe is clamped so it can't run past the shared budget. + XCTAssertEqual(Self.systemModalProbeSlice(budget: 4, deadlineRemaining: 1.5), 1.5) + // Exactly/already exhausted deadline: skip the probe entirely (0), never a negative timeout. + XCTAssertEqual(Self.systemModalProbeSlice(budget: 4, deadlineRemaining: 0), 0) + XCTAssertEqual(Self.systemModalProbeSlice(budget: 4, deadlineRemaining: -5), 0) + } + + // Simulator-only: the bounded probe body returns nil on macOS (no SpringBoard host), so the + // timeout/penalty/drain machinery below only exists on the iOS branch. +#if os(iOS) + /// Regression for #1244/#1248: drives the bounded system-modal probe through a real, + /// production-only command entry point (`snapshotFast` or `snapshotRaw` -- see the two test + /// methods below), not `boundedBlockingSystemAlertSnapshot` directly, with + /// `systemModalProbeOverrideForTesting` set to a closure that blocks past the probe's real + /// slice, forcing a real `runMainThreadWork` timeout. This is revert-sensitive on both halves + /// of the fix, for either entry point: + /// - if the entry point reverted to calling the unbounded `blockingSystemAlertSnapshot` + /// directly (or dropped the `runMainThreadWork` wrap), nothing here would ever time out, + /// so the mid-flight busy/penalty assertions below would never be met; + /// - if the `onAbandoned` penalty hook or the abandoned-work accounting were dropped, the + /// timeout would still fire, but the busy/penalty and drain assertions would not hold. + /// + /// The drain assertion is synchronized on the *real* release rather than raced: after + /// signaling the probe to finish, the background queue polls `hasAbandonedMainThreadWork()` + /// (bounded) and only then fulfills `drained`, which the test `wait(for:timeout:)`s on before + /// asserting `.idle`/`hasAbandonedMainThreadWork() == false` below -- so a slow drain fails that + /// assertion instead of racing a fixed-timing guess. + private func assertBoundedSystemModalProbeTimeoutRecoversThenReleasesOnDrain( + entryPointName: String, + callEntryPoint: @escaping (XCUIApplication, PresentationOptions) throws -> DataPayload + ) { + let targetBundleId = "com.callstack.agentdevice.runner.missing.snapshot-timeout-test" + let snapshotTarget = XCUIApplication(bundleIdentifier: targetBundleId) + let probeReleaseGate = DispatchSemaphore(value: 0) + currentApp = snapshotTarget + currentBundleId = targetBundleId + defer { + probeReleaseGate.signal() + currentApp = nil + currentBundleId = nil + systemModalProbeOverrideForTesting = nil + clearSnapshotXCTestChannelPenalty(reason: "test-cleanup") + } + + final class ResultBox { + var payload: DataPayload? + var wasBusyBeforeDrain = false + var hadAbandonedCaptureBeforeDrain = false + var wasPenalizedBeforeDrain = false + } + let box = ResultBox() + // The test owns release of the injected probe. A fixed timeout races the capture plan's + // independent fallback tiers on loaded CI hosts and can drain before the test records the + // abandoned-work state. The defer above still releases the probe if an earlier assertion or + // expectation fails. + systemModalProbeOverrideForTesting = { _ in + probeReleaseGate.wait() + return nil + } + + let completion = expectation( + description: "\(entryPointName) recovered while the probe was abandoned, then released it" + ) + let drained = expectation(description: "\(entryPointName) modal probe drained") + DispatchQueue(label: "agent-device.runner.tests.modal-probe-timeout").async { + box.payload = try? callEntryPoint( + snapshotTarget, + PresentationOptions(interactiveOnly: false, depth: nil, scope: nil, raw: false) + ) + + // 1) Penalty/busy accounting: must already be in place by the time the entry point + // returns, well before we release the still-blocked probe below. + if case .busy = self.currentMainThreadBusyState() { + box.wasBusyBeforeDrain = true + } + box.hadAbandonedCaptureBeforeDrain = self.hasAbandonedMainThreadWork() + box.wasPenalizedBeforeDrain = self.isSnapshotXCTestChannelPenalized(bundleId: self.currentBundleId) + + // 2) `box.payload` above was already produced -- through the capture plan's recovery + // tiers -- while the probe is still blocked on `probeReleaseGate`, i.e. recovered before + // drain, not queued behind it. + completion.fulfill() + + // 3) Only now let the abandoned probe finish, then block this queue (never the test's + // main-thread wait) on the *real* drain signal -- the abandoned-work count reaching zero + // -- bounded so a revert that never drains fulfills `drained` anyway and lets the + // assertions below report the regression explicitly instead of just timing out. + probeReleaseGate.signal() + let drainDeadline = Date().addingTimeInterval(5) + while self.hasAbandonedMainThreadWork(), Date() < drainDeadline { + self.sleepFor(0.002) + } + drained.fulfill() + } + + wait(for: [completion], timeout: 15) + + // 1) Penalty/busy accounting. + XCTAssertTrue( + box.wasBusyBeforeDrain, + "expected RUNNER_BUSY while the \(entryPointName) modal probe timeout is outstanding" + ) + XCTAssertTrue( + box.hadAbandonedCaptureBeforeDrain, + "onAbandoned must retain the abandoned XCTest channel work for \(entryPointName)" + ) + XCTAssertTrue( + box.wasPenalizedBeforeDrain, + "a timed-out modal probe must penalize the XCTest snapshot channel for \(entryPointName)" + ) + + // 2) Recovered response before drain. + XCTAssertNotNil( + box.payload, + "\(entryPointName) must recover a payload through the capture plan while the probe drains" + ) + + // 3) Bounded, deterministic drain barrier, then release assertions. + wait(for: [drained], timeout: 6) + guard case .idle = currentMainThreadBusyState() else { + return XCTFail("expected the runner to be idle once the abandoned \(entryPointName) probe drained") + } + XCTAssertFalse( + hasAbandonedMainThreadWork(), + "the drained probe must release the main thread for \(entryPointName)" + ) + } + + func testBoundedSystemModalProbeTimeoutRecoversThenReleasesOnDrain() { + assertBoundedSystemModalProbeTimeoutRecoversThenReleasesOnDrain(entryPointName: "snapshotFast") { + target, options in + try self.snapshotFast(app: target, options: options) + } + } + + func testBoundedSystemModalProbeTimeoutRecoversThenReleasesOnDrainForSnapshotRaw() { + assertBoundedSystemModalProbeTimeoutRecoversThenReleasesOnDrain(entryPointName: "snapshotRaw") { + target, options in + try self.snapshotRaw(app: target, options: options) + } + } +#endif + + func testDispatchRecoverySkipsBookkeepingWhileXCTestChannelOccupied() { + // The #1244 recovery shape: the modal probe abandoned an XCTest query that is still grinding on + // main, the capture recovered independently, and its response is ready. The recovery loop must + // return it without re-entering the main queue for recorded-failure/retry bookkeeping (that hop + // would block behind the abandoned query and re-stall the command), and a later command must + // still see the runner busy until the abandoned work drains. Removing the guard regresses this. + let command = try! JSONDecoder().decode( + Command.self, + from: Data(#"{"command":"snapshot","commandId":"recovery-guard"}"#.utf8) + ) + let recovered = Response(ok: false, error: .targetAppUnavailable(bundleId: nil)) + + setAbandonedMainThreadWork(1) + defer { setAbandonedMainThreadWork(0) } + guard case .busy = currentMainThreadBusyState() else { + return XCTFail("expected RUNNER_BUSY while abandoned XCTest work is outstanding") + } + + var occupiedCalls = 0 + let occupied = try! executeDispatchedWithRecovery(command: command) { + occupiedCalls += 1 + return recovered + } + XCTAssertEqual(occupiedCalls, 1, "recovered response must not retry behind abandoned XCTest work") + XCTAssertEqual(occupied.ok, false) + + setAbandonedMainThreadWork(0) + guard case .idle = currentMainThreadBusyState() else { + return XCTFail("runner should be idle once the abandoned work drained") + } + var drainedCalls = 0 + _ = try! executeDispatchedWithRecovery(command: command) { + drainedCalls += 1 + return recovered + } + XCTAssertEqual(drainedCalls, 2, "with the channel free the read-only retry runs once") + } + + private func setAbandonedMainThreadWork(_ count: Int) { + mainThreadWorkLock.lock() + abandonedMainThreadWorkCount = count + abandonedMainThreadWorkSince = count > 0 ? Date(timeIntervalSinceNow: -1) : nil + mainThreadWorkLock.unlock() + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotTimingTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotTimingTests.swift new file mode 100644 index 0000000000..24fdcfaa46 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotTimingTests.swift @@ -0,0 +1,78 @@ +import Foundation +import XCTest + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +extension RunnerTests { + func testXCTestPenaltyDecisionSeparatesAcquisitionAndPresentation() { + let slowPresentation = SnapshotBackendAttempt( + outcome: .noCapture, + timing: SnapshotCaptureTiming(acquisitionMs: 100, presentationMs: 4_000) + ) + XCTAssertNil( + Self.snapshotXCTestPenaltyReason( + kind: .recursiveTree, + attempt: slowPresentation, + slowThresholdMs: 3_000 + ) + ) + + let slowAcquisition = SnapshotBackendAttempt( + outcome: .noCapture, + timing: SnapshotCaptureTiming(acquisitionMs: 3_001, presentationMs: 100) + ) + XCTAssertEqual( + Self.snapshotXCTestPenaltyReason( + kind: .recursiveTree, + attempt: slowAcquisition, + slowThresholdMs: 3_000 + ), + "slow_tree_capture_3001ms" + ) + + let timeout = SnapshotCaptureFailure( + code: Self.xCTestSnapshotTimeoutCode, + message: "test timeout", + hint: "test" + ) + let acquisitionFailure = SnapshotBackendAttempt( + outcome: .failed(timeout, phase: .acquisition), + timing: SnapshotCaptureTiming(acquisitionMs: 100, presentationMs: 100) + ) + XCTAssertEqual( + Self.snapshotXCTestPenaltyReason( + kind: .recursiveTree, + attempt: acquisitionFailure, + slowThresholdMs: 3_000 + ), + "tree_backend_timeout" + ) + + let presentationFailure = SnapshotBackendAttempt( + outcome: .failed(timeout, phase: .presentation), + timing: SnapshotCaptureTiming(acquisitionMs: 100, presentationMs: 100) + ) + XCTAssertNil( + Self.snapshotXCTestPenaltyReason( + kind: .recursiveTree, + attempt: presentationFailure, + slowThresholdMs: 3_000 + ) + ) + } + + func testSnapshotPhaseTimerReportsAcquisitionAndPresentationSeparately() { + var now = Date(timeIntervalSinceReferenceDate: 100) + var timer = SnapshotPhaseTimer(now: { now }) + + _ = timer.measure(.acquisition) { + now = now.addingTimeInterval(2) + } + _ = timer.measure(.presentation) { + now = now.addingTimeInterval(5) + } + + XCTAssertEqual(timer.timing.acquisitionMs, 2_000, accuracy: 0.001) + XCTAssertEqual(timer.timing.presentationMs, 5_000, accuracy: 0.001) + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGesturePolicyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGesturePolicyTests.swift new file mode 100644 index 0000000000..acb3d9b032 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGesturePolicyTests.swift @@ -0,0 +1,105 @@ +import XCTest + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS) +extension RunnerTests { + func testSynthesizedGesturePolicyMarkerWritesOncePerKindUntilTheDecisionChanges() { + var written: [String] = [] + runnerMarkerWriter = { written.append($0) } + defer { + runnerMarkerWriter = { NSLog("%@", $0) } + invalidateCachedTarget(reason: "unit_test_cleanup") + } + logSynthesizedGesturePolicyDecision(kind: .coordinateTap, context: nil, fallbackAttempted: false) + logSynthesizedGesturePolicyDecision(kind: .coordinateTap, context: nil, fallbackAttempted: false) + XCTAssertEqual(written.count, 1, "a repeated decision writes no second line") + logSynthesizedGesturePolicyDecision(kind: .scroll, context: nil, fallbackAttempted: false) + XCTAssertEqual(written.count, 2, "each gesture kind states its own decision") + logSynthesizedGesturePolicyDecision(kind: .coordinateTap, context: nil, fallbackAttempted: true) + XCTAssertEqual(written.count, 3, "a changed decision writes a new line") + resetTargetBoundState() + logSynthesizedGesturePolicyDecision(kind: .coordinateTap, context: nil, fallbackAttempted: true) + XCTAssertEqual(written.count, 4, "a rebind states the same decision once more") + } +} +#endif + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +extension RunnerTests { + func testSynthesizedFallbackPolicyRequiresPrivateSynthesisForScrollWhenAxUnavailableOrUnknown() { + XCTAssertFalse( + SynthesizedFallbackPolicy.privateSynthesisRequired + .allowsXCTestCoordinateFallback(accessibilityHealth: .unavailable) + ) + XCTAssertFalse( + SynthesizedFallbackPolicy.privateSynthesisRequired + .allowsXCTestCoordinateFallback(accessibilityHealth: .unknown) + ) + XCTAssertFalse( + SynthesizedFallbackPolicy.privateSynthesisRequired + .allowsXCTestCoordinateFallback(accessibilityHealth: .healthy) + ) + } + + func testSynthesizedDragCoordinateFallbackAllowsUnknownButNotUnavailableAccessibility() { + XCTAssertTrue( + SynthesizedFallbackPolicy.xctestCoordinateWhenAccessibilityAvailable + .allowsXCTestCoordinateFallback(accessibilityHealth: .healthy) + ) + XCTAssertFalse( + SynthesizedFallbackPolicy.xctestCoordinateWhenAccessibilityAvailable + .allowsXCTestCoordinateFallback(accessibilityHealth: .unavailable) + ) + XCTAssertTrue( + SynthesizedFallbackPolicy.xctestCoordinateWhenAccessibilityAvailable + .allowsXCTestCoordinateFallback(accessibilityHealth: .unknown) + ) + } + + /// Keyboard-policy semantics only. Which command gets which policy is the table below; a probe + /// that is merely permitted still costs a live AX fetch, so the two questions stay separate. + func testSynthesizedKeyboardPolicyAllowsProbeOnlyWhenAccessibilityPermitsIt() { + XCTAssertFalse( + SynthesizedKeyboardPolicy.whenAccessibilityHealthy + .allowsProbe(accessibilityHealth: .unknown) + ) + XCTAssertTrue( + SynthesizedKeyboardPolicy.requiredWhenAvailable + .allowsProbe(accessibilityHealth: .unknown) + ) + XCTAssertFalse( + SynthesizedKeyboardPolicy.requiredWhenAvailable + .allowsProbe(accessibilityHealth: .unavailable) + ) + } + + func testSynthesizedGesturePoliciesMatchCommandContracts() { + XCTAssertEqual( + synthesizedGesturePolicy(.coordinateTap), + SynthesizedGesturePolicy( + keyboardPolicy: .never, + fallbackPolicy: .xctestCoordinateAllowed + ) + ) + XCTAssertEqual( + synthesizedGesturePolicy(.scroll), + SynthesizedGesturePolicy( + keyboardPolicy: .requiredWhenAvailable, + fallbackPolicy: .privateSynthesisRequired + ) + ) + XCTAssertEqual( + synthesizedGesturePolicy(.synthesizedDrag), + SynthesizedGesturePolicy( + keyboardPolicy: .requiredWhenAvailable, + fallbackPolicy: .xctestCoordinateWhenAccessibilityAvailable + ) + ) + } + + func testCoordinateTapTextInputProbeSkipsPenalizedXCTestChannel() { + XCTAssertTrue(shouldProbeCoordinateTapTextInput(xCTestChannelPenalized: false)) + XCTAssertFalse(shouldProbeCoordinateTapTextInput(xCTestChannelPenalized: true)) + } + +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SystemSurfaceHostPolicyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SystemSurfaceHostPolicyTests.swift new file mode 100644 index 0000000000..e16aac6e3b --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SystemSurfaceHostPolicyTests.swift @@ -0,0 +1,50 @@ +import Foundation + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +import XCTest + +private struct SystemSurfaceHostFixture: Decodable { + struct Host: Decodable { + let bundleId: String + let kind: String + } + let hosts: [Host] +} + +extension RunnerTests { + func testSystemSurfaceHostRegistryMirrorsGoldenFixture() throws { + let fixtureURL = URL(fileURLWithPath: #filePath) + .deletingLastPathComponent() // UnitTests + .deletingLastPathComponent() // AgentDeviceRunnerUITests + .deletingLastPathComponent() // AgentDeviceRunner + .deletingLastPathComponent() // runner + .deletingLastPathComponent() // apple + .deletingLastPathComponent() // repo root + .appendingPathComponent("contracts") + .appendingPathComponent("fixtures") + .appendingPathComponent("ios-system-surface-hosts.json") + let fixture = try JSONDecoder().decode( + SystemSurfaceHostFixture.self, + from: Data(contentsOf: fixtureURL) + ) + let registry = SystemSurfaceHostRegistry.hosts.map { [$0.bundleId, $0.kind.rawValue] } + let golden = fixture.hosts.map { [$0.bundleId, $0.kind] } + XCTAssertEqual(registry, golden, "SystemSurfaceHostRegistry drifted from the golden fixture") + } + + func testSystemSurfaceHostRegistryRecognizesRegisteredHosts() { + XCTAssertTrue(SystemSurfaceHostRegistry.isSystemSurfaceHost("com.apple.SafariViewService")) + XCTAssertTrue(SystemSurfaceHostRegistry.isSystemSurfaceHost("com.apple.PassbookUIService")) + XCTAssertFalse(SystemSurfaceHostRegistry.isSystemSurfaceHost("com.example.app")) + XCTAssertFalse(SystemSurfaceHostRegistry.isSystemSurfaceHost(nil)) + XCTAssertEqual( + SystemSurfaceHostRegistry.host(forBundleId: "com.apple.SafariViewService")?.kind, + .webAuth + ) + XCTAssertEqual( + SystemSurfaceHostRegistry.host(forBundleId: "com.apple.PassbookUIService")?.kind, + .payment + ) + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TapPointPolicyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TapPointPolicyTests.swift new file mode 100644 index 0000000000..791b1b2530 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TapPointPolicyTests.swift @@ -0,0 +1,52 @@ +import XCTest + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +private struct TapPointPolicyFixture: Decodable { + struct Frame: Decodable { + let x: Double + let y: Double + let width: Double + let height: Double + + var cgRect: CGRect { + CGRect(x: x, y: y, width: width, height: height) + } + } + + let name: String + let elementFrame: Frame + let windowFrame: Frame + let allowed: Bool +} + +extension RunnerTests { + // Golden parity table (ADR 0011 Layer 2): every case in + // contracts/fixtures/tap-point-policy.json must agree with the vitest twin + // (tap-point-policy-parity.test.ts). Add cases there, never fork the rule. + func testTapPointPolicyMatchesGoldenParityTable() throws { + let fixtureURL = URL(fileURLWithPath: #filePath) + .deletingLastPathComponent() // UnitTests + .deletingLastPathComponent() // AgentDeviceRunnerUITests + .deletingLastPathComponent() // AgentDeviceRunner + .deletingLastPathComponent() // runner + .deletingLastPathComponent() // apple + .deletingLastPathComponent() // repo root + .appendingPathComponent("contracts") + .appendingPathComponent("fixtures") + .appendingPathComponent("tap-point-policy.json") + let data = try Data(contentsOf: fixtureURL) + let cases = try JSONDecoder().decode([TapPointPolicyFixture].self, from: data) + XCTAssertFalse(cases.isEmpty, "parity table must not be empty") + for fixture in cases { + XCTAssertEqual( + TapPointPolicy.isAllowed( + elementFrame: fixture.elementFrame.cgRect, + windowFrame: fixture.windowFrame.cgRect + ), + fixture.allowed, + fixture.name + ) + } + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextInputCandidatePolicyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextInputCandidatePolicyTests.swift new file mode 100644 index 0000000000..ec3201275b --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextInputCandidatePolicyTests.swift @@ -0,0 +1,33 @@ +import CoreGraphics +import XCTest + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +extension RunnerTests { + func testCoordinateTextInputCandidateMustBeEnabledAndContainTheTouchPoint() { + let frame = CGRect(x: 10, y: 20, width: 100, height: 40) + let point = CGPoint(x: 50, y: 40) + + XCTAssertTrue( + isCoordinateTextInputCandidate( + enabled: true, + frame: frame, + point: point + ) + ) + XCTAssertFalse( + isCoordinateTextInputCandidate( + enabled: false, + frame: frame, + point: point + ) + ) + XCTAssertFalse( + isCoordinateTextInputCandidate( + enabled: true, + frame: frame, + point: CGPoint(x: 200, y: 200) + ) + ) + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TransportTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TransportTests.swift new file mode 100644 index 0000000000..42c0f53875 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TransportTests.swift @@ -0,0 +1,43 @@ +import XCTest +import Network + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +extension RunnerTests { + func testDuplicateCommandIdCoalescesOntoInFlightExecution() throws { + let command = try JSONDecoder().decode( + Command.self, + from: Data(#"{"command":"snapshot","commandId":"snapshot-coalesce"}"#.utf8) + ) + var primaryData: Data? + var waiterData: Data? + defer { + inFlightCommandIds.removeAll() + inFlightCommandWaiters.removeAll() + } + + XCTAssertFalse( + attachToInFlightCommandIfNeeded(command: command) { result in + primaryData = result.data + } + ) + XCTAssertTrue( + attachToInFlightCommandIfNeeded(command: command) { result in + waiterData = result.data + } + ) + + let delivered = Data("single-result".utf8) + deliverCommandResult( + command: command, + result: (delivered, false) + ) { result in + primaryData = result.data + } + + XCTAssertEqual(primaryData, delivered) + XCTAssertEqual(waiterData, delivered) + XCTAssertFalse(inFlightCommandIds.contains("snapshot-coalesce")) + XCTAssertNil(inFlightCommandWaiters["snapshot-coalesce"]) + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TvRemoteTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TvRemoteTests.swift new file mode 100644 index 0000000000..3cdcc89651 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TvRemoteTests.swift @@ -0,0 +1,25 @@ +import XCTest + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +extension RunnerTests { + func testTvRemoteButtonMappingAcceptsSupportedNamesAndRejectsUnknown() { + let supported = [ + ("select", "select"), + ("SELECT", "select"), + ("menu", "menu"), + ("home", "home"), + ("up", "up"), + ("down", "down"), + ("left", "left"), + ("right", "right"), + ] + for (raw, expected) in supported { + XCTAssertEqual(tvRemoteButton(from: raw)?.rawValue, expected) + } + + for raw in [String?(nil), "", "volumeUp", "select "] { + XCTAssertNil(tvRemoteButton(from: raw)) + } + } +} +#endif diff --git a/apple/runner/README.md b/apple/runner/README.md index 4a657b6ce8..d899f9cfce 100644 --- a/apple/runner/README.md +++ b/apple/runner/README.md @@ -33,6 +33,8 @@ Protocol and maintenance references: - `RunnerTests+Snapshot.swift`: fast/raw snapshot builders and include/filter helpers. - `RunnerTests+SystemModal.swift`: SpringBoard/system modal detection and modal snapshot shaping. - `RunnerTests+ScreenRecorder.swift`: nested `ScreenRecorder` implementation. +- `UnitTests/RunnerTests+Tests.swift`: the `AGENT_DEVICE_RUNNER_UNIT_TESTS` tests for each + source file. The packaged runner source omits this directory. ## Snapshot Strategy diff --git a/packages/contracts/src/scroll-gesture.test.ts b/packages/contracts/src/scroll-gesture.test.ts index 1c5f700560..de068535ad 100644 --- a/packages/contracts/src/scroll-gesture.test.ts +++ b/packages/contracts/src/scroll-gesture.test.ts @@ -60,8 +60,8 @@ test('buildInPageSwipeGesturePlan truncates percentage coordinates on odd viewpo // Cross-language parity table: every case in contracts/fixtures/scroll-gesture.json is asserted // here AND by the Swift port (runnerScrollGesturePlan in RunnerTests+ScrollGesture.swift, gated -// XCTest in the same file). Add vectors to the table, never to one suite — drift on either side -// turns CI red without a simulator. +// XCTest in UnitTests/RunnerTests+ScrollGestureTests.swift). Add vectors to the table, never to +// one suite — drift on either side turns CI red without a simulator. type ScrollGestureFixture = { constants: { defaultIosScrollAmount: number; @@ -210,7 +210,8 @@ test('clampGestureCoordinate returns the lower bound for non-finite coordinates' // Golden parity table: the SAME JSON is asserted against the Swift twin // (ScrollViewportPolicy in apple/runner/AgentDeviceRunner/ -// AgentDeviceRunnerUITests/RunnerScrollViewportPolicy.swift, gated XCTest in the same file), so +// AgentDeviceRunnerUITests/RunnerScrollViewportPolicy.swift, gated XCTest in +// UnitTests/RunnerTests+ScrollViewportPolicyTests.swift), so // a clip that drifts between the iOS runner and the Android/TS owner turns CI red on whichever // side changed, without a simulator. diff --git a/packages/contracts/src/snapshot-tap-point-policy.test.ts b/packages/contracts/src/snapshot-tap-point-policy.test.ts index 9c93995319..526759a786 100644 --- a/packages/contracts/src/snapshot-tap-point-policy.test.ts +++ b/packages/contracts/src/snapshot-tap-point-policy.test.ts @@ -7,9 +7,10 @@ import type { Rect } from '@agent-device/kernel/snapshot'; // ADR 0011 Layer 2 golden parity table: the SAME JSON is asserted against the // Swift twin (TapPointPolicy in apple/runner/AgentDeviceRunner/ -// AgentDeviceRunnerUITests/RunnerTapPointPolicy.swift, gated XCTest in the -// same file), so drift between the runner's ELEMENT_OFFSCREEN guard and the -// runtime's offscreen rule turns CI red on whichever side changed. +// AgentDeviceRunnerUITests/RunnerTapPointPolicy.swift, gated XCTest in +// UnitTests/RunnerTests+TapPointPolicyTests.swift), so drift between the +// runner's ELEMENT_OFFSCREEN guard and the runtime's offscreen rule turns CI +// red on whichever side changed. // // Scope: the table proves the GEOMETRIC rule only — element-frame center // inside the window frame, edges inclusive, empty frame fails open. The From 61fcc6969adc1de79211e434d0e8c5d637523399 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 23 Sep 2026 20:11:23 +0200 Subject: [PATCH 2/2] chore(gates): describe the runner's test membership without the moved inline test RunnerTapPointPolicy.swift no longer declares a test, so the declaration scan and its test stop citing it as the file whose name hid one. Part of #2792 Co-Authored-By: Claude Opus 5.5 (1M context) --- scripts/__tests__/xctest-declarations.test.ts | 4 ++-- scripts/__tests__/xctest-selection.test.ts | 2 +- scripts/xctest-declarations.ts | 5 ++--- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/scripts/__tests__/xctest-declarations.test.ts b/scripts/__tests__/xctest-declarations.test.ts index a56e7eb4ed..70edbdd68b 100644 --- a/scripts/__tests__/xctest-declarations.test.ts +++ b/scripts/__tests__/xctest-declarations.test.ts @@ -55,8 +55,8 @@ describe('the declaration scan', () => { }); test('reads a file whose name does not start with RunnerTests', () => { - // RunnerTapPointPolicy.swift is the real instance: the synchronized-root-group project - // compiles every .swift in the directory, so file naming carries no membership meaning. + // The synchronized-root-group project compiles every .swift in the directory, so file + // naming carries no membership meaning. expect( parseDeclaredTests(TARGET, [ { diff --git a/scripts/__tests__/xctest-selection.test.ts b/scripts/__tests__/xctest-selection.test.ts index 177b5726e0..20f282fdea 100644 --- a/scripts/__tests__/xctest-selection.test.ts +++ b/scripts/__tests__/xctest-selection.test.ts @@ -113,7 +113,7 @@ describe('the real tree', () => { // own regex, because the Xcode project uses a PBXFileSystemSynchronizedRootGroup — every // .swift file in it is a member. Reusing the check's own file filter would make this // tautological, and a name-based filter is exactly the bug it caught - // (RunnerTapPointPolicy.swift declares a test and does not start with "RunnerTests"). + // (RunnerTapPointPolicy.swift declared a test and does not start with "RunnerTests"). const directory = path.join(repoRoot, RUNNER_TESTS_DIR); const countAddressableMethods = (sourceDirectory: string): number => fs.readdirSync(sourceDirectory, { withFileTypes: true }).reduce((total, entry) => { diff --git a/scripts/xctest-declarations.ts b/scripts/xctest-declarations.ts index 0ab4caf1d3..59ef490648 100644 --- a/scripts/xctest-declarations.ts +++ b/scripts/xctest-declarations.ts @@ -14,9 +14,8 @@ import { activeSource, PLATFORMS, type Platform } from './swift-conditional-comp export const RUNNER_TESTS_DIR = 'apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests'; // Every .swift file below the target directory is a member: the Xcode project uses a -// PBXFileSystemSynchronizedRootGroup, so membership is the directory, not a file list. A -// `RunnerTests*` name filter would miss RunnerTapPointPolicy.swift, which declares a real -// addressable test inside `extension RunnerTests`. +// PBXFileSystemSynchronizedRootGroup, so membership is the directory, not a file list, and a +// file's name says nothing about whether it declares addressable tests. const SWIFT_SOURCE = /\.swift$/; // One ordered pass over the source. A column-0 type declaration moves the enclosing type;