diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerScrollViewportPolicy.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerScrollViewportPolicy.swift index 18c8638222..a2dad024e4 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerScrollViewportPolicy.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerScrollViewportPolicy.swift @@ -12,9 +12,10 @@ import AgentDeviceSnapshotPresentation // // The pure rule below is geometry on purpose — no XCUIApplication — so its exact decision is proven // against the golden table shared with its TS twin, `clipScrollViewportAboveKeyboard` in -// packages/contracts/src/scroll-gesture.ts, asserted in that file's test beside this one. The table -// carries only frames representable in both languages: `CGRect` standardizes a negative extent into a -// positive height at a moved origin, so a negative `height` is tested on the TS side alone. +// packages/contracts/src/scroll-gesture.ts, asserted in that file's test and in +// UnitTests/RunnerTests+ScrollViewportPolicyTests.swift. The table carries only frames +// representable in both languages: `CGRect` standardizes a negative extent into a positive height +// at a moved origin, so a negative `height` is tested on the TS side alone. // // The `extension RunnerTests` below is the one impure caller: it reads the runner's own live keyboard // frame, because a frame threaded from the daemon would predate the keyboard. @@ -248,165 +249,3 @@ extension RunnerTests { } #endif } - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS -private struct ScrollViewportPolicyFixture: Decodable { - struct Frame: Decodable { - let x: Double - let y: Double - let width: Double - let height: Double - - var cgRect: CGRect { - CGRect(x: x, y: y, width: width, height: height) - } - } - - struct Constants: Decodable { - let minVisibleFraction: Double - let accessoryAllowance: Double - } - - struct Expected: Decodable { - let kind: String - let viewport: Frame? - let keyboardMinY: Double? - let visibleHeight: Double? - } - - struct TestCase: Decodable { - let name: String - let viewport: Frame - let keyboard: Frame - let expected: Expected - } - - let constants: Constants - let cases: [TestCase] -} - -extension RunnerTests { - /// Golden parity table (#2500): every case in contracts/fixtures/scroll-keyboard-policy.json must - /// agree with the vitest twin. Add cases there, never fork the rule. - func testScrollViewportKeyboardClipMatchesGoldenParityTable() throws { - let fixture = try loadScrollViewportPolicyFixture() - XCTAssertFalse(fixture.cases.isEmpty, "parity table must not be empty") - for testCase in fixture.cases { - let clip = ScrollViewportPolicy.clip( - viewport: testCase.viewport.cgRect, - keyboard: testCase.keyboard.cgRect - ) - switch testCase.expected.kind { - case "unobstructed": - XCTAssertEqual(clip, .unobstructed, testCase.name) - case "avoided": - let expectedFrame = try XCTUnwrap(testCase.expected.viewport, testCase.name).cgRect - let expectedMinY = try XCTUnwrap(testCase.expected.keyboardMinY, testCase.name) - XCTAssertEqual( - clip, - .avoided(frame: expectedFrame, keyboardMinY: expectedMinY), - testCase.name - ) - case "occluded": - let expectedMinY = try XCTUnwrap(testCase.expected.keyboardMinY, testCase.name) - let expectedVisibleHeight = try XCTUnwrap(testCase.expected.visibleHeight, testCase.name) - XCTAssertEqual( - clip, - .occluded(keyboardMinY: expectedMinY, visibleHeight: expectedVisibleHeight), - testCase.name - ) - default: - XCTFail("unknown expected kind `\(testCase.expected.kind)` in \(testCase.name)") - } - } - } - - /// The thresholds are the table's, not this file's. The refusal reason and the runner code are - /// each one side's own vocabulary: the reason is what the host publishes, the code is what this - /// runner answers with, and neither is a shared clip constant. - func testScrollViewportPolicyUsesParityTableConstants() throws { - let constants = try loadScrollViewportPolicyFixture().constants - XCTAssertEqual(constants.minVisibleFraction, ScrollViewportPolicy.minVisibleFraction) - XCTAssertEqual(constants.accessoryAllowance, ScrollViewportPolicy.accessoryAllowance) - } - - /// A clipped landscape band shortens the frame, and `CoordinateSpaceRotation.native(point:)` derives a - /// `landscapeRight` native x from the frame's HEIGHT. Rotating inside the band therefore moves the - /// dispatched path sideways by exactly what the keyboard took, off the lane the plan was built for, - /// so the plan band and the coordinate basis stay separate values through dispatch (#2500). - func testScrollViewportDispatchKeepsTheUnclippedFrameAsItsCoordinateRotationBasis() throws { - let viewport = CGRect(x: 0, y: 0, width: 1210, height: 834) - let keyboard = CGRect(x: 0, y: 588, width: 1210, height: 246) - let clip = ScrollViewportPolicy.clip(viewport: viewport, keyboard: keyboard) - guard case .avoided(let band, let keyboardMinY) = clip else { - return XCTFail("expected a landscape keyboard to be avoided, got \(clip)") - } - XCTAssertEqual(band.height, 576) - - guard case .gesture(let gesture) = ScrollViewportPolicy.frames( - referenceFrame: viewport, - clip: clip - ).gestureDispatch(direction: .up, amount: nil, pixels: nil) else { - return XCTFail("expected a gesture inside the clipped band") - } - XCTAssertEqual(gesture.planFrame, band) - XCTAssertEqual(gesture.keyboardMinY, keyboardMinY) - XCTAssertEqual(gesture.coordinateFrame, viewport, "the rotation basis must survive the clip") - XCTAssertLessThanOrEqual( - max(gesture.plan.y1, gesture.plan.y2), - keyboard.minY - ScrollViewportPolicy.accessoryAllowance, - "a landscape swipe must stay clear of the keys" - ) - - let reported = gesture.attachingEvidence( - to: Response( - ok: true, - data: DataPayload(referenceWidth: viewport.width, referenceHeight: viewport.height), - error: nil - ) - ) - XCTAssertEqual( - reported.data?.referenceHeight, - band.height, - "the payload names the band the plan ran inside, not the synthesis frame" - ) - XCTAssertEqual(reported.data?.referenceWidth, viewport.width) - XCTAssertEqual(reported.data?.keyboardMinY, keyboardMinY) - XCTAssertEqual(reported.data?.keyboardAvoided, true) - - let orientedStartY = gesture.planFrame.minY + gesture.plan.y1 - let dispatchedFromViewport = CoordinateSpaceRotation.native( - point: CGPoint(x: gesture.planFrame.minX + gesture.plan.x1, y: orientedStartY), - in: gesture.coordinateFrame, - interfaceOrientation: RunnerInterfaceOrientation.landscapeRight - ) - let dispatchedFromBand = CoordinateSpaceRotation.native( - point: CGPoint(x: gesture.planFrame.minX + gesture.plan.x1, y: orientedStartY), - in: gesture.planFrame, - interfaceOrientation: RunnerInterfaceOrientation.landscapeRight - ) - XCTAssertEqual( - dispatchedFromViewport.x - dispatchedFromBand.x, - viewport.height - band.height, - accuracy: 0.001, - "rotating inside the clipped band would shift native x by what the keyboard took" - ) - } - - private func loadScrollViewportPolicyFixture() throws -> ScrollViewportPolicyFixture { - let fixtureURL = URL(fileURLWithPath: #filePath) - .deletingLastPathComponent() // AgentDeviceRunnerUITests - .deletingLastPathComponent() // AgentDeviceRunner - .deletingLastPathComponent() // runner - .deletingLastPathComponent() // apple - .deletingLastPathComponent() // repo root - .appendingPathComponent("contracts") - .appendingPathComponent("fixtures") - .appendingPathComponent("scroll-keyboard-policy.json") - return try JSONDecoder().decode( - ScrollViewportPolicyFixture.self, - from: Data(contentsOf: fixtureURL) - ) - } -} -#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSystemSurfaceHostPolicy.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSystemSurfaceHostPolicy.swift index 97dd753e28..9db900e9e3 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSystemSurfaceHostPolicy.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSystemSurfaceHostPolicy.swift @@ -30,51 +30,3 @@ enum SystemSurfaceHostRegistry { host(forBundleId: bundleId) != nil } } - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS -import XCTest - -private struct SystemSurfaceHostFixture: Decodable { - struct Host: Decodable { - let bundleId: String - let kind: String - } - let hosts: [Host] -} - -extension RunnerTests { - func testSystemSurfaceHostRegistryMirrorsGoldenFixture() throws { - let fixtureURL = URL(fileURLWithPath: #filePath) - .deletingLastPathComponent() // AgentDeviceRunnerUITests - .deletingLastPathComponent() // AgentDeviceRunner - .deletingLastPathComponent() // runner - .deletingLastPathComponent() // apple - .deletingLastPathComponent() // repo root - .appendingPathComponent("contracts") - .appendingPathComponent("fixtures") - .appendingPathComponent("ios-system-surface-hosts.json") - let fixture = try JSONDecoder().decode( - SystemSurfaceHostFixture.self, - from: Data(contentsOf: fixtureURL) - ) - let registry = SystemSurfaceHostRegistry.hosts.map { [$0.bundleId, $0.kind.rawValue] } - let golden = fixture.hosts.map { [$0.bundleId, $0.kind] } - XCTAssertEqual(registry, golden, "SystemSurfaceHostRegistry drifted from the golden fixture") - } - - func testSystemSurfaceHostRegistryRecognizesRegisteredHosts() { - XCTAssertTrue(SystemSurfaceHostRegistry.isSystemSurfaceHost("com.apple.SafariViewService")) - XCTAssertTrue(SystemSurfaceHostRegistry.isSystemSurfaceHost("com.apple.PassbookUIService")) - XCTAssertFalse(SystemSurfaceHostRegistry.isSystemSurfaceHost("com.example.app")) - XCTAssertFalse(SystemSurfaceHostRegistry.isSystemSurfaceHost(nil)) - XCTAssertEqual( - SystemSurfaceHostRegistry.host(forBundleId: "com.apple.SafariViewService")?.kind, - .webAuth - ) - XCTAssertEqual( - SystemSurfaceHostRegistry.host(forBundleId: "com.apple.PassbookUIService")?.kind, - .payment - ) - } -} -#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTapPointPolicy.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTapPointPolicy.swift index 379f78b734..33dde24083 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTapPointPolicy.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTapPointPolicy.swift @@ -25,53 +25,3 @@ enum TapPointPolicy { && centerY >= windowFrame.minY && centerY <= windowFrame.maxY } } - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS -private struct TapPointPolicyFixture: Decodable { - struct Frame: Decodable { - let x: Double - let y: Double - let width: Double - let height: Double - - var cgRect: CGRect { - CGRect(x: x, y: y, width: width, height: height) - } - } - - let name: String - let elementFrame: Frame - let windowFrame: Frame - let allowed: Bool -} - -extension RunnerTests { - // Golden parity table (ADR 0011 Layer 2): every case in - // contracts/fixtures/tap-point-policy.json must agree with the vitest twin - // (tap-point-policy-parity.test.ts). Add cases there, never fork the rule. - func testTapPointPolicyMatchesGoldenParityTable() throws { - let fixtureURL = URL(fileURLWithPath: #filePath) - .deletingLastPathComponent() // AgentDeviceRunnerUITests - .deletingLastPathComponent() // AgentDeviceRunner - .deletingLastPathComponent() // runner - .deletingLastPathComponent() // apple - .deletingLastPathComponent() // repo root - .appendingPathComponent("contracts") - .appendingPathComponent("fixtures") - .appendingPathComponent("tap-point-policy.json") - let data = try Data(contentsOf: fixtureURL) - let cases = try JSONDecoder().decode([TapPointPolicyFixture].self, from: data) - XCTAssertFalse(cases.isEmpty, "parity table must not be empty") - for fixture in cases { - XCTAssertEqual( - TapPointPolicy.isAllowed( - elementFrame: fixture.elementFrame.cgRect, - windowFrame: fixture.windowFrame.cgRect - ), - fixture.allowed, - fixture.name - ) - } - } -} -#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift index f77e0f6035..ed79919b17 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift @@ -317,532 +317,3 @@ extension RunnerTests { return nil } } - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS -// MARK: - In-bundle unit tests - -extension RunnerTests { - func testPrivateAXAttemptDepthsAppliesRememberedDepth() { - XCTAssertEqual( - Self.privateAXAttemptDepths(requestedDepth: 64, rememberedDepth: nil), - [64, 56, 40, 24, 12] - ) - XCTAssertEqual( - Self.privateAXAttemptDepths(requestedDepth: 64, rememberedDepth: 56), - [56, 40, 24, 12] - ) - XCTAssertEqual(Self.privateAXAttemptDepths(requestedDepth: 64, rememberedDepth: 12), [12]) - // Remembered at/above the requested depth changes nothing. - XCTAssertEqual( - Self.privateAXAttemptDepths(requestedDepth: 64, rememberedDepth: 64), - [64, 56, 40, 24, 12] - ) - // A shallower explicit request keeps its own rungs; deeper stale memory is ignored. - XCTAssertEqual(Self.privateAXAttemptDepths(requestedDepth: 24, rememberedDepth: 56), [24, 12]) - } - - /// Executed producer contract for the #1627 review blocker: a frontier whose - /// live element vanished, and one whose re-rooted request fails, must BOTH - /// count as missed — an all-miss extension reporting itself drained would - /// present a capped capture as complete. Goes red if either miss-path - /// increment in extendSnapshotFrontiers is removed. - func testDeepExtensionCountsMissedFrontiers() { - // Element vanished (list churn between serialization and extension): the - // fabricated snapshot answers nil for accessibilityElement — missed, and - // no request call is consumed. (An explicit nil property: bare NSObject - // resolves the key through a UIKit category and would take the call path.) - let orphan = RunnerAXSnapshotFrontier() - orphan.snapshot = FrontierSnapshotWithoutElementForTesting() - orphan.node = NSMutableDictionary() - // Re-rooted request fails: the element resolves but the client cannot - // serve requestSnapshotForElement — one consumed call AND a miss. - let unreachable = RunnerAXSnapshotFrontier() - unreachable.snapshot = FrontierSnapshotWithElementForTesting() - unreachable.node = NSMutableDictionary() - - var nodeCount = 0 - var truncated = ObjCBool(false) - let outcome = RunnerAXSnapshotBridge.extend( - NSMutableArray(array: [orphan, unreachable]), - axClient: NSObject(), - attributes: [], - maxDepth: 56, - maxNodes: 5_000, - nodeCount: &nodeCount, - truncated: &truncated, - callsAllowed: 8, - mergedLeaves: nil, - deadline: nil - ) - - XCTAssertEqual(outcome?[RunnerAXSnapshotDeepExtensionMissedKey] as? Int, 2) - XCTAssertEqual(outcome?[RunnerAXSnapshotDeepExtensionCallsKey] as? Int, 1) - XCTAssertEqual(outcome?[RunnerAXSnapshotDeepExtensionPendingKey] as? Int, 0) - XCTAssertEqual(outcome?[RunnerAXSnapshotDeepExtensionNodesAddedKey] as? Int, 0) - XCTAssertFalse(truncated.boolValue) - // And the consumer verdict over exactly this outcome: still depth-limited. - XCTAssertTrue( - Self.privateAXDepthLimited( - effectiveDepth: 56, requestedDepth: 64, pendingFrontiers: 0, missedFrontiers: 2)) - } - - func testPrivateAXDepthLimitedRequiresEveryFrontierResolved() { - // Un-capped capture is never depth-limited, extension or not. - XCTAssertFalse( - Self.privateAXDepthLimited( - effectiveDepth: 64, requestedDepth: 64, pendingFrontiers: nil, missedFrontiers: nil)) - // Capped with no extension outcome (never ran) stays depth-limited. - XCTAssertTrue( - Self.privateAXDepthLimited( - effectiveDepth: 56, requestedDepth: 64, pendingFrontiers: nil, missedFrontiers: nil)) - // Fully drained extension clears the verdict. - XCTAssertFalse( - Self.privateAXDepthLimited( - effectiveDepth: 56, requestedDepth: 64, pendingFrontiers: 0, missedFrontiers: 0)) - // Budget exhaustion (pending frontiers) keeps it. - XCTAssertTrue( - Self.privateAXDepthLimited( - effectiveDepth: 56, requestedDepth: 64, pendingFrontiers: 2, missedFrontiers: 0)) - // The #1627 review blocker: an all-miss extension (elements vanished or - // re-rooted requests failed) resolved nothing — it must NOT present the - // capture as complete just because the queue emptied. - XCTAssertTrue( - Self.privateAXDepthLimited( - effectiveDepth: 56, requestedDepth: 64, pendingFrontiers: 0, missedFrontiers: 8)) - } - - func testPrivateAXAcceptedDepthMemoryMatchesBundleProcessAndExpires() { - defer { clearPrivateAXAcceptedDepth(reason: "test-cleanup") } - - rememberPrivateAXAcceptedDepth(bundleId: "xyz.blueskyweb.app", processIdentifier: 111, depth: 56) - XCTAssertEqual( - rememberedPrivateAXAcceptedDepth(bundleId: "xyz.blueskyweb.app", processIdentifier: 111), - 56 - ) - XCTAssertNil(rememberedPrivateAXAcceptedDepth(bundleId: "com.other.app", processIdentifier: 111)) - // A relaunch changes the PID; the new process must re-probe the full depth even inside the - // TTL, and an unknown current PID (post-invalidation) must never match. - XCTAssertNil(rememberedPrivateAXAcceptedDepth(bundleId: "xyz.blueskyweb.app", processIdentifier: 222)) - XCTAssertNil(rememberedPrivateAXAcceptedDepth(bundleId: "xyz.blueskyweb.app", processIdentifier: nil)) - - // Expired memory stops applying (the expiry re-probes the full requested depth). - privateAXAcceptedDepthUntil = Date(timeIntervalSinceNow: -1) - XCTAssertNil(rememberedPrivateAXAcceptedDepth(bundleId: "xyz.blueskyweb.app", processIdentifier: 111)) - } - - func testPrivateAXAcceptedDepthMemoryRequiresProcessIdentifierToRecord() { - defer { clearPrivateAXAcceptedDepth(reason: "test-cleanup") } - - rememberPrivateAXAcceptedDepth(bundleId: "xyz.blueskyweb.app", processIdentifier: nil, depth: 56) - XCTAssertNil( - rememberedPrivateAXAcceptedDepth(bundleId: "xyz.blueskyweb.app", processIdentifier: nil) - ) - } - - func testViewportReadSkippedWhileXCTestChannelPenalized() { - // Pins the viewport fast path (#1587 review): every penalized private AX capture used to burn - // the full 1s main-thread timeout on a doomed viewport read before falling back. - currentBundleId = "xyz.blueskyweb.app" - defer { - currentBundleId = nil - clearSnapshotXCTestChannelPenalty(reason: "test-cleanup") - abandonedMainThreadWorkCount = 0 - } - - XCTAssertTrue(shouldReadPrivateAXViewportViaXCTest()) - - penalizeSnapshotXCTestChannel(bundleId: "xyz.blueskyweb.app", reason: "test") - XCTAssertFalse(shouldReadPrivateAXViewportViaXCTest()) - - clearSnapshotXCTestChannelPenalty(reason: "test") - XCTAssertTrue(shouldReadPrivateAXViewportViaXCTest()) - - abandonedMainThreadWorkCount = 1 - XCTAssertFalse(shouldReadPrivateAXViewportViaXCTest()) - } - - /// The wire field must reach both capture options AND the backend pin: custom - /// actions are only readable through the private AX client, so a capture that - /// asked for them but planned the XCTest tree backend would return a payload - /// that structurally cannot carry them. - func testCustomActionsRequestPinsPrivateAXBackend() throws { - let asked = try JSONDecoder().decode( - Command.self, from: Data(#"{"command":"snapshot","customActions":true}"#.utf8)) - let options = Self.presentationOptions(from: asked) - XCTAssertTrue(options.customActions) - XCTAssertEqual(options.preferredBackend, SnapshotBackendKind.privateAX.rawValue) - XCTAssertTrue( - Self.snapshotXCTestChannelTreatedAsPenalized( - penalized: false, preferredBackend: options.preferredBackend)) - - // An explicit pin is never overwritten by the implied one. - let pinned = try JSONDecoder().decode( - Command.self, - from: Data(#"{"command":"snapshot","customActions":true,"preferredBackend":"tree"}"#.utf8)) - XCTAssertEqual(Self.presentationOptions(from: pinned).preferredBackend, "tree") - - // And the default capture neither asks nor pins. - let bare = try JSONDecoder().decode(Command.self, from: Data(#"{"command":"snapshot"}"#.utf8)) - XCTAssertFalse(Self.presentationOptions(from: bare).customActions) - XCTAssertNil(Self.presentationOptions(from: bare).preferredBackend) - } - - /// A request-pinned backend degraded nothing, so its verdict must not claim - /// slow accessibility work — that reason drives a user-facing warning. - func testRequestPinnedBackendReportsItsOwnReason() { - let requested = Self.xcTestChannelStateFirstFailure( - .deferredToIndependentBackend, requestPinnedBackend: true) - XCTAssertEqual(requested?.code, "requested-backend") - XCTAssertFalse(requested?.reason.contains("slow accessibility work") ?? true) - - // The circuit breaker's own deferral keeps its established code and wording. - let breaker = Self.xcTestChannelStateFirstFailure(.deferredToIndependentBackend) - XCTAssertEqual(breaker?.code, "deferred") - - // The bounded probe and the healthy plan are untouched by the new flag. - XCTAssertEqual( - Self.xcTestChannelStateFirstFailure(.boundedXCTestProbe, requestPinnedBackend: true)?.code, - "budget") - XCTAssertNil(Self.xcTestChannelStateFirstFailure(.normal, requestPinnedBackend: true)) - } - - /// The disclosure only exists if the counts survive the bridge boundary, and - /// "did not ask" must stay distinguishable from "read none". - func testCustomActionCoverageParsesOnlyCompletePairs() { - let complete: [String: Any] = [ - RunnerAXSnapshotCustomActionsReadKey: 12, - RunnerAXSnapshotCustomActionsCandidatesKey: 19, - RunnerAXSnapshotCustomActionsTruncatedKey: 2, - RunnerAXSnapshotCustomActionsBlockedKey: true, - ] - let coverage = Self.privateAXCustomActionCoverage(complete) - XCTAssertEqual(coverage?.read, 12) - XCTAssertEqual(coverage?.candidates, 19) - XCTAssertEqual(coverage?.truncated, 2) - XCTAssertEqual(coverage?.blocked, true) - - // Absent key = the capture never asked; it must not read as (0, 0), which - // would warn "0 of 0" on every default capture. - XCTAssertNil(Self.privateAXCustomActionCoverage(nil)) - // The bridge in this target always writes all four keys, so a partial - // dictionary is malformed and is dropped whole. - for key in complete.keys { - var partial = complete - partial.removeValue(forKey: key) - XCTAssertNil(Self.privateAXCustomActionCoverage(partial), "missing \(key)") - } - } - - /// The AX call cannot be cancelled once issued, so the read deadline frees - /// only the caller — the call keeps running. Without containment, repeating - /// `snapshot --actions` against a wedged element would stack orphaned reads, - /// all sharing one XCAXClient. This pins the containment: one serial queue and - /// a single-flight refusal that adds no work while a read is outstanding. - func testHungCustomActionReadIsContainedAndRecovers() { - let hung = HungAXClientForTesting() - let element = NSObject() - let dispatchesBefore = RunnerAXSnapshotBridge.customActionReadDispatchCount() - let blockedBefore = RunnerAXSnapshotBridge.customActionReadBlockedCount() - defer { hung.release() } - - // 1. First read wedges. The caller is freed by the deadline, but the call is - // still out there, so it stays counted in flight. - var completed = ObjCBool(true) - let firstStarted = Date() - let first = RunnerAXSnapshotBridge.customActionNames( - forElement: element, axClient: hung, completed: &completed) - XCTAssertNil(first) - XCTAssertFalse(completed.boolValue) - XCTAssertGreaterThanOrEqual(-firstStarted.timeIntervalSinceNow, 0.9) - XCTAssertEqual(RunnerAXSnapshotBridge.customActionReadsInFlight(), 1) - XCTAssertEqual( - RunnerAXSnapshotBridge.customActionReadDispatchCount(), dispatchesBefore + 1) - - // 2. Repeats do NOT accumulate: no new dispatch, still exactly one in - // flight, and every repeat is refused by single-flight admission. - for _ in 0..<5 { - XCTAssertNil( - RunnerAXSnapshotBridge.customActionNames( - forElement: element, axClient: hung, completed: &completed)) - XCTAssertFalse(completed.boolValue) - } - XCTAssertEqual(RunnerAXSnapshotBridge.customActionReadsInFlight(), 1) - XCTAssertEqual( - RunnerAXSnapshotBridge.customActionReadDispatchCount(), dispatchesBefore + 1) - XCTAssertEqual(RunnerAXSnapshotBridge.customActionReadBlockedCount(), blockedBefore + 5) - - // 3. A capture in that state discloses the skip rather than presenting the - // unread elements as action-free — and spends no read budget doing it. - let leaf = RunnerAXSnapshotFrontier() - leaf.snapshot = FrontierSnapshotWithElementForTesting() - leaf.node = NSMutableDictionary(dictionary: ["label": "feedItem", "children": []]) - let coverage = RunnerAXSnapshotBridge.annotateCustomActions( - onMergedLeaves: [leaf], axClient: hung, limit: 12, rootFrame: .zero, deadline: nil) - XCTAssertEqual(coverage[RunnerAXSnapshotCustomActionsBlockedKey] as? Bool, true) - XCTAssertEqual(coverage[RunnerAXSnapshotCustomActionsReadKey] as? Int, 0) - XCTAssertEqual(coverage[RunnerAXSnapshotCustomActionsCandidatesKey] as? Int, 1) - XCTAssertEqual( - RunnerAXSnapshotBridge.customActionReadDispatchCount(), dispatchesBefore + 1) - XCTAssertEqual( - Self.privateAXCustomActionCoverage(coverage), - SnapshotCustomActionCoverage(read: 0, candidates: 1, truncated: 0, blocked: true)) - - // 4. Recovery: once the wedged call returns, reads resume by themselves. - hung.release() - let recovered = expectation(description: "in-flight drains") - DispatchQueue.global().async { - while RunnerAXSnapshotBridge.customActionReadsInFlight() > 0 { - usleep(20_000) - } - recovered.fulfill() - } - wait(for: [recovered], timeout: 5) - - completed = ObjCBool(false) - XCTAssertNil( - RunnerAXSnapshotBridge.customActionNames( - forElement: element, axClient: hung, completed: &completed)) - // Completed (the fake answers nil actions), which is the point: the pass is - // live again rather than latched off. - XCTAssertTrue(completed.boolValue) - XCTAssertEqual( - RunnerAXSnapshotBridge.customActionReadDispatchCount(), dispatchesBefore + 2) - } - - /// The element budget bounds how many elements we read; these caps bound what - /// any ONE element can put in the response. Clipping must be reported, since - /// a clipped list looks exactly like a complete one. - func testActionNamesAreCappedPerElementAndReported() { - var truncated = ObjCBool(true) - - // Under both caps: untouched, nothing to report. - let small = ["Reply", "Repost"] - XCTAssertEqual( - RunnerAXSnapshotBridge.cappedActionNames(small, truncated: &truncated), small) - XCTAssertFalse(truncated.boolValue) - - // More actions than the per-element cap: clipped to the first 8, reported. - let many = (1...20).map { "Action \($0)" } - let cappedMany = RunnerAXSnapshotBridge.cappedActionNames(many, truncated: &truncated) - XCTAssertEqual(cappedMany.count, 8) - XCTAssertEqual(cappedMany.first, "Action 1") - XCTAssertTrue(truncated.boolValue) - - // A single very long name is shortened, reported, and stays one string. - let long = String(repeating: "a", count: 500) - let cappedLong = RunnerAXSnapshotBridge.cappedActionNames([long], truncated: &truncated) - XCTAssertEqual(cappedLong.count, 1) - XCTAssertTrue(truncated.boolValue) - XCTAssertLessThan(cappedLong[0].count, long.count) - XCTAssertTrue(cappedLong[0].hasSuffix("…")) - - // Empty input is not "truncated". - XCTAssertEqual(RunnerAXSnapshotBridge.cappedActionNames([], truncated: &truncated), []) - XCTAssertFalse(truncated.boolValue) - } - - /// Action names annotated by the bridge must survive into the emitted node — - /// the whole point of the capture is that the merged card names its hidden - /// affordances. - func testPrivateAXNodesCarryAnnotatedCustomActions() { - let tree: [String: Any] = [ - "type": Int(XCUIElement.ElementType.application.rawValue), - "label": "Blue Sky", - "frame": ["x": 0, "y": 0, "width": 390, "height": 844], - "children": [ - [ - "type": Int(XCUIElement.ElementType.link.rawValue), - "label": "feedItem-by-whiskers.test", - "frame": ["x": 0, "y": 100, "width": 390, "height": 200], - "actions": ["Reply", "Repost", "Open post options menu"], - "children": [], - ], - [ - "type": Int(XCUIElement.ElementType.button.rawValue), - "label": "Compose", - "frame": ["x": 300, "y": 700, "width": 60, "height": 60], - "children": [], - ], - ], - ] - let nodes = privateAXAcquisition( - rawRoot: tree, - hint: CaptureHint( - projection: .regular, depth: nil, regularPresentedDepth: nil, - interactiveOnly: false, customActions: false) - ) - - let card = nodes.first { $0.label == "feedItem-by-whiskers.test" } - XCTAssertEqual(card?.actions, ["Reply", "Repost", "Open post options menu"]) - // A node the bridge did not annotate stays absent, not empty. - XCTAssertNil(nodes.first { $0.label == "Compose" }?.actions) - } - - func testPrivateAXAcquisitionDoesNotInterpretScope() { - let tree: [String: Any] = [ - "type": 1, "label": "App", - "children": [ - [ - "type": 9, "identifier": "homeScreen", - "children": [ - ["type": 48, "label": "Post body without the scope text", "children": []] - ], - ], - ["type": 9, "label": "unrelated sibling", "children": []], - ], - ] - // Scope never reaches acquisition: the hint derived for a scoped request carries no scope, - // and the backend has no way to interpret one. - let nodes = privateAXAcquisition( - rawRoot: tree, - hint: SnapshotPresentation.captureHint( - for: PresentationOptions( - interactiveOnly: false, - depth: nil, - scope: "homeScreen", - raw: false - ) - ) - ) - - let labels = nodes.compactMap { $0.label ?? $0.identifier } - XCTAssertTrue(labels.contains("homeScreen")) - // Descendants of the matched scope are included even when they do not contain the text. - XCTAssertTrue(labels.contains("Post body without the scope text")) - XCTAssertTrue(labels.contains("unrelated sibling")) - } - - func testPrivateAXInteractiveFiltersLoginLikeHiddenDrawer() throws { - let tree: [String: Any] = [ - "type": Int(XCUIElement.ElementType.application.rawValue), - "label": "Blue Sky", - "frame": ["x": 0, "y": 0, "width": 390, "height": 844], - "children": [ - [ - "type": Int(XCUIElement.ElementType.scrollView.rawValue), - "frame": ["x": 0, "y": 0, "width": 390, "height": 844], - "children": [ - [ - "type": Int(XCUIElement.ElementType.image.rawValue), - "label": "Callstack", - "frame": ["x": 145, "y": 104, "width": 100, "height": 100], - "children": [], - ], - [ - "type": Int(XCUIElement.ElementType.staticText.rawValue), - "label": "Welcome back", - "frame": ["x": 32, "y": 260, "width": 326, "height": 32], - "children": [], - ], - [ - "type": Int(XCUIElement.ElementType.textField.rawValue), - "label": "Email", - "identifier": "login.email", - "frame": ["x": 32, "y": 348, "width": 326, "height": 48], - "children": [], - ], - [ - "type": Int(XCUIElement.ElementType.secureTextField.rawValue), - "label": "Password", - "identifier": "login.password", - "frame": ["x": 32, "y": 412, "width": 326, "height": 48], - "children": [], - ], - [ - "type": Int(XCUIElement.ElementType.button.rawValue), - "label": "Sign in", - "identifier": "login.submit", - "frame": ["x": 32, "y": 492, "width": 326, "height": 52], - "children": [], - ], - [ - "type": Int(XCUIElement.ElementType.link.rawValue), - "label": "Forgot password?", - "frame": ["x": 128, "y": 568, "width": 134, "height": 32], - "children": [], - ], - [ - "type": Int(XCUIElement.ElementType.button.rawValue), - "label": "Admin settings", - "frame": ["x": -260, "y": 184, "width": 220, "height": 44], - "children": [], - ], - [ - "type": Int(XCUIElement.ElementType.other.rawValue), - "frame": ["x": 16, "y": 184, "width": 220, "height": 44], - "children": [], - ], - ], - ] - ], - ] - let viewport = CGRect(x: 0, y: 0, width: 390, height: 844) - let hint = CaptureHint( - projection: .regular, depth: nil, regularPresentedDepth: nil, - interactiveOnly: true, customActions: false) - let acquired = SnapshotGeometrySpace.normalized( - nodes: privateAXAcquisition(rawRoot: tree, hint: hint), - viewport: viewport, - interfaceOrientation: RunnerInterfaceOrientation.portrait - ) - // Acquisition serializes the drawer too; the shared fold is what hides it (#1797). - XCTAssertTrue(acquired.compactMap(\.label).contains("Admin settings")) - - let capture = try SnapshotPresentation.presentRegular( - SnapshotAcquisition( - hint: hint, nodes: acquired, truncated: false, effectiveDepth: nil, viewport: viewport), - options: PresentationOptions(interactiveOnly: true, depth: nil, scope: nil, raw: false), - policy: .cursorProjected - ) - let labels = capture.nodes.compactMap { $0.label } - XCTAssertEqual( - labels, - ["Blue Sky", "Callstack", "Welcome back", "Email", "Password", "Sign in", "Forgot password?"] - ) - XCTAssertFalse(labels.contains("Admin settings")) - } -} - -/// Stands in for an AX client whose `attributesForElement:` never returns — -/// the wedged-server case the containment exists for. `release()` lets the -/// hung call finish so recovery is observable. -private final class HungAXClientForTesting: NSObject { - private let gate = DispatchSemaphore(value: 0) - private let releasedOnce = NSLock() - private var released = false - - @objc(attributesForElement:attributes:error:) - func attributes(forElement element: Any, attributes: Any, error: NSErrorPointer) -> Any? { - releasedOnce.lock() - let alreadyReleased = released - releasedOnce.unlock() - // Once the wedge clears, the server answers normally again — that is what - // makes the recovery leg a recovery rather than a second hang. - if alreadyReleased { - return nil - } - gate.wait() - return nil - } - - func release() { - releasedOnce.lock() - defer { releasedOnce.unlock() } - guard !released else { return } - released = true - gate.signal() - } -} - -/// Minimal snapshot stand-in whose accessibilityElement resolves (so the -/// extension proceeds to the request) while the paired fake client cannot -/// serve it — the failed-re-root miss path. -private final class FrontierSnapshotWithElementForTesting: NSObject { - @objc let accessibilityElement = NSObject() -} - -/// The vanished-element case: KVC resolves the property and gets nil. -private final class FrontierSnapshotWithoutElementForTesting: NSObject { - @objc let accessibilityElement: NSObject? = nil -} -#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift index 483862a3af..60df7eea72 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift @@ -188,7 +188,7 @@ extension RunnerTests { return buttons.first(where: { isDismissButton($0.label) }) ?? buttons.last } - private func isAcceptButton(_ label: String) -> Bool { + func isAcceptButton(_ label: String) -> Bool { let normalized = label.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() return [ "ok", @@ -201,12 +201,6 @@ extension RunnerTests { ].contains(normalized) || normalized.hasPrefix("confirm") } -#if AGENT_DEVICE_RUNNER_UNIT_TESTS - func testAlertAcceptTreatsOpenAsAffirmative() { - XCTAssertTrue(isAcceptButton("Open")) - } -#endif - private func isDismissButton(_ label: String) -> Bool { [ "cancel", diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+BlockingSystemModalResolution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+BlockingSystemModalResolution.swift index 2b92c09bdc..687d36d8b8 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+BlockingSystemModalResolution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+BlockingSystemModalResolution.swift @@ -79,39 +79,3 @@ enum RemoteHostedSystemModalPolicy { state == .runningForeground } } - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS -extension RunnerTests { - func testRemoteHostProbeRunsOnlyWhenSpringboardModalHasNoActions() { - XCTAssertTrue(RemoteHostedSystemModalPolicy.shouldProbeRemoteHost(springboardActionCount: 0)) - XCTAssertFalse(RemoteHostedSystemModalPolicy.shouldProbeRemoteHost(springboardActionCount: 1)) - XCTAssertFalse(RemoteHostedSystemModalPolicy.shouldProbeRemoteHost(springboardActionCount: 3)) - } - - func testRemoteHostStateGateFailsClosedToForeground() { - XCTAssertTrue(RemoteHostedSystemModalPolicy.isEligibleHostState(.runningForeground)) - XCTAssertFalse(RemoteHostedSystemModalPolicy.isEligibleHostState(.runningBackground)) - XCTAssertFalse(RemoteHostedSystemModalPolicy.isEligibleHostState(.notRunning)) - XCTAssertFalse(RemoteHostedSystemModalPolicy.isEligibleHostState(.unknown)) - } - - // No SpringBoard host (`hasSpringBoardSystemModalHost`) means modal resolution must return - // `.absent` without probing com.apple.springboard (#1351). Written for tvOS, where no lane - // ever executed it; `resolveBlockingSystemModal` takes that decision at RUNTIME off the same - // flag on macOS, so the host lane runs the real branch on every PR. - // - // Its former sibling `testBlockingSystemAlertSnapshotIsNilOnTvOS` is deleted rather than - // widened: `blockingSystemAlertSnapshot` is `#if os(macOS) return nil`, so on the only lane - // that could run it the assertion would pin a compile-time literal — a green that no change - // to the runner could turn red. The runtime gate it meant to cover is this test's subject, - // and the nil it returns on macOS is the compiler's business, not a test's. - #if os(tvOS) || os(macOS) - func testResolveBlockingSystemModalIsAbsentWithoutSpringBoardHost() { - guard case .absent = resolveBlockingSystemModal(deadline: .distantFuture) else { - XCTFail("blocking system-modal resolution must be .absent without a SpringBoard host") - return - } - } - #endif -} -#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index 2e87e03529..c1aa92ca27 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -1,32 +1,6 @@ import XCTest import AgentDeviceSnapshotPresentation -#if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS) -import ObjectiveC.runtime - -private final class RunnerSynthesizedSwipeFailureStub: NSObject { - @objc(synthesizeSwipeWithApplication:resolvedWindow:x:y:x2:y2:durationMs:) - class func synthesizeSwipe( - application: XCUIApplication, - resolvedWindow: Any?, - x: Double, - y: Double, - x2: Double, - y2: Double, - durationMs: Double - ) -> String? { - "forced private synthesis failure" - } -} - -private final class RunnerSynthesizedTapFailureStub: NSObject { - @objc(synthesizeTapWithApplication:resolvedWindow:x:y:) - class func synthesizeTap(application: XCUIApplication, resolvedWindow: Any?, x: Double, y: Double) -> String? { - "forced private synthesis failure" - } -} -#endif - extension RunnerTests { // MARK: - Main Thread Dispatch @@ -110,7 +84,7 @@ extension RunnerTests { /// Single factory for the success payload every gesture returns (message + gesture timing + /// an optional touch/drag visualization frame), so the field shape lives in one place. - private func gestureResponse( + func gestureResponse( message: String, timing: (gestureStartUptimeMs: Double, gestureEndUptimeMs: Double), frame: GestureFrame = .none, @@ -165,7 +139,7 @@ extension RunnerTests { /// Gesture plans already return canonical centroid endpoints from the portable runtime. /// Keep runner timing/fallback diagnostics, but do not leak the coordinate-drag adapter's /// visualization frame into only the fast-fling response shape. - private func canonicalPlannedGestureResponse(_ response: Response) -> Response { + func canonicalPlannedGestureResponse(_ response: Response) -> Response { guard response.ok, let data = response.data else { return response } return Response( ok: true, @@ -191,678 +165,6 @@ extension RunnerTests { return gestureResponse(message: plan.intent, timing: timing) } -#if AGENT_DEVICE_RUNNER_UNIT_TESTS - func testGestureResponseIncludesSynthesizedTapFallbackDiagnostics() { - let response = gestureResponse( - message: "tapped", - timing: (gestureStartUptimeMs: 1, gestureEndUptimeMs: 2), - fallback: GestureFallback( - strategy: "xctest-coordinate-tap", - message: "Runner synthesized coordinate tap is unavailable", - hint: "Using XCTest coordinate tap fallback." - ) - ) - - XCTAssertEqual(response.ok, true) - XCTAssertEqual(response.data?.gestureFallback, "xctest-coordinate-tap") - XCTAssertEqual( - response.data?.gestureFallbackMessage, - "Runner synthesized coordinate tap is unavailable" - ) - XCTAssertEqual(response.data?.gestureFallbackHint, "Using XCTest coordinate tap fallback.") - } - - func testGestureResponseIncludesMaestroNonHittableFallbackUsage() { - let response = gestureResponse( - message: "tapped via non-hittable coordinate fallback", - timing: (gestureStartUptimeMs: 1, gestureEndUptimeMs: 2), - frame: .touch(nil), - maestroNonHittableCoordinateFallbackUsed: true - ) - - XCTAssertEqual(response.data?.maestroNonHittableCoordinateFallbackUsed, true) - } - - func testCanonicalPlannedGestureResponseOmitsDragFrameAndPreservesDiagnostics() { - let response = gestureResponse( - message: "fling", - timing: (gestureStartUptimeMs: 1, gestureEndUptimeMs: 2), - frame: .drag( - DragVisualizationFrame( - x: 160, - y: 150, - x2: 40, - y2: 150, - referenceWidth: 200, - referenceHeight: 300 - ) - ), - fallback: GestureFallback( - strategy: "xctest-coordinate-drag", - message: "Private synthesis unavailable", - hint: "Using XCTest coordinate fallback." - ) - ) - - let canonical = canonicalPlannedGestureResponse(response) - - XCTAssertEqual(canonical.data?.gestureStartUptimeMs, 1) - XCTAssertEqual(canonical.data?.gestureEndUptimeMs, 2) - XCTAssertEqual(canonical.data?.gestureFallback, "xctest-coordinate-drag") - XCTAssertEqual(canonical.data?.gestureFallbackMessage, "Private synthesis unavailable") - XCTAssertEqual(canonical.data?.gestureFallbackHint, "Using XCTest coordinate fallback.") - XCTAssertNil(canonical.data?.x) - XCTAssertNil(canonical.data?.y) - XCTAssertNil(canonical.data?.x2) - XCTAssertNil(canonical.data?.y2) - XCTAssertNil(canonical.data?.referenceWidth) - XCTAssertNil(canonical.data?.referenceHeight) - } - -#if os(iOS) - func testSinglePointerFlingFallsBackToXCTestCoordinateDragWhenPrivateSynthesisFails() throws { - let selector = NSSelectorFromString( - "synthesizeSwipeWithApplication:resolvedWindow:x:y:x2:y2:durationMs:" - ) - guard - let synthesizedSwipeMethod = class_getClassMethod(RunnerSynthesizedGesture.self, selector), - let failureStubMethod = class_getClassMethod(RunnerSynthesizedSwipeFailureStub.self, selector) - else { - XCTFail("unable to install synthesized swipe failure stub") - return - } - let originalImplementation = method_getImplementation(synthesizedSwipeMethod) - method_setImplementation( - synthesizedSwipeMethod, - method_getImplementation(failureStubMethod) - ) - app.launch() - runnerAccessibilityHealth = .healthy - defer { - method_setImplementation(synthesizedSwipeMethod, originalImplementation) - invalidateCachedTarget(reason: "unit_test_cleanup") - app.terminate() - } - let command = try runnerCommandFixture( - """ - {"command":"gesture","commandId":"gesture-fling-fallback","gesturePlan":{"topology":"single","intent":"fling","executionProfile":"endpoint-hold","durationMs":100,"viewport":{"x":0,"y":0,"width":200,"height":300},"pointers":[{"pointerId":0,"samples":[{"offsetMs":0,"point":{"x":160,"y":150}},{"offsetMs":100,"point":{"x":40,"y":150}}]}]}} - """ - ) - - let response = try executeOnMainPrepared(command: command, activeApp: app) - - XCTAssertTrue(response.ok) - XCTAssertEqual(response.data?.message, "fling") - XCTAssertEqual(response.data?.gestureFallback, "xctest-coordinate-drag") - XCTAssertEqual(response.data?.gestureFallbackMessage, "forced private synthesis failure") - XCTAssertEqual( - response.data?.gestureFallbackHint, - "Private XCTest event synthesis is required for AX-free coordinate drag on iOS; update Xcode if this persists." - ) - XCTAssertNil(response.data?.x) - XCTAssertNil(response.data?.y) - XCTAssertNil(response.data?.x2) - XCTAssertNil(response.data?.y2) - } - - func testSelectorTapFallsBackToXCTestCoordinateWhenPrivateSynthesisFails() throws { - let selector = NSSelectorFromString("synthesizeTapWithApplication:resolvedWindow:x:y:") - guard - let synthesizedTapMethod = class_getClassMethod(RunnerSynthesizedGesture.self, selector), - let failureStubMethod = class_getClassMethod(RunnerSynthesizedTapFailureStub.self, selector) - else { - XCTFail("unable to install synthesized tap failure stub") - return - } - let originalImplementation = method_getImplementation(synthesizedTapMethod) - method_setImplementation( - synthesizedTapMethod, - method_getImplementation(failureStubMethod) - ) - app.launch() - currentApp = app - runnerAccessibilityHealth = .healthy - defer { - method_setImplementation(synthesizedTapMethod, originalImplementation) - invalidateCachedTarget(reason: "unit_test_cleanup") - app.terminate() - } - let command = try runnerCommandFixture( - #"{"command":"tap","commandId":"selector-tap-fallback","selectorKey":"label","selectorValue":"Agent Device Runner","synthesized":true}"# - ) - - let response = try executeOnMainPrepared(command: command, activeApp: app) - - XCTAssertTrue(response.ok) - XCTAssertEqual(response.data?.message, "tapped") - XCTAssertEqual(response.data?.gestureFallback, "xctest-coordinate-tap") - XCTAssertEqual(response.data?.gestureFallbackMessage, "forced private synthesis failure") - XCTAssertEqual( - response.data?.gestureFallbackHint, - "Falling back to XCTest coordinate tap may be slower and can still need a healthy accessibility tree." - ) - } -#endif - -#if os(iOS) - // `waitForTextEntryReadiness`'s hardware-keyboard fallback returns early only on confirmed - // focus (#1874), and `keyboardFocusConfirmed` reads that from the app-wide focus predicate this - // bundle otherwise refuses to trust. Two XCTest facts it rests on, neither a repository - // invariant: the predicate reports a responder that shows NO software keyboard at all, and it - // names the element well enough to tell the tapped field from another one. The fixture field is - // the exact shape the fallback exists for — a real responder with an empty `inputView` — so this - // is where both are observable. If either regressed, readiness would silently stop taking the - // fallback and spend the full readinessTimeout on every hardware-keyboard field, which no other - // assertion would notice. - func testHardwareKeyboardResponderConfirmsItsOwnKeyboardFocus() throws { - app.launchArguments = ["--agent-device-text-entry-regression"] - app.launch() - defer { - invalidateCachedTarget(reason: "unit_test_cleanup") - app.terminate() - } - XCTAssertTrue(app.waitForExistence(timeout: appExistenceTimeout)) - - let textField = app.textFields["agent-device-hardware-keyboard-input"] - XCTAssertTrue(textField.waitForExistence(timeout: appExistenceTimeout)) - let otherElement = app.staticTexts["Agent Device Runner"] - XCTAssertTrue(otherElement.waitForExistence(timeout: appExistenceTimeout)) - XCTAssertFalse( - keyboardFocusConfirmed(app: app, element: textField), - "an untapped field must not confirm focus, or the fallback would fire immediately" - ) - - let tapCommand = try runnerCommandFixture( - #"{"command":"tap","commandId":"tap-focus-confirmation","selectorKey":"id","selectorValue":"agent-device-hardware-keyboard-input"}"# - ) - let tapResponse = try executeOnMainPrepared(command: tapCommand, activeApp: app) - XCTAssertTrue(tapResponse.ok, String(describing: tapResponse.error)) - try XCTSkipIf( - isKeyboardVisible(app: app), - "software keyboard is up: this simulator cannot exercise the hidden-keyboard responder path" - ) - - let deadline = Date().addingTimeInterval(TextEntryTiming.readinessTimeout) - var confirmed = keyboardFocusConfirmed(app: app, element: textField) - while !confirmed && Date() < deadline { - sleepFor(TextEntryTiming.pollInterval) - confirmed = keyboardFocusConfirmed(app: app, element: textField) - } - XCTAssertTrue(confirmed, "a tapped responder must confirm its own keyboard focus") - XCTAssertFalse( - keyboardFocusConfirmed(app: app, element: otherElement), - "focus held by another element must read as a refusal, never as this element's focus" - ) - } -#endif - - func testInjectedTapRecordedFailureGateIsTapOnlyAndCountGated() { - // The seam's recording side cannot run in-bundle (a real XCTIssue would - // fail this very test run — same constraint the record(_:) suppression - // tests document); the live daemon proof covers it. This pins the gate. - XCTAssertFalse(RunnerTests.shouldInjectTapRecordedFailure(command: .tap, remaining: 0)) - XCTAssertTrue(RunnerTests.shouldInjectTapRecordedFailure(command: .tap, remaining: 1)) - XCTAssertFalse(RunnerTests.shouldInjectTapRecordedFailure(command: .type, remaining: 1)) - XCTAssertFalse(RunnerTests.shouldInjectTapRecordedFailure(command: .snapshot, remaining: 1)) - } - - func testXCTestRecordedFailureResponseFailsMutatingSuccesses() throws { - let command = try runnerCommandFixture(#"{"command":"tap","commandId":"tap-1"}"#) - let response = Response(ok: true, data: DataPayload(message: "tapped")) - - let failureResponse = xctestRecordedFailureResponse(command: command, response: response) - - XCTAssertEqual(failureResponse?.ok, false) - XCTAssertEqual(failureResponse?.error?.code, "XCTEST_RECORDED_FAILURE") - XCTAssertEqual( - failureResponse?.error?.message, - "XCTest recorded a failure while executing tap; the action may not have been performed." - ) - } - - func testXCTestRecordedFailureResponseFailsActionButtonSuccess() throws { - // The Action Button press carries no settle and no post-action observation, so this conversion is - // the only evidence the press landed. That is why the press is not classified runner-lifecycle: - // `isLifecycle` would silence the conversion here (#2699, #2702 review). - let command = try runnerCommandFixture(#"{"command":"actionButton","commandId":"action-button-1"}"#) - let response = Response(ok: true, data: DataPayload(message: "actionButton")) - - let failureResponse = xctestRecordedFailureResponse(command: command, response: response) - - XCTAssertEqual(failureResponse?.ok, false) - XCTAssertEqual(failureResponse?.error?.code, "XCTEST_RECORDED_FAILURE") - XCTAssertEqual( - failureResponse?.error?.message, - "XCTest recorded a failure while executing actionButton; the action may not have been performed." - ) - } - - func testXCTestRecordedFailureResponseDoesNotWrapReadOnlyOrRunnerFatalResponses() throws { - let snapshotCommand = try runnerCommandFixture(#"{"command":"snapshot","commandId":"snapshot-1"}"#) - let tapCommand = try runnerCommandFixture(#"{"command":"tap","commandId":"tap-1"}"#) - let runnerFatalResponse = Response( - ok: true, - data: DataPayload(runnerFatal: true, runnerFatalReason: "ax_snapshot_unavailable") - ) - - XCTAssertNil( - xctestRecordedFailureResponse( - command: snapshotCommand, - response: Response(ok: true, data: DataPayload(nodes: [], truncated: false)) - ) - ) - XCTAssertNil(xctestRecordedFailureResponse(command: tapCommand, response: runnerFatalResponse)) - } - - // Simulator-only from here to the matching #endif: these launch the host app, route through - // SpringBoard, or assert the iOS-only alert/system-modal branches. Tests outside the - // `os(iOS)` regions in this file are pure runner decisions and also run on the macOS host - // lane (ci.yml) — see the classification convention in RunnerTests.swift. -#if os(iOS) - func testMissingBundleCommandInvalidatesCompleteCachedTargetState() throws { - app.launch() - currentApp = app - currentBundleId = "com.example.stale-target" - currentAppProcessIdentifier = 42 - snapshotXCTestPenaltyWarmupExemptionPending = true - defer { - invalidateCachedTarget(reason: "unit_test_cleanup") - app.terminate() - } - let command = try runnerCommandFixture( - #"{"command":"snapshot","commandId":"snapshot-without-bundle"}"# - ) - - _ = prepareActiveCommandContext(command: command) - - XCTAssertNil(currentApp) - XCTAssertNil(currentBundleId) - XCTAssertNil(currentAppProcessIdentifier) - XCTAssertFalse(snapshotXCTestPenaltyWarmupExemptionPending) - } - - func testSkipAppActivationPreflightIncludesForegroundCachedCoordinateOnlyTaps() throws { - app.launch() - currentApp = app - currentBundleId = nil - defer { - currentApp = nil - currentBundleId = nil - app.terminate() - } - let tap = try runnerCommandFixture( - #"{"command":"tap","commandId":"tap-1","x":10,"y":20}"# - ) - - XCTAssertTrue(shouldSkipAppActivationPreflight(tap)) - } - - func testSkipAppActivationPreflightRejectsMissingChangedAndBackgroundTargets() throws { - let coordinateTap = try runnerCommandFixture( - #"{"command":"tap","commandId":"tap-1","x":10,"y":20}"# - ) - currentApp = nil - currentBundleId = nil - XCTAssertFalse(shouldSkipAppActivationPreflight(coordinateTap)) - - app.launch() - currentApp = app - currentBundleId = "com.example.current" - defer { - currentApp = nil - currentBundleId = nil - app.terminate() - } - let changedBundleTap = try runnerCommandFixture( - #"{"command":"tap","commandId":"tap-2","appBundleId":"com.example.other","x":10,"y":20}"# - ) - - XCTAssertFalse(shouldSkipAppActivationPreflight(changedBundleTap)) - - app.terminate() - currentApp = app - currentBundleId = nil - - XCTAssertFalse(shouldSkipAppActivationPreflight(coordinateTap)) - } - - func testActionButtonPressSkipsAppActivationPreflightWithoutBeingRunnerLifecycle() throws { - currentApp = nil - currentBundleId = nil - let press = try runnerCommandFixture(#"{"command":"actionButton","commandId":"action-button-1"}"#) - - // The skip is its own decision, reached without the lifecycle flag that would also drop the - // recorded-failure conversion; no cached target and no foreground app is required for it. - XCTAssertFalse(isRunnerLifecycleCommand(.actionButton)) - XCTAssertTrue(shouldSkipAppActivationPreflight(press)) - } - - func testPrepareActiveCommandContextRoutesBlockingSystemModalToSpringboard() throws { - blockingSystemModalPresenceOverrideForTesting = true - currentApp = nil - currentBundleId = nil - defer { - blockingSystemModalPresenceOverrideForTesting = nil - currentApp = nil - currentBundleId = nil - } - let tap = try runnerCommandFixture( - #"{"command":"tap","commandId":"tap-1","x":10,"y":20}"# - ) - - let preparation = prepareActiveCommandContext( - command: tap, - routeToSpringboard: shouldRouteToSpringboardBlockingSystemModal(tap) - ) - - guard case .context(let context) = preparation else { - XCTFail("expected command context") - return - } - XCTAssertTrue(context.app === springboard) - } - - func testExecuteDispatchedReturnsBusyBeforeBlockingSystemModalProbeDrains() throws { - app.launch() - currentApp = app - currentBundleId = nil - defer { - currentApp = nil - currentBundleId = nil - systemModalProbeOverrideForTesting = nil - clearSnapshotXCTestChannelPenalty(reason: "test-cleanup") - app.terminate() - } - - final class ResultBox { - var response: Response? - var error: Error? - var commandRecoveredBeforeRelease = false - var wasBusyBeforeRelease = false - var hadAbandonedProbeBeforeRelease = false - var drained = false - } - let box = ResultBox() - let probeStarted = expectation(description: "system-modal routing probe started") - let verificationFinished = expectation(description: "command recovery and modal probe drain verified") - let probeReleaseGate = DispatchSemaphore(value: 0) - let commandFinishedGate = DispatchSemaphore(value: 0) - systemModalProbeOverrideForTesting = { _ in - probeStarted.fulfill() - _ = probeReleaseGate.wait(timeout: .now() + 15) - return DataPayload(message: "late system modal") - } - - let command = try runnerCommandFixture( - #"{"command":"tap","commandId":"bounded-modal-routing","x":10,"y":20}"# - ) - DispatchQueue(label: "agent-device.runner.tests.modal-routing-probe").async { - do { - box.response = try self.executeDispatched(command: command) - } catch { - box.error = error - } - commandFinishedGate.signal() - } - DispatchQueue(label: "agent-device.runner.tests.modal-routing-probe-verifier").async { - let commandWait = commandFinishedGate.wait( - timeout: .now() + self.systemModalProbeBudget + 3 - ) - box.commandRecoveredBeforeRelease = commandWait == .success - && box.error == nil - && box.response?.error?.code == "RUNNER_BUSY" - if case .busy = self.currentMainThreadBusyState() { - box.wasBusyBeforeRelease = true - } - box.hadAbandonedProbeBeforeRelease = self.hasAbandonedMainThreadWork() - - // The XCTest main thread is blocked inside the injected probe, so this verifier owns the - // ordered release after recording the command result and abandoned-work state above. - probeReleaseGate.signal() - let deadline = Date().addingTimeInterval(5) - while self.hasAbandonedMainThreadWork(), Date() < deadline { - self.sleepFor(0.002) - } - box.drained = !self.hasAbandonedMainThreadWork() - verificationFinished.fulfill() - } - - wait(for: [probeStarted, verificationFinished], timeout: 15) - XCTAssertTrue( - box.commandRecoveredBeforeRelease, - "the public coordinate tap must return RUNNER_BUSY before the blocked modal probe drains" - ) - XCTAssertTrue(box.wasBusyBeforeRelease) - XCTAssertTrue(box.hadAbandonedProbeBeforeRelease) - XCTAssertTrue(box.drained) - guard case .idle = currentMainThreadBusyState() else { - return XCTFail("expected the runner to become idle after the routing probe drained") - } - XCTAssertFalse(hasAbandonedMainThreadWork()) - } - - func testSkipAppActivationPreflightRejectsSelectorAndMixedSequenceGestures() throws { - app.launch() - currentApp = app - currentBundleId = nil - defer { - currentApp = nil - currentBundleId = nil - app.terminate() - } - let selectorTap = try runnerCommandFixture( - #"{"command":"tap","commandId":"tap-1","selectorKey":"label","selectorValue":"Search","synthesized":true}"# - ) - let standardDrag = try runnerCommandFixture( - #"{"command":"drag","commandId":"drag-1","x":10,"y":20,"x2":30,"y2":40}"# - ) - let mixedSequence = try runnerCommandFixture( - """ - {"command":"sequence","commandId":"seq-1","steps":[ - {"kind":"tap","x":10,"y":20,"synthesized":true}, - {"kind":"doubleTap","x":30,"y":40} - ]} - """ - ) - - XCTAssertFalse(shouldSkipAppActivationPreflight(selectorTap)) - XCTAssertFalse(shouldSkipAppActivationPreflight(standardDrag)) - XCTAssertFalse(shouldSkipAppActivationPreflight(mixedSequence)) - } - - // Launches nothing, but still simulator-only: `shouldSkipAppActivationPreflight` is - // `#if os(iOS) …guards… #else return false #endif`, so on macOS this asserts a compile-time - // literal and no edit to the iOS body could make it red. Its five siblings above and below - // are gated for the same reason. - func testSkipAppActivationPreflightRequiresCachedForegroundTarget() throws { - currentApp = nil - currentBundleId = nil - let scroll = try runnerCommandFixture( - #"{"command":"scroll","commandId":"scroll-1","direction":"down","pixels":400}"# - ) - - XCTAssertFalse(shouldSkipAppActivationPreflight(scroll)) - } - - func testSkipAppActivationPreflightKeepsDragScrollAndSequenceOnForegroundGuard() throws { - app.launch() - currentApp = app - currentBundleId = nil - defer { - currentApp = nil - currentBundleId = nil - app.terminate() - } - let drag = try runnerCommandFixture( - #"{"command":"drag","commandId":"drag-1","x":10,"y":20,"x2":30,"y2":40}"# - ) - let scroll = try runnerCommandFixture( - #"{"command":"scroll","commandId":"scroll-1","direction":"down","pixels":400}"# - ) - let sequence = try runnerCommandFixture( - """ - {"command":"sequence","commandId":"seq-1","steps":[ - {"kind":"tap","x":10,"y":20,"synthesized":true}, - {"kind":"longPress","x":10,"y":200,"durationMs":300} - ]} - """ - ) - - XCTAssertFalse(shouldSkipAppActivationPreflight(drag)) - XCTAssertFalse(shouldSkipAppActivationPreflight(scroll)) - XCTAssertFalse(shouldSkipAppActivationPreflight(sequence)) - } - - func testSkipAppActivationPreflightIncludesAlertCommands() throws { - let alert = try runnerCommandFixture( - #"{"command":"alert","commandId":"alert-1","action":"get"}"# - ) - - XCTAssertTrue(shouldSkipAppActivationPreflight(alert)) - } -#endif - - func testDispatchReturnsBusyBeforeQueueingMainThreadWork() throws { - let command = try runnerCommandFixture(#"{"command":"snapshot","commandId":"snapshot-busy"}"#) - abandonedMainThreadWorkCount = 1 - abandonedMainThreadWorkSince = Date(timeIntervalSinceNow: -2) - defer { - abandonedMainThreadWorkCount = 0 - abandonedMainThreadWorkSince = nil - } - - let response = try execute(command: command) - - XCTAssertFalse(response.ok) - XCTAssertEqual(response.error?.code, "RUNNER_BUSY") - XCTAssertTrue(response.error?.message.contains("previous command") == true) - } - - func testDispatchReturnsWedgedBeforeQueueingMainThreadWork() throws { - let command = try runnerCommandFixture(#"{"command":"snapshot","commandId":"snapshot-wedged"}"#) - abandonedMainThreadWorkCount = 1 - abandonedMainThreadWorkSince = Date(timeIntervalSinceNow: -(mainThreadWedgeThreshold + 1)) - defer { - abandonedMainThreadWorkCount = 0 - abandonedMainThreadWorkSince = nil - } - - let response = try execute(command: command) - - XCTAssertFalse(response.ok) - XCTAssertEqual(response.error?.code, "RUNNER_WEDGED") - XCTAssertTrue(response.error?.hint?.contains("runner session will be restarted") == true) - } - - func testPostSnapshotDelayMarkDoesNotQueueBehindAbandonedMainThreadWork() { - abandonedMainThreadWorkCount = 1 - defer { - abandonedMainThreadWorkCount = 0 - needsPostSnapshotInteractionDelay = false - } - - let finished = expectation(description: "off-main caller finished") - DispatchQueue(label: "agent-device.runner.tests.post-snapshot-delay").async { - self.setNeedsPostSnapshotInteractionDelay() - finished.fulfill() - } - - wait(for: [finished], timeout: 1) - mainThreadWorkLock.lock() - let abandonedWorkCount = abandonedMainThreadWorkCount - mainThreadWorkLock.unlock() - XCTAssertEqual(abandonedWorkCount, 1, "the skipped mark must not add an abandoned unit") - XCTAssertFalse(needsPostSnapshotInteractionDelay) - } - - func testSnapshotFailureInvalidationQueuesBehindAbandonedMainThreadWorkWithoutWaiting() { - currentBundleId = "com.example.stale-target" - defer { currentBundleId = nil } - - final class ResultBox { - var elapsed: TimeInterval? - var bundleStillCachedWhileBlocked: Bool? - var abandonedWhileBlocked: Int? - } - let box = ResultBox() - let mainBlocked = DispatchSemaphore(value: 0) - let releaseMain = DispatchSemaphore(value: 0) - let finished = expectation(description: "invalidation returned while main was blocked") - - DispatchQueue(label: "agent-device.runner.tests.snapshot-invalidation").async { - _ = try? self.runMainThreadWork( - "command_execution", - timeout: 0, - timeoutError: self.mainThreadExecutionTimeoutError - ) { - mainBlocked.signal() - _ = releaseMain.wait(timeout: .now() + 5) - return true - } - _ = mainBlocked.wait(timeout: .now() + 2) - let startedAt = Date() - self.invalidateCachedTargetAfterSnapshotFailure() - box.elapsed = Date().timeIntervalSince(startedAt) - box.bundleStillCachedWhileBlocked = self.currentBundleId != nil - self.mainThreadWorkLock.lock() - box.abandonedWhileBlocked = self.abandonedMainThreadWorkCount - self.mainThreadWorkLock.unlock() - releaseMain.signal() - finished.fulfill() - } - - wait(for: [finished], timeout: 8) - let drainDeadline = Date().addingTimeInterval(2) - while hasAbandonedMainThreadWork() || currentBundleId != nil, Date() < drainDeadline { - sleepFor(0.005) - } - - XCTAssertLessThan( - box.elapsed ?? .infinity, - 0.5, - "the failed capture must not wait behind abandoned main-thread work" - ) - XCTAssertEqual( - box.bundleStillCachedWhileBlocked, - true, - "the drop must queue behind the blocked main thread, not run early" - ) - XCTAssertEqual(box.abandonedWhileBlocked, 1, "the deferred drop must not add an abandoned unit") - XCTAssertFalse(hasAbandonedMainThreadWork()) - XCTAssertNil(currentBundleId, "the drop must run once the main thread frees") - } -#endif - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS - /// Routes `command` through the transport's inline and queued paths. The calling test's main - /// thread serves the command's main-thread work while it waits. - func execute(command: Command) throws -> Response { - dispatchPrecondition(condition: .onQueue(.main)) - if let response = inlineResponse(for: command) { - return response - } - final class ResultBox { - var result: Result? - } - let box = ResultBox() - let executed = XCTestExpectation(description: "\(command.command.rawValue) executed off main") - enqueueAccepted(command: command) { result in - box.result = result - executed.fulfill() - } - guard XCTWaiter.wait(for: [executed], timeout: mainThreadExecutionTimeout + 5) == .completed, - let result = box.result - else { - throw NSError( - domain: RunnerErrorDomain.general, - code: RunnerErrorCode.commandReturnedNoResponse, - userInfo: [NSLocalizedDescriptionKey: "command did not finish on the command queue"] - ) - } - return try result.get() - } -#endif - func executeAccepted(command: Command) throws -> Response { commandJournal.start(command: command) pendingTargetActivation = nil @@ -967,7 +269,7 @@ extension RunnerTests { } } - private func executeDispatched(command: Command) throws -> Response { + func executeDispatched(command: Command) throws -> Response { // XCTest work cannot be cancelled mid-flight: once the watchdog abandons a main-queue // block, queueing more main-thread commands behind it only buries the runner deeper. // Refuse fast instead so the daemon backs off while the abandoned work drains; past the @@ -1238,7 +540,7 @@ extension RunnerTests { } } - private func setNeedsPostSnapshotInteractionDelay() { + func setNeedsPostSnapshotInteractionDelay() { guard !hasAbandonedMainThreadWork() else { NSLog("AGENT_DEVICE_RUNNER_POST_SNAPSHOT_DELAY_MARK_SKIPPED_XCTEST_OCCUPIED") return @@ -1256,7 +558,7 @@ extension RunnerTests { } } - private func invalidateCachedTargetAfterSnapshotFailure() { + func invalidateCachedTargetAfterSnapshotFailure() { // Abandoned work ahead of this hop cannot be cancelled: queue the drop behind it without // waiting, so the failed capture answers now and the next command still finds the target gone. guard !hasAbandonedMainThreadWork() else { @@ -1424,7 +726,7 @@ extension RunnerTests { ) } - private func prepareActiveCommandContext( + func prepareActiveCommandContext( command: Command, routeToSpringboard: Bool = false ) -> ActiveCommandPreparation { @@ -2446,7 +1748,7 @@ extension RunnerTests { return currentXCTestFailureCount() > failureCountBefore } - private func xctestRecordedFailureResponse(command: Command, response: Response) -> Response? { + func xctestRecordedFailureResponse(command: Command, response: Response) -> Response? { guard response.ok else { return nil } if response.data?.runnerFatal == true { return nil @@ -2464,13 +1766,7 @@ extension RunnerTests { ) } -#if AGENT_DEVICE_RUNNER_UNIT_TESTS - func runnerCommandFixture(_ json: String) throws -> Command { - try JSONDecoder().decode(Command.self, from: Data(json.utf8)) - } -#endif - - private func shouldSkipAppActivationPreflight(_ command: Command) -> Bool { + func shouldSkipAppActivationPreflight(_ command: Command) -> Bool { #if os(iOS) if command.command == .alert { return true @@ -2495,7 +1791,7 @@ extension RunnerTests { #endif } - private func shouldRouteToSpringboardBlockingSystemModal( + func shouldRouteToSpringboardBlockingSystemModal( _ command: Command ) -> Bool { #if os(iOS) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandJournal.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandJournal.swift index 9ed3b9f51f..c91955116d 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandJournal.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandJournal.swift @@ -144,340 +144,3 @@ final class RunnerCommandJournal { } } } - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS -extension RunnerTests { - func testUptimeBypassesCommandJournal() throws { - let command = runnerJournalCommand("uptime", id: "uptime-probe") - - let response = try execute(command: command) - let status = commandJournal.status(normalizedCommandId: "uptime-probe") - - XCTAssertEqual(response.ok, true) - XCTAssertNotNil(response.data?.currentUptimeMs) - XCTAssertEqual(status.lifecycleState, RunnerCommandLifecycleState.notAccepted.rawValue) - } - - func testStampingCurrentUptimePreservesPayload() { - let stamped = Response(ok: true, data: DataPayload(message: "recording started")) - .stampingCurrentUptimeMs(123.5) - - XCTAssertEqual(stamped.ok, true) - XCTAssertEqual(stamped.data?.message, "recording started") - XCTAssertEqual(stamped.data?.currentUptimeMs, 123.5) - } - - func testStampingCurrentUptimeCreatesPayloadWhenNil() { - let stamped = Response(ok: true).stampingCurrentUptimeMs(456.0) - - XCTAssertEqual(stamped.ok, true) - XCTAssertEqual(stamped.data?.currentUptimeMs, 456.0) - } - - func testStampingCurrentUptimeSkipsErrorResponses() { - let response = Response(ok: false, error: ErrorPayload(message: "boom")) - let stamped = response.stampingCurrentUptimeMs(789.0) - - XCTAssertEqual(stamped.ok, false) - XCTAssertNil(stamped.data) - XCTAssertEqual(stamped.error?.message, "boom") - } - - func testStampingCurrentMainThreadBusyPreservesPayload() { - let stamped = Response(ok: true, data: DataPayload(nodes: [], truncated: false)) - .stampingCurrentMainThreadBusy(true) - - XCTAssertEqual(stamped.ok, true) - XCTAssertEqual(stamped.data?.runnerMainThreadBusy, true) - } - - func testStampingCurrentMainThreadBusySkipsErrorResponses() { - let response = Response(ok: false, error: ErrorPayload(code: "RUNNER_BUSY", message: "busy")) - let stamped = response.stampingCurrentMainThreadBusy(true) - - XCTAssertEqual(stamped.ok, false) - XCTAssertNil(stamped.data) - XCTAssertEqual(stamped.error?.code, "RUNNER_BUSY") - } - - func testMainThreadBusyStateReportsOccupancy() { - XCTAssertFalse(MainThreadBusyState.idle.reportsMainThreadBusy) - XCTAssertTrue(MainThreadBusyState.busy(abandonedForSeconds: 5).reportsMainThreadBusy) - XCTAssertTrue(MainThreadBusyState.wedged(abandonedForSeconds: 200).reportsMainThreadBusy) - XCTAssertEqual( - Response(ok: true).stampingCurrentMainThreadBusy(false).data?.runnerMainThreadBusy, false) - } - - func testCommandFailedResponseTagsMainThreadTimeoutWithTypedCode() { - let timeout = NSError( - domain: RunnerErrorDomain.general, - code: RunnerErrorCode.mainThreadExecutionTimedOut, - userInfo: [NSLocalizedDescriptionKey: "main thread execution timed out"] - ) - - let response = commandFailedResponse(from: timeout) - - XCTAssertEqual(response.ok, false) - XCTAssertEqual(response.error?.code, RunnerWireErrorCode.mainThreadTimeout) - } - - func testCommandFailedResponseKeepsGenericCodeForOtherErrors() { - let other = NSError(domain: "SomeOtherDomain", code: 99, userInfo: nil) - - let response = commandFailedResponse(from: other) - - XCTAssertEqual(response.error?.code, "COMMAND_FAILED") - } - - func testJournalStoredResponseStaysUnstamped() throws { - let journal = RunnerCommandJournal() - let recordStart = runnerJournalCommand("recordStart", id: "record-start-anchor") - - journal.accept(command: recordStart) - journal.finish( - command: recordStart, - response: Response(ok: true, data: DataPayload(message: "recording started")) - ) - - let status = journal.status(normalizedCommandId: "record-start-anchor") - let responseJson = try XCTUnwrap(status.lifecycleResponseJson) - XCTAssertFalse(responseJson.contains("currentUptimeMs")) - } - - func testCommandJournalRetentionPolicy() throws { - let journal = RunnerCommandJournal() - - let uptime = runnerJournalCommand("uptime", id: "small-scalar") - journal.accept(command: uptime) - journal.finish( - command: uptime, - response: Response(ok: true, data: DataPayload(currentUptimeMs: 12.5)) - ) - - let scalarStatus = journal.status(normalizedCommandId: "small-scalar") - XCTAssertEqual(scalarStatus.lifecycleState, RunnerCommandLifecycleState.completed.rawValue) - XCTAssertEqual(scalarStatus.lifecycleResponseOk, true) - XCTAssertNotNil(scalarStatus.lifecycleResponseJson) - let scalarResponse = try decodeRunnerJournalResponse(scalarStatus.lifecycleResponseJson) - XCTAssertEqual(scalarResponse.data?.currentUptimeMs, 12.5) - - let querySelector = runnerJournalCommand("querySelector", id: "small-object") - journal.accept(command: querySelector) - journal.finish( - command: querySelector, - response: Response(ok: true, data: DataPayload(found: true, nodes: [runnerJournalNode()])) - ) - - let objectStatus = journal.status(normalizedCommandId: "small-object") - XCTAssertNotNil(objectStatus.lifecycleResponseJson) - let objectResponse = try decodeRunnerJournalResponse(objectStatus.lifecycleResponseJson) - XCTAssertEqual(objectResponse.data?.found, true) - XCTAssertEqual(objectResponse.data?.nodes?.count, 1) - - let snapshot = runnerJournalCommand("snapshot", id: "snapshot-tree") - journal.accept(command: snapshot) - journal.finish( - command: snapshot, - response: Response(ok: true, data: DataPayload(nodes: [runnerJournalNode()], truncated: false)) - ) - - let snapshotStatus = journal.status(normalizedCommandId: "snapshot-tree") - XCTAssertEqual(snapshotStatus.lifecycleState, RunnerCommandLifecycleState.completed.rawValue) - XCTAssertEqual(snapshotStatus.lifecycleResponseOk, true) - XCTAssertNil(snapshotStatus.lifecycleResponseJson) - - let screenshot = runnerJournalCommand("screenshot", id: "screenshot-artifact") - journal.accept(command: screenshot) - journal.finish( - command: screenshot, - response: Response(ok: true, data: DataPayload(message: "tmp/screenshot-1.png")) - ) - - let screenshotStatus = journal.status(normalizedCommandId: "screenshot-artifact") - XCTAssertEqual(screenshotStatus.lifecycleState, RunnerCommandLifecycleState.completed.rawValue) - XCTAssertEqual(screenshotStatus.lifecycleResponseOk, true) - XCTAssertNil(screenshotStatus.lifecycleResponseJson) - - let scroll = runnerJournalCommand("scroll", id: "scroll-drag") - journal.accept(command: scroll) - journal.finish( - command: scroll, - response: Response( - ok: true, - data: DataPayload( - message: "scrolled", - gestureStartUptimeMs: 1, - gestureEndUptimeMs: 2, - x: 155, - y: 420, - x2: 155, - y2: 301, - referenceWidth: 300, - referenceHeight: 600 - ) - ) - ) - - let scrollStatus = journal.status(normalizedCommandId: "scroll-drag") - XCTAssertEqual(scrollStatus.lifecycleState, RunnerCommandLifecycleState.completed.rawValue) - XCTAssertEqual(scrollStatus.lifecycleResponseOk, true) - XCTAssertNotNil(scrollStatus.lifecycleResponseJson) - let scrollResponse = try decodeRunnerJournalResponse(scrollStatus.lifecycleResponseJson) - XCTAssertEqual(scrollResponse.data?.x, 155) - XCTAssertEqual(scrollResponse.data?.y, 420) - XCTAssertEqual(scrollResponse.data?.x2, 155) - XCTAssertEqual(scrollResponse.data?.y2, 301) - XCTAssertEqual(scrollResponse.data?.referenceWidth, 300) - XCTAssertEqual(scrollResponse.data?.referenceHeight, 600) - - let largeRead = runnerJournalCommand("readText", id: "large-read") - journal.accept(command: largeRead) - journal.finish( - command: largeRead, - response: Response(ok: true, data: DataPayload(text: String(repeating: "x", count: 17 * 1024))) - ) - - let largeReadStatus = journal.status(normalizedCommandId: "large-read") - XCTAssertEqual(largeReadStatus.lifecycleState, RunnerCommandLifecycleState.completed.rawValue) - XCTAssertEqual(largeReadStatus.lifecycleResponseOk, true) - XCTAssertNil(largeReadStatus.lifecycleResponseJson) - } - - func testCommandJournalKeepsErrorMetadataWhenResponseJsonIsDropped() { - let journal = RunnerCommandJournal() - let snapshot = runnerJournalCommand("snapshot", id: "snapshot-error") - let hint = "Try a smaller read such as snapshot -s -d 8." - - journal.accept(command: snapshot) - journal.finish( - command: snapshot, - response: Response( - ok: false, - error: ErrorPayload( - code: "IOS_AX_SNAPSHOT_FAILED", - message: "iOS XCTest snapshot failed while serializing the accessibility tree.", - hint: hint - ) - ) - ) - - let status = journal.status(normalizedCommandId: "snapshot-error") - XCTAssertEqual(status.lifecycleState, RunnerCommandLifecycleState.failed.rawValue) - XCTAssertEqual(status.lifecycleResponseOk, false) - XCTAssertNil(status.lifecycleResponseJson) - XCTAssertEqual(status.lifecycleErrorCode, "IOS_AX_SNAPSHOT_FAILED") - XCTAssertEqual( - status.lifecycleErrorMessage, - "iOS XCTest snapshot failed while serializing the accessibility tree." - ) - XCTAssertEqual(status.lifecycleErrorHint, hint) - } - - func testCommandJournalRetainsCompletedSequenceResults() throws { - let journal = RunnerCommandJournal() - let sequence = runnerJournalCommand("sequence", id: "sequence-completed") - let results = (0..<20).map { _ in - SequenceStepResult( - ok: true, - kind: "tap", - errorCode: nil, - errorMessage: nil, - gestureStartUptimeMs: 100, - gestureEndUptimeMs: 120 - ) - } - - journal.accept(command: sequence) - journal.finish( - command: sequence, - response: Response( - ok: true, - data: DataPayload( - message: "sequence", - completedSteps: 20, - failedStepIndex: nil, - sequenceResults: results - ) - ) - ) - - let status = journal.status(normalizedCommandId: "sequence-completed") - XCTAssertEqual(status.lifecycleState, RunnerCommandLifecycleState.completed.rawValue) - XCTAssertEqual(status.lifecycleResponseOk, true) - let json = try XCTUnwrap(status.lifecycleResponseJson) - // Worst-case 20-step response must stay under the 16KB journal retention cap. - XCTAssertLessThan(json.utf8.count, 16 * 1024) - let decoded = try decodeRunnerJournalResponse(status.lifecycleResponseJson) - XCTAssertEqual(decoded.data?.completedSteps, 20) - XCTAssertEqual(decoded.data?.sequenceResults?.count, 20) - } - - func testCommandJournalRetainsFailedSequenceResults() throws { - let journal = RunnerCommandJournal() - let sequence = runnerJournalCommand("sequence", id: "sequence-failed") - let longError = String(repeating: "z", count: 200) - let results: [SequenceStepResult] = [ - SequenceStepResult(ok: true, kind: "tap", errorCode: nil, errorMessage: nil, - gestureStartUptimeMs: 100, gestureEndUptimeMs: 120), - SequenceStepResult(ok: true, kind: "tap", errorCode: nil, errorMessage: nil, - gestureStartUptimeMs: 130, gestureEndUptimeMs: 150), - SequenceStepResult(ok: false, kind: "longPress", errorCode: "UNSUPPORTED_OPERATION", - errorMessage: longError, gestureStartUptimeMs: 160, gestureEndUptimeMs: 180), - ] - - journal.accept(command: sequence) - journal.finish( - command: sequence, - response: Response( - ok: true, - data: DataPayload( - message: "sequence", - completedSteps: 2, - failedStepIndex: 2, - sequenceResults: results - ) - ) - ) - - let status = journal.status(normalizedCommandId: "sequence-failed") - XCTAssertEqual(status.lifecycleState, RunnerCommandLifecycleState.completed.rawValue) - let decoded = try decodeRunnerJournalResponse(status.lifecycleResponseJson) - XCTAssertEqual(decoded.data?.completedSteps, 2) - XCTAssertEqual(decoded.data?.failedStepIndex, 2) - XCTAssertEqual(decoded.data?.sequenceResults?.count, 3) - XCTAssertEqual(decoded.data?.sequenceResults?[2].ok, false) - XCTAssertEqual(decoded.data?.sequenceResults?[2].errorCode, "UNSUPPORTED_OPERATION") - } - - private func runnerJournalCommand(_ command: String, id: String) -> Command { - let json = #"{"command":"\#(command)","commandId":"\#(id)"}"# - return try! JSONDecoder().decode(Command.self, from: Data(json.utf8)) - } - - private func runnerJournalNode() -> PresentedNode { - SnapshotPresentation.singleElementRead( - RawAXNode( - index: 0, - type: "button", - label: "Continue", - identifier: "continue", - value: nil, - rect: SnapshotRect(x: 10, y: 20, width: 100, height: 44), - enabled: true, - focused: nil, - selected: nil, - hittable: true, - depth: 0, - parentIndex: nil, - hiddenContentAbove: nil, - hiddenContentBelow: nil - ) - ) - } - - private func decodeRunnerJournalResponse(_ responseJson: String?) throws -> Response { - let responseJson = try XCTUnwrap(responseJson) - return try JSONDecoder().decode(Response.self, from: Data(responseJson.utf8)) - } -} -#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index 8f0a101429..002459b19c 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -602,27 +602,4 @@ extension RunnerTests { #endif } #endif - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS - func testDesktopScrollWheelDeltasMapDirections() { - XCTAssertEqual(desktopScrollWheelDeltas(direction: .up, pixels: 120).vertical, 120) - XCTAssertEqual(desktopScrollWheelDeltas(direction: .down, pixels: 120).vertical, -120) - XCTAssertEqual(desktopScrollWheelDeltas(direction: .left, pixels: 120).horizontal, 120) - XCTAssertEqual(desktopScrollWheelDeltas(direction: .right, pixels: 120).horizontal, -120) - } - - func testDesktopScrollWheelDeltaEventsHonorDurationAndPreservePixels() { - let events = desktopScrollWheelDeltaEvents(direction: .down, pixels: 200, durationMs: 50) - XCTAssertEqual(events.count, 4) - XCTAssertEqual(events.map(\.vertical).reduce(0, +), -200) - XCTAssertEqual(events.map(\.horizontal).reduce(0, +), 0) - XCTAssertEqual(desktopScrollEventIntervalSeconds(durationMs: 50, eventCount: events.count), 0.05 / 3.0) - } - - func testDesktopScrollWheelDeltaEventsKeepInstantScrollSingleEvent() { - let events = desktopScrollWheelDeltaEvents(direction: .down, pixels: 200, durationMs: 0) - XCTAssertEqual(events.count, 1) - XCTAssertEqual(events.first?.vertical, -200) - } -#endif } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Keyboard.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Keyboard.swift index a3fab3fad7..d743ec2a7c 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Keyboard.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Keyboard.swift @@ -447,145 +447,3 @@ func runnerScreenshotStabilitySettled( guard let first = window.first, let firstData = first else { return false } return window.allSatisfy { $0 == firstData } } - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS -extension RunnerTests { - func testKeyboardBandFactReadFailureIsStatedAsUnmeasurableNotAbsence() { - // Absence would let a consumer claim the screen is clear of a keyboard on the strength of a read - // that never answered. - let fact = runnerKeyboardBandFact( - readSucceeded: false, - exists: false, - frame: CGRect(x: 0, y: 198, width: 402, height: 204) - ) - XCTAssertEqual(fact, .unmeasurable(RunnerKeyboardBandReason.queryFailed)) - XCTAssertEqual(fact.payload.kind, "unmeasurable") - XCTAssertEqual(fact.payload.reason, RunnerKeyboardBandReason.queryFailed) - XCTAssertNil(fact.payload.frame) - } - - func testKeyboardBandFactPublishesAbsenceWhenTheQueryFindsNoKeyboard() { - let fact = runnerKeyboardBandFact(readSucceeded: true, exists: false, frame: .zero) - XCTAssertEqual(fact, .absent) - XCTAssertEqual(fact.payload.kind, "absent") - XCTAssertNil(fact.payload.frame) - XCTAssertNil(fact.payload.reason) - } - - func testKeyboardBandFactPublishesTheMeasuredBandInAppOrientationSpace() { - // The landscape band measured on iPhone 17 Pro (iOS 26.2) after #2653: full width across the - // bottom of a 402 pt-tall app, which is what the tree reported as a strip down the left edge. - let frame = CGRect(x: 0, y: 198, width: 874, height: 204) - let fact = runnerKeyboardBandFact(readSucceeded: true, exists: true, frame: frame) - XCTAssertEqual(fact, .visible(frame)) - let payload = fact.payload - XCTAssertEqual(payload.kind, "visible") - XCTAssertEqual(payload.frame, SnapshotRect(x: 0, y: 198, width: 874, height: 204)) - XCTAssertNil(payload.reason) - } - - func testKeyboardBandFactRefusesUnusableGeometryInsteadOfClaimingAbsence() { - let nan = CGFloat(Double.nan) - let infinite = CGFloat.infinity - let unusable: [CGRect] = [ - .zero, - CGRect(x: 0, y: 198, width: 0, height: 204), - CGRect(x: 0, y: 198, width: 874, height: -1), - CGRect(x: 0, y: 198, width: -874, height: 204), - CGRect(x: 0, y: nan, width: 874, height: 204), - CGRect(x: 0, y: 198, width: nan, height: 204), - CGRect(x: infinite, y: 198, width: 874, height: 204) - ] - for frame in unusable { - let fact = runnerKeyboardBandFact(readSucceeded: true, exists: true, frame: frame) - XCTAssertEqual( - fact, - .unmeasurable(RunnerKeyboardBandReason.unusableFrame), - "expected \(frame) to be refused as a band" - ) - } - } - - func testKeyboardBandFactPayloadRoundTripsThroughTheWireShape() throws { - let cases: [RunnerKeyboardBandFact] = [ - .visible(CGRect(x: 0, y: 583, width: 402, height: 291)), - .absent, - .unmeasurable(RunnerKeyboardBandReason.queryTimeout) - ] - for fact in cases { - let data = try JSONEncoder().encode(fact.payload) - // `encodeIfPresent` for the two optional fields: a fact carries its own evidence and nothing - // else, so the daemon never has to distinguish a null from an absent key. - let object = try XCTUnwrap(JSONSerialization.jsonObject(with: data) as? [String: Any]) - switch fact { - case .visible: - XCTAssertNil(object["reason"]) - XCTAssertNotNil(object["frame"]) - case .absent: - XCTAssertNil(object["frame"]) - XCTAssertNil(object["reason"]) - case .unmeasurable: - XCTAssertNil(object["frame"]) - XCTAssertNotNil(object["reason"]) - } - XCTAssertEqual(try JSONDecoder().decode(KeyboardBandFactPayload.self, from: data), fact.payload) - } - } - - func testRunnerScreenshotStabilitySettledNeedsEnoughSamples() { - XCTAssertFalse(runnerScreenshotStabilitySettled([], requiredConsecutiveMatches: 3)) - XCTAssertFalse(runnerScreenshotStabilitySettled([Data([1])], requiredConsecutiveMatches: 3)) - let frame = Data([1, 2, 3]) - XCTAssertFalse( - runnerScreenshotStabilitySettled([frame, frame], requiredConsecutiveMatches: 3) - ) - } - - func testRunnerScreenshotStabilitySettledTrueWhenWindowMatches() { - let frame = Data([1, 2, 3]) - XCTAssertTrue( - runnerScreenshotStabilitySettled([Data([9]), frame, frame, frame], requiredConsecutiveMatches: 3) - ) - } - - func testRunnerScreenshotStabilitySettledFalseOnMidWindowMismatch() { - // A momentary pause (two matching samples) followed by resumed movement - // must not read as settled: the 3-sample window still spans the mismatch. - let frame = Data([1, 2, 3]) - let moved = Data([4, 5, 6]) - XCTAssertFalse( - runnerScreenshotStabilitySettled([frame, frame, moved], requiredConsecutiveMatches: 3) - ) - } - - func testRunnerScreenshotStabilitySettledFalseOnFailedCapture() { - // A nil sample (failed screenshot) never counts as a match, even against - // other nils — an unverifiable run must not look "stable". - XCTAssertFalse(runnerScreenshotStabilitySettled([nil, nil, nil], requiredConsecutiveMatches: 3)) - let frame = Data([1, 2, 3]) - XCTAssertFalse( - runnerScreenshotStabilitySettled([frame, frame, nil], requiredConsecutiveMatches: 3) - ) - } - - func testRunnerScreenshotStabilitySettledOnlyLooksAtTheTrailingWindow() { - // An older mismatch before the trailing window must not block settlement - // once the required run of most-recent samples agrees. - let frame = Data([9]) - XCTAssertTrue( - runnerScreenshotStabilitySettled( - [Data([1]), Data([2]), frame, frame, frame], - requiredConsecutiveMatches: 3 - ) - ) - } - - func testRunnerScreenshotStabilitySettledRejectsDegenerateRequirement() { - // Fewer than 2 required matches would make any single sample "settled" — - // guard against a misconfigured caller rather than silently no-op the wait. - XCTAssertFalse( - runnerScreenshotStabilitySettled([Data([1])], requiredConsecutiveMatches: 1) - ) - } -} -#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift index 57e4467cd3..0314cbab9b 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift @@ -505,33 +505,3 @@ extension RunnerTests { usleep(useconds_t(delay * 1_000_000)) } } - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS -extension RunnerTests { - func testResettingTargetBoundStateForgetsTheLastWrittenMarkers() { - defer { invalidateCachedTarget(reason: "unit_test_cleanup") } - lastLoggedFastAppGuardLine = "AGENT_DEVICE_RUNNER_FAST_APP_GUARD bundle=app state=4" - lastLoggedGesturePolicyLines[.scroll] = "AGENT_DEVICE_RUNNER_SYNTHESIZED_GESTURE_POLICY kind=scroll" - resetTargetBoundState() - XCTAssertNil(lastLoggedFastAppGuardLine, "a rebind must state the guard once more") - XCTAssertTrue(lastLoggedGesturePolicyLines.isEmpty, "a rebind must state the policy once more") - } - - func testFastAppGuardMarkerWritesOnceUntilTheFactChanges() { - var written: [String] = [] - runnerMarkerWriter = { written.append($0) } - defer { - runnerMarkerWriter = { NSLog("%@", $0) } - invalidateCachedTarget(reason: "unit_test_cleanup") - } - writeFastAppGuardMarker(bundleId: "com.example.app", state: .runningForeground) - writeFastAppGuardMarker(bundleId: "com.example.app", state: .runningForeground) - XCTAssertEqual(written.count, 1, "a repeated fact writes no second line") - writeFastAppGuardMarker(bundleId: "com.example.other", state: .runningForeground) - XCTAssertEqual(written.count, 2, "a changed fact writes a new line") - resetTargetBoundState() - writeFastAppGuardMarker(bundleId: "com.example.other", state: .runningForeground) - XCTAssertEqual(written.count, 3, "a rebind states the same fact once more") - } -} -#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+MainThreadWork.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+MainThreadWork.swift index c0dafe8f05..d624b10ba6 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+MainThreadWork.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+MainThreadWork.swift @@ -135,95 +135,3 @@ extension RunnerTests { ) } } - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS -extension RunnerTests { - func testRunMainThreadWorkExecutesOffMainCallerOnMainThread() { - final class ResultBox { - var observedMainThread: Bool? - var error: Error? - } - let box = ResultBox() - let finished = expectation(description: "off-main caller finished") - - DispatchQueue(label: "agent-device.runner.tests.off-main").async { - do { - box.observedMainThread = try self.runMainThreadWork( - "command_execution", - timeout: 1, - timeoutError: self.mainThreadExecutionTimeoutError - ) { - Thread.isMainThread - } - } catch { - box.error = error - } - finished.fulfill() - } - - wait(for: [finished], timeout: 2) - XCTAssertNil(box.error) - XCTAssertEqual(box.observedMainThread, true) - } - - func testRunMainThreadWorkTimeoutMarksAbandonedUntilDrained() { - final class ResultBox { - var error: Error? - var abandonedCount: Int? - var abandonedSinceSet: Bool? - var busyWhileAbandoned = false - } - let box = ResultBox() - let releaseWork = DispatchSemaphore(value: 0) - let observedAbandoned = DispatchSemaphore(value: 0) - let timedOut = expectation(description: "off-main caller timed out") - - DispatchQueue(label: "agent-device.runner.tests.timeout").async { - do { - _ = try self.runMainThreadWork( - "command_execution", - timeout: 0, - timeoutError: self.mainThreadExecutionTimeoutError - ) { - _ = releaseWork.wait(timeout: .now() + 2) - return true - } - } catch { - box.error = error - } - self.mainThreadWorkLock.lock() - box.abandonedCount = self.abandonedMainThreadWorkCount - box.abandonedSinceSet = self.abandonedMainThreadWorkSince != nil - self.mainThreadWorkLock.unlock() - if case .busy = self.currentMainThreadBusyState() { - box.busyWhileAbandoned = true - } - observedAbandoned.signal() - timedOut.fulfill() - } - DispatchQueue(label: "agent-device.runner.tests.release-timeout").async { - _ = observedAbandoned.wait(timeout: .now() + 2) - releaseWork.signal() - } - - wait(for: [timedOut], timeout: 3) - let drainDeadline = Date().addingTimeInterval(2) - while hasAbandonedMainThreadWork(), Date() < drainDeadline { - sleepFor(0.005) - } - - XCTAssertEqual((box.error as NSError?)?.code, RunnerErrorCode.mainThreadExecutionTimedOut) - XCTAssertEqual(box.abandonedCount, 1) - XCTAssertEqual(box.abandonedSinceSet, true) - XCTAssertTrue(box.busyWhileAbandoned) - XCTAssertFalse(hasAbandonedMainThreadWork(), "drained work must release the main thread") - mainThreadWorkLock.lock() - let sinceCleared = abandonedMainThreadWorkSince == nil - mainThreadWorkLock.unlock() - XCTAssertTrue(sinceCleared) - guard case .idle = currentMainThreadBusyState() else { - return XCTFail("expected the runner idle once the abandoned work drained") - } - } -} -#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift index 650d7559bb..a4df398737 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift @@ -303,170 +303,4 @@ extension RunnerTests { let element = app.descendants(matching: .any).matching(predicate).firstMatch return element.exists ? element : nil } - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS - func testTopLeadingNavigationFallbackPointTargetsHeaderControlBand() throws { - let point = try XCTUnwrap( - Self.topLeadingNavigationFallbackPoint( - in: CGRect(x: 0, y: 0, width: 430, height: 932) - ) - ) - - XCTAssertEqual(point.x, 34.4, accuracy: 0.01) - XCTAssertEqual(point.y, 132, accuracy: 0.01) - } - - func testTopLeadingNavigationFallbackPointRejectsInvalidFrame() { - XCTAssertNil(Self.topLeadingNavigationFallbackPoint(in: .infinite)) - XCTAssertNil(Self.topLeadingNavigationFallbackPoint(in: .zero)) - } - - func testNavigationBackControlRankPrefersBackThenCloseThenCancel() { - XCTAssertEqual(Self.navigationBackControlRank(label: "Back", identifier: ""), 0) - XCTAssertEqual(Self.navigationBackControlRank(label: "Close", identifier: ""), 1) - XCTAssertEqual(Self.navigationBackControlRank(label: "Cancel search", identifier: ""), 2) - XCTAssertNil(Self.navigationBackControlRank(label: "Search for more feeds", identifier: "")) - } - - func testNavigationBackPredicateUsesTheSharedKeywordTable() { - let predicate = Self.navigationBackPredicate() - - XCTAssertTrue(predicate.evaluate(with: ["label": "Back", "identifier": ""])) - XCTAssertTrue(predicate.evaluate(with: ["label": "", "identifier": "close-button"])) - XCTAssertFalse(predicate.evaluate(with: ["label": "Search for more feeds", "identifier": ""])) - } - - func testTopNavigationControlFrameAcceptsOnlyHeaderBand() { - let window = CGRect(x: 0, y: 0, width: 430, height: 932) - - XCTAssertTrue( - Self.isTopNavigationControlFrame( - CGRect(x: 340, y: 84, width: 72, height: 44), - in: window - ) - ) - XCTAssertFalse( - Self.isTopNavigationControlFrame( - CGRect(x: 20, y: 760, width: 72, height: 44), - in: window - ) - ) - XCTAssertFalse(Self.isTopNavigationControlFrame(.infinite, in: window)) - } - - func testNavigationVisualVerificationSeparatesNoChangeFromNoSample() { - XCTAssertEqual( - Self.navigationVisualObservation(before: Data([1, 2, 3]), after: Data([1, 2, 4])), - .changed - ) - XCTAssertEqual( - Self.navigationVisualObservation(before: Data([1, 2, 3]), after: Data([1, 2, 3])), - .unchanged - ) - // A missing sample is neither a change nor a no-change; treating it as "unchanged" would let a - // capture that refused become the reason the `back` command claims no control exists (#2728). - XCTAssertEqual(Self.navigationVisualObservation(before: nil, after: Data([1])), .unobserved) - XCTAssertEqual(Self.navigationVisualObservation(before: Data([1]), after: nil), .unobserved) - XCTAssertEqual(Self.navigationVisualObservation(before: nil, after: nil), .unobserved) - } - - func testNavigationFallbackReportsTheRefusalItHitNotADefaultCode() { - // The refusal from the most recent sample wins, so the code names what the fallback last looked at - // before giving up; an earlier refusal is reported only when the later sample carried none (#2728). - let after = NavigationVisualSample( - data: nil, - refusalCode: "APP_SCREEN_WINDOW_UNRESOLVED", - refusalHint: "after hint" - ) - let before = NavigationVisualSample( - data: nil, - refusalCode: "APP_SCREEN_UNRESOLVED", - refusalHint: "before hint" - ) - let laterWins = Self.navigationFallbackErrorPayload(after: after, before: before) - XCTAssertEqual(laterWins.code, "APP_SCREEN_WINDOW_UNRESOLVED") - XCTAssertEqual(laterWins.hint, "after hint") - - let onlyBefore = Self.navigationFallbackErrorPayload( - after: NavigationVisualSample(data: nil), - before: before - ) - XCTAssertEqual(onlyBefore.code, "APP_SCREEN_UNRESOLVED") - XCTAssertEqual(onlyBefore.hint, "before hint") - - // Neither side named a reason (unreachable on iOS): a real capture code, never a bare failure. - let unnamed = Self.navigationFallbackErrorPayload( - after: NavigationVisualSample(data: nil), - before: NavigationVisualSample(data: nil) - ) - XCTAssertEqual(unnamed.code, "APP_SCREEN_UNRESOLVED") - XCTAssertTrue(unnamed.message.contains("unknown outcome")) - } - - func testVerifyNavigationFallbackOutcomeReportsUnresolvedWindowWithoutSystemSurface() { - // The in-app `back` fallback ran its tap but the app resolved no window. It must name that refusal - // as an unknown outcome AND must not have sampled the system surface: capturing SpringBoard's home - // screen twice reads as "unchanged" and launders a wrong-process frame into "no back control - // exists" (#2728). This drives the SAME `navigationFallbackSample` production calls, handing it a - // system surface that fails the test if consulted — so reverting the fallback to sample SpringBoard - // (or dropping the refusal code) turns this red, which an inline `.never` re-creation could not. - var askedSystemSurface = false - let sample = Self.navigationFallbackSample( - resolvingApp: { .failure(.unresolvedWindow) }, - systemSurface: { - askedSystemSurface = true - return .failure(.unresolvedWindow) - }, - encoding: { _ in Data([1, 2, 3]) } - ) - XCTAssertFalse(askedSystemSurface, "the in-app fallback samples the app only, never SpringBoard") - - XCTAssertNil(sample.data) - XCTAssertEqual(sample.refusalCode, "APP_SCREEN_WINDOW_UNRESOLVED") - - let observation = Self.navigationVisualObservation(before: sample.data, after: sample.data) - XCTAssertEqual(observation, .unobserved) - - switch Self.inAppBackOutcome(observation: observation, before: sample, after: sample) { - case .unverified(let payload): - XCTAssertEqual(payload.code, "APP_SCREEN_WINDOW_UNRESOLVED") - case .performed, .unavailable: - XCTFail("a refused capture must report an unknown outcome, not 'no back control'") - } - } - - func testNavigationVisualSampleDistinguishesEncodedFrameFromRefusal() { - // The same capture entry point yields three different samples, and only the refusal ones may carry - // a code: an encoded frame is evidence, a resolved-but-unencodable image and a refusal are not - // (#2728). Reverting the mapping to a plain no-sample loses the reason a host keys on. - let captured = CapturedAppScreen( - image: RunnerImage(), - displayID: 3, - pixelWidth: 12, - pixelHeight: 24, - pixelsPerPoint: 3 - ) - - let encoded = Self.navigationVisualSample( - from: .success(captured), - encoding: { _ in Data([7, 7]) } - ) - XCTAssertEqual(encoded.data, Data([7, 7])) - XCTAssertNil(encoded.refusalCode) - - let unencodable = Self.navigationVisualSample( - from: .success(captured), - encoding: { _ in nil } - ) - XCTAssertNil(unencodable.data) - XCTAssertEqual(unencodable.refusalCode, "APP_SCREEN_CAPTURE_UNRENDERABLE") - - let refused = Self.navigationVisualSample( - from: .failure(.unresolvedScreen), - encoding: { _ in Data([7, 7]) } - ) - XCTAssertNil(refused.data) - XCTAssertEqual(refused.refusalCode, "APP_SCREEN_UNRESOLVED") - } -#endif } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+PrivateAXPresentation.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+PrivateAXPresentation.swift index 202e2d061b..b95b1dc5a1 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+PrivateAXPresentation.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+PrivateAXPresentation.swift @@ -89,212 +89,3 @@ extension RunnerTests { (value as? Bool) ?? (value as? NSNumber)?.boolValue } } - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS -extension RunnerTests { - fileprivate static func privateAXFrame( - _ x: Double, _ y: Double, _ width: Double, _ height: Double - ) -> [String: Any] { - ["x": x, "y": y, "width": width, "height": height] - } - - /// A scroll container whose second row is scrolled out of the viewport, plus an unlabeled - /// decoration: the shapes the regular projection folds away and the raw projection must keep. - fileprivate static var privateAXScrolledFixture: [String: Any] { - let frame = privateAXFrame - return ["type": Int(XCUIElement.ElementType.application.rawValue), - "label": "Element", "frame": frame(0, 0, 402, 874), "children": [[ - "type": Int(XCUIElement.ElementType.scrollView.rawValue), "frame": frame(0, 96, 402, 700), - "actions": ["Scroll down"], - "children": [ - ["type": Int(XCUIElement.ElementType.button.rawValue), "label": "Profile picture", - "frame": frame(16, 120, 44, 44), - "children": [["type": Int(XCUIElement.ElementType.image.rawValue), - "frame": frame(16, 120, 1, 1)]]], - ["type": Int(XCUIElement.ElementType.button.rawValue), "label": "Theme", - "frame": frame(16, 900, 360, 44)]]]]] - } - - /// Acquire with the private-AX serializer, run the one normalization pass the production capture - /// plan runs (`captureWithBackend`), then present through the shared regular fold -- the production - /// route for this backend since the fold moved into presentation (#1797, #2661). - fileprivate func privateAXRegularPresentation( - rawRoot: [String: Any], - viewport: CGRect, - interactiveOnly: Bool = false - ) throws -> [PresentedNode] { - let hint = CaptureHint( - projection: .regular, depth: nil, regularPresentedDepth: nil, - interactiveOnly: interactiveOnly, customActions: false) - let nodes = privateAXNormalizedAcquisition( - rawRoot: rawRoot, hint: hint, viewport: viewport, - interfaceOrientation: RunnerInterfaceOrientation.portrait) - return try SnapshotPresentation.presentRegular( - SnapshotAcquisition( - hint: hint, nodes: nodes, truncated: false, effectiveDepth: nil, viewport: viewport, - interfaceOrientation: RunnerInterfaceOrientation.portrait), - options: PresentationOptions( - interactiveOnly: interactiveOnly, depth: nil, scope: nil, raw: false), - policy: .cursorProjected - ).nodes - } - - /// Acquire, then normalize once -- the exact pair `captureWithBackend` runs for this backend. - fileprivate func privateAXNormalizedAcquisition( - rawRoot: [String: Any], - hint: CaptureHint, - viewport: CGRect, - interfaceOrientation: Int - ) -> [RawAXNode] { - SnapshotGeometrySpace.normalized( - nodes: privateAXAcquisition(rawRoot: rawRoot, hint: hint), - viewport: viewport, - interfaceOrientation: interfaceOrientation - ) - } - - /// The one normalization pass has to reach the node it publishes: this asserts the rotated rect of - /// a key under a turned surface host, and an untouched sibling under the app's own window, so a - /// pass that drops the space a subtree declared fails here. - func testPrivateAXAcquisitionPublishesATurnedSurfaceHostInAppOrientationSpace() { - let frame = Self.privateAXFrame - let appWindow: [String: Any] = [ - "type": Int(XCUIElement.ElementType.window.rawValue), - "frame": frame(0, 0, 874, 402), - "children": [ - ["type": Int(XCUIElement.ElementType.button.rawValue), "label": "Home", - "frame": frame(204, 323, 91, 55), "children": []] - ] - ] - let keyboardWindow: [String: Any] = [ - "type": Int(XCUIElement.ElementType.window.rawValue), - "frame": frame(0, 0, 874, 402), - "children": [ - ["type": Int(XCUIElement.ElementType.other.rawValue), - "frame": frame(0, 0, 402, 874), - "children": [ - ["type": Int(XCUIElement.ElementType.key.rawValue), "label": "q", - "frame": frame(154, 77, 45, 72), "children": []] - ]] - ] - ] - let hint = CaptureHint( - projection: .raw, depth: nil, regularPresentedDepth: nil, - interactiveOnly: false, customActions: false) - let nodes = privateAXNormalizedAcquisition( - rawRoot: [ - "type": Int(XCUIElement.ElementType.application.rawValue), - "label": "Element", "frame": frame(0, 0, 874, 402), - "children": [appWindow, keyboardWindow] - ], - hint: hint, - viewport: CGRect(x: 0, y: 0, width: 874, height: 402), - interfaceOrientation: RunnerInterfaceOrientation.landscapeRight - ) - - // Measured on iPhone 17 Pro (26.2): the key plane's left column arrives 154 pt along the device's - // long axis and comes back 203 pt down the app's short one. - XCTAssertEqual( - nodes.first { $0.label == "q" }?.rect, - SnapshotRect(x: 77, y: 203, width: 72, height: 45) - ) - XCTAssertEqual( - nodes.first { $0.label == "Home" }?.rect, - SnapshotRect(x: 204, y: 323, width: 91, height: 55) - ) - } - - func testPrivateAXRegularPresentationProjectsToViewportAndKeepsScrollHint() throws { - let nodes = try privateAXRegularPresentation( - rawRoot: Self.privateAXScrolledFixture, - viewport: CGRect(x: 0, y: 0, width: 402, height: 874)) - XCTAssertEqual(nodes.compactMap(\.label), ["Element", "Profile picture"]) - let scrollView = nodes.first { $0.type == "ScrollView" } - XCTAssertEqual(scrollView?.hiddenContentBelow, true) - XCTAssertEqual(scrollView?.actions, ["Scroll down"]) - } - - /// #1797 D4: the raw projection is the acquired tree. The offscreen row and the sub-pixel - /// decoration the regular projection folds away are both present, at traversal depth, and every - /// regular node still appears -- `regular ⊆ raw` on the same capture. - func testPrivateAXRawProjectionKeepsEveryAcquiredNode() throws { - let viewport = CGRect(x: 0, y: 0, width: 402, height: 874) - let root = Self.privateAXScrolledFixture - let regular = try privateAXRegularPresentation(rawRoot: root, viewport: viewport, - interactiveOnly: true) - let raw = privateAXNormalizedAcquisition(rawRoot: root, - hint: CaptureHint( - projection: .raw, depth: nil, regularPresentedDepth: nil, - interactiveOnly: false, customActions: false), - viewport: viewport, - interfaceOrientation: RunnerInterfaceOrientation.portrait) - - XCTAssertEqual(raw.map(\.type), ["Application", "ScrollView", "Button", "Image", "Button"]) - XCTAssertEqual(raw.map(\.depth), [0, 1, 2, 3, 2]) - XCTAssertEqual(raw.map(\.parentIndex), [nil, 0, 1, 2, 1]) - XCTAssertEqual(raw.compactMap(\.label), ["Element", "Profile picture", "Theme"]) - // The offscreen row is a reported fact in raw, and reported facts do not become hittable - // just because the projection kept them. - XCTAssertEqual(raw.last?.hittable, false) - XCTAssertTrue(raw.allSatisfy { $0.hiddenContentAbove == nil && $0.hiddenContentBelow == nil }) - - let rawKeys = Set(raw.map { "\($0.type)-\($0.label ?? "")-\($0.rect.y)" }) - for node in regular { - XCTAssertTrue( - rawKeys.contains("\(node.type)-\(node.label ?? "")-\(node.rect.y)"), - "regular node \(node.type)/\(node.label ?? "") is missing from the raw projection" - ) - } - XCTAssertGreaterThan(raw.count, regular.count) - } - - /// Raw depth is traversal depth, so a raw `--depth` request is the one narrowing this backend - /// can prove complete. - func testPrivateAXRawProjectionAppliesRequestedTraversalDepth() { - let raw = privateAXNormalizedAcquisition(rawRoot: Self.privateAXScrolledFixture, - hint: CaptureHint( - projection: .raw, depth: 2, regularPresentedDepth: nil, - interactiveOnly: false, customActions: false), - viewport: CGRect(x: 0, y: 0, width: 402, height: 874), - interfaceOrientation: RunnerInterfaceOrientation.portrait) - XCTAssertEqual(raw.map(\.type), ["Application", "ScrollView", "Button", "Button"]) - XCTAssertEqual(raw.map(\.depth), [0, 1, 2, 2]) - } - - func testPrivateAXPresentationKeepsOffscreenSubtreeExcludedWhenChildFramesAreClamped() throws { - let frame = Self.privateAXFrame - let root: [String: Any] = ["type": Int(XCUIElement.ElementType.application.rawValue), - "label": "Element", "frame": frame(0, 0, 402, 874), "children": [[ - "type": Int(XCUIElement.ElementType.table.rawValue), "frame": frame(0, 96, 402, 700), - "children": [["type": Int(XCUIElement.ElementType.cell.rawValue), - "label": "Theme", "frame": frame(0, 900, 402, 44), "children": [[ - "type": Int(XCUIElement.ElementType.staticText.rawValue), - "label": "Theme", "frame": frame(16, 96, 120, 44)], - ["type": Int(XCUIElement.ElementType.switch.rawValue), - "label": "Theme", "frame": frame(340, 96, 46, 44)]]]]]]] - - let nodes = try privateAXRegularPresentation( - rawRoot: root, viewport: CGRect(x: 0, y: 0, width: 402, height: 874)) - - XCTAssertEqual(nodes.compactMap(\.label), ["Element"]) - XCTAssertEqual(nodes.first { $0.type == "Table" }?.hiddenContentBelow, true) - } - - func testPrivateAXGeometrylessSemanticsAreNeverActionableOrScrollContexts() throws { - let zero = ["x": 0, "y": 0, "width": 0, "height": 0] - let root: [String: Any] = ["type": Int(XCUIElement.ElementType.application.rawValue), - "label": "Element", "frame": ["x": 0, "y": 0, "width": 402, "height": 874], - "children": [["type": Int(XCUIElement.ElementType.scrollView.rawValue), - "label": "Settings semantics", "frame": zero, "children": [[ - "type": Int(XCUIElement.ElementType.button.rawValue), "label": "Theme", "frame": zero], - ["type": Int(XCUIElement.ElementType.other.rawValue), "frame": zero]]]]] - let nodes = try privateAXRegularPresentation( - rawRoot: root, viewport: CGRect(x: 0, y: 0, width: 402, height: 874), - interactiveOnly: true) - XCTAssertEqual(nodes.compactMap(\.label), ["Element", "Settings semantics", "Theme"]) - XCTAssertEqual(nodes.filter { $0.index != 0 }.map(\.hittable), [false, false]) - XCTAssertFalse(nodes.contains { $0.type == "Other" }) - XCTAssertTrue(nodes.allSatisfy { $0.hiddenContentAbove == nil && $0.hiddenContentBelow == nil }) - } -} -#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollGesture.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollGesture.swift index f6ebb6866b..326eec698f 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollGesture.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollGesture.swift @@ -5,8 +5,9 @@ import XCTest // This is a deliberate two-place invariant: the daemon keeps the TS implementation (for Android, // recording, and reported-pixels), and the runner places the gesture with this Swift copy. Both // ports are asserted against the same table, contracts/fixtures/scroll-gesture.json (gated -// XCTest at the bottom of this file, vitest twin packages/contracts/src/scroll-gesture.test.ts) — -// if you change the math in either language, update the other and the table. +// XCTest in UnitTests/RunnerTests+ScrollGestureTests.swift, vitest twin +// packages/contracts/src/scroll-gesture.test.ts) — if you change the math in either language, +// update the other and the table. // // All inputs here are positive (reference dims, travel, center), so Swift's `.rounded()` // (half away from zero) matches JS `Math.round` (half up) on every value computed below. @@ -96,198 +97,3 @@ func runnerDragCommandDefaults(_ command: Command) -> RunnerDragCommandDefaults durationMs: command.durationMs ?? runnerDefaultDragDurationMs ) } - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS -private struct ScrollGestureFixture: Decodable { - struct Constants: Decodable { - let defaultIosScrollAmount: Double - let defaultMobileScrollDurationMs: Double - let defaultIosScrollDurationMs: Double - let defaultScrollAmount: Double - let defaultEdgePaddingFraction: Double - let ordinaryScrollReleaseBehavior: String - let edgeScrollReleaseBehavior: String - } - struct Expected: Decodable { - let x1: Double - let y1: Double - let x2: Double - let y2: Double - let pixels: Double - } - struct Case: Decodable { - let name: String - let direction: String - let amount: Double? - let pixels: Double? - let referenceWidth: Double - let referenceHeight: Double - let expected: Expected - } - - let constants: Constants - let cases: [Case] -} - -extension RunnerTests { - // Cross-language parity table: every case in contracts/fixtures/scroll-gesture.json must agree - // with the vitest twin (packages/contracts/src/scroll-gesture.test.ts). Add vectors there, - // never fork the math. - private func loadScrollGestureFixture() throws -> ScrollGestureFixture { - let fixtureURL = URL(fileURLWithPath: #filePath) - .deletingLastPathComponent() // AgentDeviceRunnerUITests - .deletingLastPathComponent() // AgentDeviceRunner - .deletingLastPathComponent() // runner - .deletingLastPathComponent() // apple - .deletingLastPathComponent() // repo root - .appendingPathComponent("contracts") - .appendingPathComponent("fixtures") - .appendingPathComponent("scroll-gesture.json") - return try JSONDecoder().decode(ScrollGestureFixture.self, from: Data(contentsOf: fixtureURL)) - } - - func testRunnerScrollGesturePlanMatchesParityTable() throws { - let fixture = try loadScrollGestureFixture() - XCTAssertFalse(fixture.cases.isEmpty, "parity table must not be empty") - for testCase in fixture.cases { - let plan = try XCTUnwrap( - runnerScrollGesturePlan( - direction: try XCTUnwrap(RunnerScrollDirection(rawValue: testCase.direction)), - amount: testCase.amount, - pixels: testCase.pixels, - referenceWidth: testCase.referenceWidth, - referenceHeight: testCase.referenceHeight - ), - testCase.name - ) - XCTAssertEqual(plan.x1, testCase.expected.x1, testCase.name) - XCTAssertEqual(plan.y1, testCase.expected.y1, testCase.name) - XCTAssertEqual(plan.x2, testCase.expected.x2, testCase.name) - XCTAssertEqual(plan.y2, testCase.expected.y2, testCase.name) - XCTAssertEqual(plan.travelPixels, testCase.expected.pixels, testCase.name) - } - } - - // The planner constants are private on both sides; the table pins them behaviourally on a - // 1000px axis where every rounding step is exact. - func testRunnerScrollGesturePlanUsesParityTableConstants() throws { - let constants = try loadScrollGestureFixture().constants - let defaultScroll = try JSONDecoder().decode( - Command.self, from: Data(#"{"command":"scroll"}"#.utf8)) - let defaults = runnerDragCommandDefaults(defaultScroll) - XCTAssertEqual(defaults.durationMs, constants.defaultIosScrollDurationMs) - XCTAssertEqual(defaults.scrollAmount, constants.defaultIosScrollAmount) - let defaulted = try XCTUnwrap( - runnerScrollGesturePlan( - direction: .down, amount: nil, pixels: nil, referenceWidth: 1000, referenceHeight: 1000 - ) - ) - XCTAssertEqual(defaulted.travelPixels, 1000 * constants.defaultScrollAmount) - let saturated = try XCTUnwrap( - runnerScrollGesturePlan( - direction: .down, amount: 10, pixels: nil, referenceWidth: 1000, referenceHeight: 1000 - ) - ) - XCTAssertEqual( - saturated.travelPixels, 1000 - 2 * 1000 * constants.defaultEdgePaddingFraction) - } - - func testRunnerScrollAndDragCommandDefaultsStayDistinct() throws { - func command(_ json: String) throws -> Command { - try JSONDecoder().decode(Command.self, from: Data(json.utf8)) - } - - let pixelScroll = runnerDragCommandDefaults( - try command(#"{"command":"scroll","pixels":120}"#)) - XCTAssertNil(pixelScroll.scrollAmount) - XCTAssertEqual(pixelScroll.durationMs, 400) - - let drag = runnerDragCommandDefaults(try command(#"{"command":"drag"}"#)) - XCTAssertNil(drag.scrollAmount) - XCTAssertEqual(drag.durationMs, 250) - - let explicitScroll = runnerDragCommandDefaults( - try command(#"{"command":"scroll","amount":0.5,"durationMs":125}"#)) - XCTAssertEqual(explicitScroll.scrollAmount, 0.5) - XCTAssertEqual(explicitScroll.durationMs, 125) - - let explicitDrag = runnerDragCommandDefaults( - try command(#"{"command":"drag","durationMs":125}"#)) - XCTAssertEqual(explicitDrag.durationMs, 125) - } - - func testRunnerScrollReleaseBehaviorSelectsTheDragProfile() throws { - let constants = try loadScrollGestureFixture().constants - let controlled = try XCTUnwrap(ScrollReleaseBehavior(rawValue: constants.ordinaryScrollReleaseBehavior)) - let inertial = try XCTUnwrap(ScrollReleaseBehavior(rawValue: constants.edgeScrollReleaseBehavior)) - XCTAssertEqual( - scrollDragProfile(releaseBehavior: nil), - .controlledScroll - ) - XCTAssertEqual( - scrollDragProfile(releaseBehavior: controlled), - .controlledScroll - ) - XCTAssertEqual( - scrollDragProfile(releaseBehavior: inertial), - .fastSwipe - ) - } - - func testControlledScrollProfileUsesReliableCadenceAndMonotonicDeceleration() { - XCTAssertEqual(RunnerControlledScrollFrameCount(350), 21) - XCTAssertEqual(RunnerControlledScrollFrameCount(400), 24) - XCTAssertEqual(RunnerControlledScrollFrameCount(500), 30) - XCTAssertEqual(RunnerControlledScrollFrameCount(1_000), 30) - XCTAssertEqual(RunnerControlledScrollFrameCount(10_000), 30) - - let frameCount = RunnerControlledScrollFrameCount(400) - let progress = (0...frameCount).map { - RunnerControlledScrollProgress(Double($0) / Double(frameCount)) - } - let deltas = zip(progress.dropFirst(), progress).map { $0.0 - $0.1 } - XCTAssertEqual(progress.first, 0) - XCTAssertEqual(progress.last, 1) - XCTAssertTrue(zip(deltas, deltas.dropFirst()).allSatisfy { $0.1 <= $0.0 }) - - let iPhoneViewportPoints = 874.0 - let defaultFingerTravel = iPhoneViewportPoints * 0.65 - let finalSampleTravel = (1 - progress[progress.count - 2]) * defaultFingerTravel - XCTAssertLessThan(finalSampleTravel, 0.1) - } - - func testRunnerScrollGesturePlanRejectsUnknownDirection() { - XCTAssertNil(RunnerScrollDirection(rawValue: "sideways")) - } - - func testRunnerScrollGesturePlanRejectsInvalidAmountAndPixels() { - XCTAssertNil( - runnerScrollGesturePlan( - direction: .down, - amount: 0, - pixels: nil, - referenceWidth: 300, - referenceHeight: 600 - ) - ) - XCTAssertNil( - runnerScrollGesturePlan( - direction: .down, - amount: nil, - pixels: -10, - referenceWidth: 300, - referenceHeight: 600 - ) - ) - XCTAssertNil( - runnerScrollGesturePlan( - direction: .down, - amount: .infinity, - pixels: nil, - referenceWidth: 300, - referenceHeight: 600 - ) - ) - } -} -#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift index 8b0b5da622..b6ea551520 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift @@ -239,188 +239,3 @@ extension RunnerTests { Response(ok: false, error: ErrorPayload(code: "INVALID_ARGS", message: message)) } } - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS -// MARK: - In-bundle unit tests (device-free) - -extension RunnerTests { - func testSequenceDecodesStepsFromWire() throws { - let json = """ - {"command":"sequence","commandId":"seq-1","steps":[ - {"kind":"tap","x":100,"y":200}, - {"kind":"doubleTap","x":101,"y":200}, - {"kind":"longPress","x":102,"y":200,"durationMs":300,"pauseMs":50} - ]} - """ - let command = try JSONDecoder().decode(Command.self, from: Data(json.utf8)) - XCTAssertEqual(command.command, .sequence) - XCTAssertEqual(command.steps?.count, 3) - XCTAssertEqual(command.steps?[0].kind, "tap") - XCTAssertEqual(command.steps?[1].kind, "doubleTap") - XCTAssertEqual(command.steps?[2].pauseMs, 50) - } - - func testSequenceAcceptsDoubleTapKind() { - // A doubleTap step missing coords must fail on the coords check, not the kind allowlist — - // proving "doubleTap" passes validateSequenceStep without needing a device to execute on. - let response = executeSequenceForTest(steps: [ - sequenceStep(kind: "doubleTap", x: nil) - ]) - XCTAssertEqual(response.ok, false) - XCTAssertEqual(response.error?.code, "INVALID_ARGS") - XCTAssertTrue(response.error?.message.contains("requires finite x and y") ?? false) - XCTAssertFalse(response.error?.message.contains("unsupported kind") ?? true) - } - - func testSequenceRejectsUnknownKind() throws { - let response = executeSequenceForTest(steps: [ - sequenceStep(kind: "tap", x: 1, y: 2), - sequenceStep(kind: "pinch", x: 3, y: 4), - ]) - XCTAssertEqual(response.ok, false) - XCTAssertEqual(response.error?.code, "INVALID_ARGS") - XCTAssertTrue(response.error?.message.contains("step 1") ?? false) - XCTAssertTrue(response.error?.message.contains("pinch") ?? false) - } - - func testSequenceRejectsEmpty() { - let response = executeSequenceForTest(steps: []) - XCTAssertEqual(response.ok, false) - XCTAssertEqual(response.error?.code, "INVALID_ARGS") - } - - func testSequenceRejectsTooManySteps() { - let steps = (0..<21).map { _ in sequenceStep(kind: "tap", x: 1, y: 2) } - let response = executeSequenceForTest(steps: steps) - XCTAssertEqual(response.ok, false) - XCTAssertEqual(response.error?.code, "INVALID_ARGS") - XCTAssertTrue(response.error?.message.contains("at most 20") ?? false) - } - - func testSequenceHasSynthesizedCoordinateStep() { - XCTAssertTrue( - sequenceHasSynthesizedCoordinateStep([ - sequenceStep(kind: "tap", x: 1, y: 2, synthesized: true), - ]) - ) - XCTAssertFalse( - sequenceHasSynthesizedCoordinateStep([ - sequenceStep(kind: "tap", x: 1, y: 2), - sequenceStep(kind: "doubleTap", x: 1, y: 2, synthesized: true), - ]) - ) - } - - func testAssembleSequencePreservesOrderOnSuccess() { - let steps = [ - sequenceStep(kind: "tap", x: 1, y: 1), - sequenceStep(kind: "longPress", x: 2, y: 2), - sequenceStep(kind: "tap", x: 3, y: 3), - ] - var calls: [Int] = [] - let execution = assembleSequenceExecution(steps: steps) { index, _ in - calls.append(index) - return SequenceStepOutcome( - outcome: .performed, - gestureStartUptimeMs: Double(index * 10), - gestureEndUptimeMs: Double(index * 10 + 5) - ) - } - XCTAssertEqual(calls, [0, 1, 2]) - XCTAssertEqual(execution.completedSteps, 3) - XCTAssertNil(execution.failedStepIndex) - XCTAssertEqual(execution.results.map { $0.kind }, ["tap", "longPress", "tap"]) - XCTAssertEqual(execution.gestureStartUptimeMs, 0) - XCTAssertEqual(execution.gestureEndUptimeMs, 25) - } - - func testAssembleSequenceStopsAtFirstFailure() { - let steps = [ - sequenceStep(kind: "tap", x: 1, y: 1), - sequenceStep(kind: "longPress", x: 2, y: 2), - sequenceStep(kind: "tap", x: 3, y: 3), - ] - var calls: [Int] = [] - let execution = assembleSequenceExecution(steps: steps) { index, _ in - calls.append(index) - if index == 1 { - return SequenceStepOutcome( - outcome: .unsupported(message: "long press unsupported", hint: nil), - gestureStartUptimeMs: 10, - gestureEndUptimeMs: 15 - ) - } - return SequenceStepOutcome(outcome: .performed, gestureStartUptimeMs: 0, gestureEndUptimeMs: 5) - } - // Step 2 is never invoked. - XCTAssertEqual(calls, [0, 1]) - XCTAssertEqual(execution.completedSteps, 1) - XCTAssertEqual(execution.failedStepIndex, 1) - // results.count == completedSteps + 1 (the failed step). - XCTAssertEqual(execution.results.count, 2) - XCTAssertEqual(execution.results[1].ok, false) - XCTAssertEqual(execution.results[1].errorCode, "UNSUPPORTED_OPERATION") - XCTAssertEqual(execution.results[1].errorMessage, "long press unsupported") - } - - func testSequenceWorstCaseResponseStaysUnderJournalCap() throws { - let longMessage = String(repeating: "e", count: 200) - let results = (0..<20).map { index in - SequenceStepResult( - ok: index < 19, - kind: "longPress", - errorCode: index < 19 ? nil : "UNSUPPORTED_OPERATION", - errorMessage: index < 19 ? nil : longMessage, - gestureStartUptimeMs: 123456.789, - gestureEndUptimeMs: 123466.789 - ) - } - let response = Response( - ok: true, - data: DataPayload( - message: "sequence", - completedSteps: 19, - failedStepIndex: 19, - sequenceResults: results - ) - ) - let encoded = try JSONEncoder().encode(response) - XCTAssertLessThan(encoded.count, 16 * 1024) - } - - private func sequenceStep( - kind: String, - x: Double?, - y: Double? = nil, - synthesized: Bool? = nil - ) -> SequenceStep { - SequenceStep( - kind: kind, - x: x, - y: y, - durationMs: nil, - pauseMs: nil, - synthesized: synthesized - ) - } - - /// Validation runs before any executor call, so the INVALID_ARGS paths are exercised without - /// reaching the device executor (which is never invoked when validation rejects). - private func executeSequenceForTest(steps: [SequenceStep]) -> Response { - let command = makeSequenceCommand(steps: steps) - return executeSequence(command: command, activeApp: app) - } - - /// Build a sequence Command via JSON so the test does not depend on the memberwise init's - /// parameter order. - private func makeSequenceCommand(steps: [SequenceStep]) -> Command { - struct SequenceCommandFixture: Encodable { - let command = "sequence" - let commandId = "seq-test" - let steps: [SequenceStep] - } - let data = try! JSONEncoder().encode(SequenceCommandFixture(steps: steps)) - return try! JSONDecoder().decode(Command.self, from: data) - } -} -#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift index 83686731cf..14fe138387 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift @@ -5,12 +5,12 @@ extension RunnerTests { static let axSnapshotErrorCode = "IOS_AX_SNAPSHOT_FAILED" static let axSnapshotFailureMessage = "iOS XCTest snapshot failed while serializing the accessibility tree." - private static let axSnapshotUnavailableReason = "ax_snapshot_unavailable" + static let axSnapshotUnavailableReason = "ax_snapshot_unavailable" static let axSnapshotHint = "Snapshot state is unavailable because XCTest could not serialize this iOS accessibility tree. This can be specific to the current screen. Use plain screenshot, not screenshot --overlay-refs, as visual truth; navigate with coordinate commands if needed; then retry snapshot -i after reaching another screen. If you own the app and need full-tree inspection, simplify this screen's accessibility tree and expose stable ids on actionable controls." - private static let rawSnapshotTooLargeCode = "IOS_RAW_SNAPSHOT_TOO_LARGE" - private static let rawSnapshotMaxNodes = 5_000 - private static let rawSnapshotTooLargeHint = + static let rawSnapshotTooLargeCode = "IOS_RAW_SNAPSHOT_TOO_LARGE" + static let rawSnapshotMaxNodes = 5_000 + static let rawSnapshotTooLargeHint = "Raw iOS snapshot exceeded the runner payload guard. Use regular snapshot for visible UI, or scope/depth-limit raw snapshot when inspecting a large accessibility tree." // Runaway guard for the regular tree walk: a work bound only. A screen that trips it raises this // number in ADR 0004's name rather than bounding the walk by geometry again. @@ -523,11 +523,11 @@ extension RunnerTests { ) } - private func recoveredSnapshotMessage(_ failure: SnapshotCaptureFailure) -> String { + func recoveredSnapshotMessage(_ failure: SnapshotCaptureFailure) -> String { return "\(failure.message) Hint: \(failure.hint)" } - private func rawSnapshotTooLargeFailure(nodeCount: Int) -> SnapshotCaptureFailure { + func rawSnapshotTooLargeFailure(nodeCount: Int) -> SnapshotCaptureFailure { SnapshotCaptureFailure( code: Self.rawSnapshotTooLargeCode, message: "iOS raw snapshot exceeded \(Self.rawSnapshotMaxNodes) nodes while walking node \(nodeCount).", @@ -558,246 +558,4 @@ extension RunnerTests { runnerFatalReason: runnerFatalReason ) } - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS - func testSnapshotAccessibilityUnavailableMarksSparseSnapshotRunnerFatal() { - currentApp = app - currentBundleId = "com.example.app" - - let payload = snapshotAccessibilityUnavailable( - failure: SnapshotCaptureFailure( - code: Self.axSnapshotErrorCode, - message: Self.axSnapshotFailureMessage, - hint: Self.axSnapshotHint - ) - ) - - XCTAssertEqual(payload.message, "\(Self.axSnapshotFailureMessage) Hint: \(Self.axSnapshotHint)") - XCTAssertEqual(payload.nodes?.count, 1) - XCTAssertEqual(payload.nodes?.first?.type, "Application") - XCTAssertEqual(payload.truncated, true) - XCTAssertEqual(payload.runnerFatal, true) - XCTAssertEqual(payload.runnerFatalReason, Self.axSnapshotUnavailableReason) - // The planned terminal result carries the structured verdict like every other planned - // snapshot — downstream sparse handling keys off it, not off node shapes. - XCTAssertEqual(payload.snapshotQuality?.state, "sparse") - XCTAssertEqual(payload.snapshotQuality?.reasonCode, "ax-rejected") - XCTAssertEqual(payload.snapshotQuality?.reason, Self.axSnapshotFailureMessage) - XCTAssertNil(currentApp) - XCTAssertNil(currentBundleId) - } - - func testRecoveredSnapshotMessagePreservesHint() { - let message = recoveredSnapshotMessage( - SnapshotCaptureFailure( - code: Self.axSnapshotErrorCode, - message: Self.axSnapshotFailureMessage, - hint: Self.axSnapshotHint - ) - ) - - XCTAssertTrue(message.contains(Self.axSnapshotFailureMessage)) - XCTAssertTrue(message.contains(Self.axSnapshotHint)) - } - - func testRawSnapshotTooLargeFailureIsStructured() { - let failure = rawSnapshotTooLargeFailure(nodeCount: Self.rawSnapshotMaxNodes + 1) - - XCTAssertEqual(failure.code, Self.rawSnapshotTooLargeCode) - XCTAssertTrue(failure.message.contains("\(Self.rawSnapshotMaxNodes) nodes")) - XCTAssertEqual(failure.hint, Self.rawSnapshotTooLargeHint) - } - - func testSystemModalProbeSliceSharesAndClampsToPlanDeadline() { - // Fresh plan deadline: the probe gets its full dedicated budget. - XCTAssertEqual(Self.systemModalProbeSlice(budget: 4, deadlineRemaining: 20), 4) - // Nearly-spent plan deadline: the probe is clamped so it can't run past the shared budget. - XCTAssertEqual(Self.systemModalProbeSlice(budget: 4, deadlineRemaining: 1.5), 1.5) - // Exactly/already exhausted deadline: skip the probe entirely (0), never a negative timeout. - XCTAssertEqual(Self.systemModalProbeSlice(budget: 4, deadlineRemaining: 0), 0) - XCTAssertEqual(Self.systemModalProbeSlice(budget: 4, deadlineRemaining: -5), 0) - } - - // Simulator-only: the bounded probe body returns nil on macOS (no SpringBoard host), so the - // timeout/penalty/drain machinery below only exists on the iOS branch. -#if os(iOS) - /// Regression for #1244/#1248: drives the bounded system-modal probe through a real, - /// production-only command entry point (`snapshotFast` or `snapshotRaw` -- see the two test - /// methods below), not `boundedBlockingSystemAlertSnapshot` directly, with - /// `systemModalProbeOverrideForTesting` set to a closure that blocks past the probe's real - /// slice, forcing a real `runMainThreadWork` timeout. This is revert-sensitive on both halves - /// of the fix, for either entry point: - /// - if the entry point reverted to calling the unbounded `blockingSystemAlertSnapshot` - /// directly (or dropped the `runMainThreadWork` wrap), nothing here would ever time out, - /// so the mid-flight busy/penalty assertions below would never be met; - /// - if the `onAbandoned` penalty hook or the abandoned-work accounting were dropped, the - /// timeout would still fire, but the busy/penalty and drain assertions would not hold. - /// - /// The drain assertion is synchronized on the *real* release rather than raced: after - /// signaling the probe to finish, the background queue polls `hasAbandonedMainThreadWork()` - /// (bounded) and only then fulfills `drained`, which the test `wait(for:timeout:)`s on before - /// asserting `.idle`/`hasAbandonedMainThreadWork() == false` below -- so a slow drain fails that - /// assertion instead of racing a fixed-timing guess. - private func assertBoundedSystemModalProbeTimeoutRecoversThenReleasesOnDrain( - entryPointName: String, - callEntryPoint: @escaping (XCUIApplication, PresentationOptions) throws -> DataPayload - ) { - let targetBundleId = "com.callstack.agentdevice.runner.missing.snapshot-timeout-test" - let snapshotTarget = XCUIApplication(bundleIdentifier: targetBundleId) - let probeReleaseGate = DispatchSemaphore(value: 0) - currentApp = snapshotTarget - currentBundleId = targetBundleId - defer { - probeReleaseGate.signal() - currentApp = nil - currentBundleId = nil - systemModalProbeOverrideForTesting = nil - clearSnapshotXCTestChannelPenalty(reason: "test-cleanup") - } - - final class ResultBox { - var payload: DataPayload? - var wasBusyBeforeDrain = false - var hadAbandonedCaptureBeforeDrain = false - var wasPenalizedBeforeDrain = false - } - let box = ResultBox() - // The test owns release of the injected probe. A fixed timeout races the capture plan's - // independent fallback tiers on loaded CI hosts and can drain before the test records the - // abandoned-work state. The defer above still releases the probe if an earlier assertion or - // expectation fails. - systemModalProbeOverrideForTesting = { _ in - probeReleaseGate.wait() - return nil - } - - let completion = expectation( - description: "\(entryPointName) recovered while the probe was abandoned, then released it" - ) - let drained = expectation(description: "\(entryPointName) modal probe drained") - DispatchQueue(label: "agent-device.runner.tests.modal-probe-timeout").async { - box.payload = try? callEntryPoint( - snapshotTarget, - PresentationOptions(interactiveOnly: false, depth: nil, scope: nil, raw: false) - ) - - // 1) Penalty/busy accounting: must already be in place by the time the entry point - // returns, well before we release the still-blocked probe below. - if case .busy = self.currentMainThreadBusyState() { - box.wasBusyBeforeDrain = true - } - box.hadAbandonedCaptureBeforeDrain = self.hasAbandonedMainThreadWork() - box.wasPenalizedBeforeDrain = self.isSnapshotXCTestChannelPenalized(bundleId: self.currentBundleId) - - // 2) `box.payload` above was already produced -- through the capture plan's recovery - // tiers -- while the probe is still blocked on `probeReleaseGate`, i.e. recovered before - // drain, not queued behind it. - completion.fulfill() - - // 3) Only now let the abandoned probe finish, then block this queue (never the test's - // main-thread wait) on the *real* drain signal -- the abandoned-work count reaching zero - // -- bounded so a revert that never drains fulfills `drained` anyway and lets the - // assertions below report the regression explicitly instead of just timing out. - probeReleaseGate.signal() - let drainDeadline = Date().addingTimeInterval(5) - while self.hasAbandonedMainThreadWork(), Date() < drainDeadline { - self.sleepFor(0.002) - } - drained.fulfill() - } - - wait(for: [completion], timeout: 15) - - // 1) Penalty/busy accounting. - XCTAssertTrue( - box.wasBusyBeforeDrain, - "expected RUNNER_BUSY while the \(entryPointName) modal probe timeout is outstanding" - ) - XCTAssertTrue( - box.hadAbandonedCaptureBeforeDrain, - "onAbandoned must retain the abandoned XCTest channel work for \(entryPointName)" - ) - XCTAssertTrue( - box.wasPenalizedBeforeDrain, - "a timed-out modal probe must penalize the XCTest snapshot channel for \(entryPointName)" - ) - - // 2) Recovered response before drain. - XCTAssertNotNil( - box.payload, - "\(entryPointName) must recover a payload through the capture plan while the probe drains" - ) - - // 3) Bounded, deterministic drain barrier, then release assertions. - wait(for: [drained], timeout: 6) - guard case .idle = currentMainThreadBusyState() else { - return XCTFail("expected the runner to be idle once the abandoned \(entryPointName) probe drained") - } - XCTAssertFalse( - hasAbandonedMainThreadWork(), - "the drained probe must release the main thread for \(entryPointName)" - ) - } - - func testBoundedSystemModalProbeTimeoutRecoversThenReleasesOnDrain() { - assertBoundedSystemModalProbeTimeoutRecoversThenReleasesOnDrain(entryPointName: "snapshotFast") { - target, options in - try self.snapshotFast(app: target, options: options) - } - } - - func testBoundedSystemModalProbeTimeoutRecoversThenReleasesOnDrainForSnapshotRaw() { - assertBoundedSystemModalProbeTimeoutRecoversThenReleasesOnDrain(entryPointName: "snapshotRaw") { - target, options in - try self.snapshotRaw(app: target, options: options) - } - } -#endif - - func testDispatchRecoverySkipsBookkeepingWhileXCTestChannelOccupied() { - // The #1244 recovery shape: the modal probe abandoned an XCTest query that is still grinding on - // main, the capture recovered independently, and its response is ready. The recovery loop must - // return it without re-entering the main queue for recorded-failure/retry bookkeeping (that hop - // would block behind the abandoned query and re-stall the command), and a later command must - // still see the runner busy until the abandoned work drains. Removing the guard regresses this. - let command = try! JSONDecoder().decode( - Command.self, - from: Data(#"{"command":"snapshot","commandId":"recovery-guard"}"#.utf8) - ) - let recovered = Response(ok: false, error: .targetAppUnavailable(bundleId: nil)) - - setAbandonedMainThreadWork(1) - defer { setAbandonedMainThreadWork(0) } - guard case .busy = currentMainThreadBusyState() else { - return XCTFail("expected RUNNER_BUSY while abandoned XCTest work is outstanding") - } - - var occupiedCalls = 0 - let occupied = try! executeDispatchedWithRecovery(command: command) { - occupiedCalls += 1 - return recovered - } - XCTAssertEqual(occupiedCalls, 1, "recovered response must not retry behind abandoned XCTest work") - XCTAssertEqual(occupied.ok, false) - - setAbandonedMainThreadWork(0) - guard case .idle = currentMainThreadBusyState() else { - return XCTFail("runner should be idle once the abandoned work drained") - } - var drainedCalls = 0 - _ = try! executeDispatchedWithRecovery(command: command) { - drainedCalls += 1 - return recovered - } - XCTAssertEqual(drainedCalls, 2, "with the channel free the read-only retry runs once") - } - - private func setAbandonedMainThreadWork(_ count: Int) { - mainThreadWorkLock.lock() - abandonedMainThreadWorkCount = count - abandonedMainThreadWorkSince = count > 0 ? Date(timeIntervalSinceNow: -1) : nil - mainThreadWorkLock.unlock() - } -#endif - } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotBackendCapabilities.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotBackendCapabilities.swift index e4c43b54bd..4a5b478639 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotBackendCapabilities.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotBackendCapabilities.swift @@ -86,81 +86,3 @@ enum SnapshotBackendKind: String, CaseIterable { } } } - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS -private struct SnapshotBackendParityFixture: Decodable { - struct Availability: Decodable { - let simulator: Bool - let physicalDevice: Bool - } - - struct Backend: Decodable { - let name: String - let forceable: Bool - let supportsRawProjection: Bool - let regularDepth: String - let hittable: String - let availability: Availability - } - - let backends: [Backend] -} - -extension RunnerTests { - private func loadSnapshotBackendParityFixture() throws -> SnapshotBackendParityFixture { - let fixtureURL = URL(fileURLWithPath: #filePath) - .deletingLastPathComponent() // AgentDeviceRunnerUITests - .deletingLastPathComponent() // AgentDeviceRunner - .deletingLastPathComponent() // runner - .deletingLastPathComponent() // apple - .deletingLastPathComponent() // repo root - .appendingPathComponent("contracts") - .appendingPathComponent("fixtures") - .appendingPathComponent("ios-snapshot-backends.json") - return try JSONDecoder().decode( - SnapshotBackendParityFixture.self, - from: Data(contentsOf: fixtureURL) - ) - } - - /// The JSON table is the cross-runtime declaration used by the TypeScript capability registry - /// and this runner. A backend case, forceability branch, projection/depth claim, or availability - /// change that is not classified in both implementations fails before an iOS smoke can drift. - func testSnapshotBackendDeclarationsMatchCapabilityFixture() throws { - let fixture = try loadSnapshotBackendParityFixture() - XCTAssertEqual( - fixture.backends.map(\.name), - SnapshotBackendKind.allCases.map(\.rawValue) - ) - - for expected in fixture.backends { - guard let backend = SnapshotBackendKind(rawValue: expected.name) else { - XCTFail("fixture contains an unknown snapshot backend: \(expected.name)") - continue - } - XCTAssertEqual(backend.isForceable, expected.forceable, expected.name) - XCTAssertEqual(backend.supportsRawProjection, expected.supportsRawProjection, expected.name) - XCTAssertEqual( - backend.regularDepthCapability.rawValue, - expected.regularDepth, - "regular depth capability: \(expected.name)" - ) - XCTAssertEqual( - backend.hittableSemantics, - expected.hittable, - "hittable semantics: \(expected.name)" - ) - XCTAssertEqual( - backend.isAvailable(on: .simulator), - expected.availability.simulator, - "simulator availability: \(expected.name)" - ) - XCTAssertEqual( - backend.isAvailable(on: .physicalDevice), - expected.availability.physicalDevice, - "physical-device availability: \(expected.name)" - ) - } - } -} -#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift index 16a999ab5d..b6e9946957 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift @@ -629,7 +629,7 @@ extension RunnerTests { // MARK: Outcome stamping - private func stampedSnapshotPayload( + func stampedSnapshotPayload( _ capture: SnapshotBackendCapture, backend: SnapshotBackendKind, state: String, @@ -666,493 +666,3 @@ extension RunnerTests { ) } } - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS -// MARK: - In-bundle unit tests - -extension RunnerTests { - private func planTestNode( - index: Int, - type: String, - label: String? = nil, - identifier: String? = nil, - hittable: Bool = false, - parentIndex: Int? = nil - ) -> PresentedNode { - SnapshotPresentation.singleElementRead( - RawAXNode( - index: index, - type: type, - label: label, - identifier: identifier, - value: nil, - rect: SnapshotRect(.zero), - enabled: true, - focused: nil, - selected: nil, - hittable: hittable, - depth: parentIndex == nil ? 0 : 1, - parentIndex: parentIndex, - hiddenContentAbove: nil, - hiddenContentBelow: nil - ) - ) - } - - func testSparsePayloadReasonMatrix() { - let root = planTestNode(index: 0, type: "Application", label: "Example App", hittable: true) - let window = planTestNode(index: 1, type: "Window", parentIndex: 0) - let button = planTestNode(index: 1, type: "Button", label: "Ok", hittable: true, parentIndex: 0) - let shell = planTestNode( - index: 1, - type: "Other", - identifier: "appShell", - parentIndex: 0 - ) - let serializationPlaceholder = planTestNode( - index: 2, - type: "Other", - label: "[object Object]", - parentIndex: 1 - ) - - // Labeled, hittable root over a bare window is still sparse. - XCTAssertNotNil(Self.sparsePayloadReason(DataPayload(nodes: [root, window], truncated: false))) - // Deadline-truncated near-empty sweep needs recovery even with one real control. - XCTAssertNotNil(Self.sparsePayloadReason(DataPayload(nodes: [root, button], truncated: true))) - // The same tiny tree from a completed sweep is a legitimately minimal screen. - XCTAssertNil(Self.sparsePayloadReason(DataPayload(nodes: [root, button], truncated: false))) - // Container metadata plus a stringified serialization placeholder is not readable UI. - XCTAssertNotNil( - Self.sparsePayloadReason( - DataPayload(nodes: [root, shell, serializationPlaceholder], truncated: false) - ) - ) - let actionableShell = planTestNode( - index: 1, - type: "Other", - identifier: "checkout", - hittable: true, - parentIndex: 0 - ) - XCTAssertNil( - Self.sparsePayloadReason(DataPayload(nodes: [root, actionableShell], truncated: false)) - ) - // Empty payloads are degraded. - XCTAssertNotNil(Self.sparsePayloadReason(DataPayload(nodes: [], truncated: false))) - } - - func testCollapsedLeafIndexesFlagsMergedContainersOnly() { - let root = planTestNode(index: 0, type: "Application", label: "App") - let merged = planTestNode( - index: 1, - type: "Other", - label: (0...30).map { "Row \($0), Tap" }.joined(separator: ", "), - parentIndex: 0 - ) - let prose = planTestNode( - index: 2, - type: "StaticText", - label: (0...30).map { "clause \($0)" }.joined(separator: ", "), - parentIndex: 0 - ) - XCTAssertEqual(Self.collapsedLeafIndexes([root, merged, prose]), [1]) - XCTAssertNil(Self.collapsedLeafIndexes([root, prose])) - } - - func testTerminalFailsClosedOnInteractiveAxFailureRegardlessOfSparseBest() { - // Interactive AX failure must invalidate + fail closed; a later tier's sparse synthetic-root - // "best" must never downgrade this to a returned-sparse payload (regression: best == nil guard). - XCTAssertEqual( - Self.resolveSnapshotPlanTerminal(terminal: .sparseWithFatalOnAXFailure, interactiveOnly: true), - .failClosed - ) - XCTAssertEqual( - Self.resolveSnapshotPlanTerminal(terminal: .sparseWithFatalOnAXFailure, interactiveOnly: false), - .sparseBest - ) - XCTAssertEqual( - Self.resolveSnapshotPlanTerminal(terminal: .throwOnAXFailure, interactiveOnly: true), - .throwAxFailure - ) - } - - func testXCTestChannelStateFirstFailureStampsDeferredCodeOnlyForDeferral() { - XCTAssertNil(Self.xcTestChannelStateFirstFailure(.normal)) - XCTAssertEqual(Self.xcTestChannelStateFirstFailure(.deferredToIndependentBackend)?.code, "deferred") - XCTAssertEqual(Self.xcTestChannelStateFirstFailure(.boundedXCTestProbe)?.code, "budget") - } - - func testSnapshotQualityCarriesPhaseTimingAtResponseLevel() { - let timing = SnapshotCaptureTiming(acquisitionMs: 12, presentationMs: 34) - let capture = SnapshotBackendCapture( - payload: DataPayload( - nodes: [planTestNode(index: 0, type: "Application", label: "App")], - truncated: false - ), - effectiveDepth: nil, - timing: timing - ) - - let payload = stampedSnapshotPayload( - capture, - backend: .recursiveTree, - state: "healthy", - reason: nil - ) - - XCTAssertEqual(payload.snapshotQuality?.timing, timing) - XCTAssertEqual(payload.nodes?.count, 1) - } - - func testStampedPayloadCarriesDisclosuresOnlyInTheVerdict() { - let root = planTestNode(index: 0, type: "Application", label: "App") - let merged = planTestNode( - index: 1, - type: "Other", - label: (0...30).map { "Tab \($0)" }.joined(separator: ", "), - parentIndex: 0 - ) - let coverage = SnapshotCustomActionCoverage( - read: 12, candidates: 19, truncated: 0, blocked: false) - let silent = stampedSnapshotPayload( - SnapshotBackendCapture( - payload: DataPayload(nodes: [root, merged], truncated: false), - effectiveDepth: nil, - customActions: coverage - ), - backend: .recursiveTree, - state: "healthy", - reason: nil - ) - XCTAssertNil(silent.message) - XCTAssertEqual(silent.snapshotQuality?.customActions, coverage) - XCTAssertEqual(silent.snapshotQuality?.collapsedLeafIndexes, [1]) - - let underlying = stampedSnapshotPayload( - SnapshotBackendCapture( - payload: DataPayload(message: "underlying", nodes: [root], truncated: false), - effectiveDepth: 4 - ), - backend: .privateAX, - state: "recovered", - reason: (reason: "tree capture timed out", code: "budget") - ) - XCTAssertEqual(underlying.message, "underlying") - } - - func testStampedPayloadTruncationTracksCompletenessNotRecoveryProvenance() { - let complete = SnapshotBackendCapture( - payload: DataPayload( - nodes: [ - planTestNode(index: 0, type: "Application", label: "App"), - planTestNode(index: 1, type: "Button", label: "Open", parentIndex: 0), - ], - truncated: false - ), - effectiveDepth: nil - ) - let deferred: (reason: String, code: String) = ( - "XCTest-backed snapshot tiers were deferred after recent slow accessibility work", "deferred" - ) - - // The CI signature behind `is absent ... capture was truncated`: a complete private AX - // tree selected while the XCTest channel is penalized is whole, and must say so. - let recovered = stampedSnapshotPayload( - complete, backend: .privateAX, state: "recovered", reason: deferred) - XCTAssertEqual(recovered.snapshotQuality?.state, "recovered") - XCTAssertEqual(recovered.truncated, false) - - let depthLimited = stampedSnapshotPayload( - SnapshotBackendCapture(payload: complete.payload, effectiveDepth: 56), - backend: .privateAX, state: "recovered", reason: deferred) - XCTAssertEqual(depthLimited.truncated, true) - - let cappedPayload = stampedSnapshotPayload( - SnapshotBackendCapture( - payload: DataPayload(nodes: complete.payload.nodes ?? [], truncated: true), - effectiveDepth: nil), - backend: .recursiveTree, state: "healthy", reason: nil) - XCTAssertEqual(cappedPayload.truncated, true) - - let sparse = stampedSnapshotPayload( - complete, backend: .querySweep, state: "sparse", - reason: ("snapshot returned no semantic controls or content", "sparse-tree")) - XCTAssertEqual(sparse.truncated, true) - } - - func testSnapshotQualityCarriesUnscopedQualityPayload() { - let quality = DataPayload( - nodes: [planTestNode(index: 0, type: "Application", label: "App")], - truncated: false - ) - let capture = SnapshotBackendCapture( - payload: quality, - effectiveDepth: nil, - qualityPayload: quality - ) - - let payload = stampedSnapshotPayload( - capture, - backend: .recursiveTree, - state: "healthy", - reason: nil - ) - - XCTAssertEqual(payload.qualityPayload?.nodes.count, 1) - XCTAssertEqual(payload.qualityPayload?.truncated, false) - XCTAssertNil(payload.qualityPayload?.scope) - } - - func testDirectPresentationDoesNotClaimPlanTiming() { - let options = PresentationOptions( - interactiveOnly: false, - depth: nil, - scope: nil, - raw: true - ) - let result = SnapshotPresentation.presentRaw( - SnapshotAcquisition( - hint: SnapshotPresentation.captureHint(for: options), - nodes: [], - truncated: false, - effectiveDepth: nil, - viewport: .infinite - ), - options: options - ) - let capture = Self.makeSnapshotBackendCapture(from: result) - - let payload = stampedSnapshotPayload( - capture, - backend: .recursiveTree, - state: "healthy", - reason: nil - ) - - XCTAssertNil(payload.snapshotQuality?.timing) - } - - /// The raw plan is derived from what each backend can actually serve, not from a second - /// hand-maintained list. Non-vacuity: flipping `querySweep.supportsRawProjection` to true adds it - /// to the plan and fails the first two assertions — which is exactly the shape of #1797 D4, a - /// `--raw` request answered by a backend that has no hierarchy to return. - func testRawDiagnosticPlanCarriesOnlyBackendsThatCanServeRaw() { - XCTAssertEqual(Self.rawDiagnosticPlan, [.recursiveTree, .privateAX]) - XCTAssertEqual( - SnapshotBackendKind.allCases.filter { !$0.supportsRawProjection }, [.querySweep]) - XCTAssertTrue(Self.rawDiagnosticPlan.allSatisfy(\.supportsRawProjection)) - // Tree-first error propagation is the raw plan's other contract (ADR 0004). - XCTAssertEqual(Self.rawDiagnosticPlan.first, .recursiveTree) - } - - /// A projection mismatch is a runner bug, not an accessibility failure: it must not take the - /// AX-failure terminal route (rethrow / fail-closed), just drop its tier with a named reason. - func testProjectionMismatchFailureIsStructuredAndNotAnAxFailure() { - let failure = Self.snapshotProjectionMismatchFailure( - .querySweep, requested: .raw, acquired: .regular) - XCTAssertEqual(failure.code, "IOS_SNAPSHOT_PROJECTION_MISMATCH") - XCTAssertTrue(failure.message.contains("queries")) - XCTAssertTrue(failure.message.contains("raw")) - XCTAssertFalse(Self.isAxSnapshotFailure(failure)) - } - - /// #1634 P2: the decoded wire field must reach capture options and its - /// applicable plan. A pinned REGULAR capture defers to privateAX-first; the - /// RAW diagnostic plan is never rerouted by the pin — raw keeps tree-first - /// error propagation, which is exactly why raw baselines are excluded from - /// corroboration daemon-side. - func testDecodedPreferredBackendReachesOptionsAndApplicablePlan() throws { - let json = #"{"command":"snapshot","preferredBackend":"private-ax"}"# - let command = try JSONDecoder().decode(Command.self, from: Data(json.utf8)) - let options = Self.presentationOptions(from: command) - XCTAssertEqual(options.preferredBackend, "private-ax") - XCTAssertFalse(options.raw) - - let treated = Self.snapshotXCTestChannelTreatedAsPenalized( - penalized: false, preferredBackend: options.preferredBackend) - let pinned = Self.effectiveSnapshotCapturePlan( - Self.regularVisiblePlan, - xCTestChannelPenalized: treated, - preferredBackend: options.preferredBackend - ) - XCTAssertEqual(pinned.plan, [.privateAX]) - XCTAssertEqual(pinned.xCTestChannelState, .deferredToIndependentBackend) - - let raw = Self.effectiveSnapshotCapturePlan( - Self.rawDiagnosticPlan, - xCTestChannelPenalized: treated, - preferredBackend: options.preferredBackend - ) - XCTAssertEqual(raw.plan, Self.rawDiagnosticPlan) - - // A command without the field decodes to no pin and a normal plan. - let bare = try JSONDecoder().decode( - Command.self, from: Data(#"{"command":"snapshot"}"#.utf8)) - XCTAssertNil(Self.presentationOptions(from: bare).preferredBackend) - } - - /// #1635: the force seam must select the recursive tree even when the XCTest - /// channel is currently penalized. Without the preferred-backend argument, - /// this call returns the independent private-AX recovery plan instead. - func testPreferredTreeBackendPinsRegularPlanAndLeavesStructuredEvidence() { - let forced = Self.effectiveSnapshotCapturePlan( - Self.regularVisiblePlan, - xCTestChannelPenalized: true, - preferredBackend: SnapshotBackendKind.recursiveTree.rawValue - ) - XCTAssertEqual(forced.plan, [.recursiveTree]) - XCTAssertEqual(forced.xCTestChannelState, .normal) - XCTAssertEqual( - Self.xcTestChannelStateFirstFailure( - forced.xCTestChannelState, - preferredBackend: forced.preferredBackend?.rawValue - )?.code, - "requested-backend" - ) - } - - /// Same-backend evidence probes: a daemon-pinned private-AX capture takes the - /// penalized route even with a healthy channel, so tap-outcome corroboration - /// baselines and probes are always captured by the same backend (backends are - /// never comparable views of a screen). Composed with the plan rule, the pin - /// yields the privateAX-first deferred plan. - func testPreferredPrivateAXBackendPlansAsPenalized() { - XCTAssertTrue( - Self.snapshotXCTestChannelTreatedAsPenalized(penalized: false, preferredBackend: "private-ax")) - XCTAssertTrue( - Self.snapshotXCTestChannelTreatedAsPenalized(penalized: true, preferredBackend: nil)) - XCTAssertFalse( - Self.snapshotXCTestChannelTreatedAsPenalized(penalized: false, preferredBackend: nil)) - XCTAssertFalse( - Self.snapshotXCTestChannelTreatedAsPenalized(penalized: false, preferredBackend: "tree")) - - let pinned = Self.effectiveSnapshotCapturePlan( - Self.regularVisiblePlan, - xCTestChannelPenalized: Self.snapshotXCTestChannelTreatedAsPenalized( - penalized: false, preferredBackend: "private-ax" - ), - preferredBackend: "private-ax" - ) - XCTAssertEqual(pinned.plan, [.privateAX]) - XCTAssertEqual(pinned.xCTestChannelState, .deferredToIndependentBackend) - } - - func testEffectiveSnapshotCapturePlanDefersXCTestBackedTiersOnlyWhenPenalizedRegularPlan() { - let regular = Self.effectiveSnapshotCapturePlan( - Self.regularVisiblePlan, - xCTestChannelPenalized: true - ) - XCTAssertEqual(regular.plan, [.privateAX]) - XCTAssertEqual(regular.xCTestChannelState, .deferredToIndependentBackend) - XCTAssertNil(regular.treeCaptureSliceBudgetOverride) - - let unpenalized = Self.effectiveSnapshotCapturePlan( - Self.regularVisiblePlan, - xCTestChannelPenalized: false - ) - XCTAssertEqual(unpenalized.plan, Self.regularVisiblePlan) - XCTAssertEqual(unpenalized.xCTestChannelState, .normal) - XCTAssertNil(unpenalized.treeCaptureSliceBudgetOverride) - - // The raw diagnostic plan preserves tree-first error propagation even under penalty. - let raw = Self.effectiveSnapshotCapturePlan( - Self.rawDiagnosticPlan, - xCTestChannelPenalized: true - ) - XCTAssertEqual(raw.plan, Self.rawDiagnosticPlan) - XCTAssertEqual(raw.xCTestChannelState, .normal) - XCTAssertNil(raw.treeCaptureSliceBudgetOverride) - } - - func testEffectiveSnapshotCapturePlanUsesBoundedXCTestProbeWhenNoIndependentBackendRuns() { - let physicalDevicePlan = Self.effectiveSnapshotCapturePlan( - Self.regularVisiblePlan, - xCTestChannelPenalized: true, - availableBackends: [.recursiveTree, .querySweep] - ) - - XCTAssertEqual(physicalDevicePlan.plan, [.recursiveTree, .querySweep]) - XCTAssertEqual(physicalDevicePlan.xCTestChannelState, .boundedXCTestProbe) - XCTAssertEqual( - physicalDevicePlan.treeCaptureSliceBudgetOverride, - Self.penalizedXCTestProbeTreeSliceBudget - ) - } - - func testSnapshotXCTestChannelPenaltyMatchesBundleAndExpires() { - defer { - snapshotXCTestChannelPenaltyBundleId = nil - snapshotXCTestChannelPenaltyUntil = .distantPast - } - - penalizeSnapshotXCTestChannel(bundleId: "xyz.blueskyweb.app", reason: "test") - XCTAssertTrue(isSnapshotXCTestChannelPenalized(bundleId: "xyz.blueskyweb.app")) - XCTAssertFalse(isSnapshotXCTestChannelPenalized(bundleId: "com.other.app")) - - // A penalty recorded without a bundle applies to any current target. - penalizeSnapshotXCTestChannel(bundleId: nil, reason: "test") - XCTAssertTrue(isSnapshotXCTestChannelPenalized(bundleId: "com.other.app")) - - // Expired penalties stop applying. - snapshotXCTestChannelPenaltyUntil = Date(timeIntervalSinceNow: -1) - XCTAssertFalse(isSnapshotXCTestChannelPenalized(bundleId: "com.other.app")) - } - - func testAbandonedMainThreadWorkSkipsOnlyXCTestBackedSnapshotTiers() { - abandonedMainThreadWorkCount = 1 - defer { abandonedMainThreadWorkCount = 0 } - - XCTAssertTrue(shouldSkipSnapshotBackendForAbandonedMainThreadWork(.recursiveTree)) - XCTAssertTrue(shouldSkipSnapshotBackendForAbandonedMainThreadWork(.querySweep)) - XCTAssertFalse(shouldSkipSnapshotBackendForAbandonedMainThreadWork(.privateAX)) - } - -#if os(iOS) - /// #2403: a plan pinned to private AX serves a regular `--depth` request through acquisition - /// and presentation. With a backend depth gate in `captureWithBackend`, private AX returns no - /// capture, the plan falls through to the synthetic sparse root, and the daemon rejects that - /// zero-rect root as a missing viewport. - func testPrivateAXPinnedRegularDepthReachesAcquisitionAndPresentation() throws { - app.launchArguments = ["--agent-device-selector-read-regression"] - app.launch() - currentApp = app - currentBundleId = nil - defer { - currentApp = nil - clearPrivateAXAcceptedDepth(reason: "test-cleanup") - app.terminate() - } - func capture(depth: Int?) throws -> DataPayload { - try runSnapshotCapturePlan( - Self.regularVisiblePlan, - app: app, - options: PresentationOptions( - interactiveOnly: false, - depth: depth, - scope: nil, - raw: false, - preferredBackend: SnapshotBackendKind.privateAX.rawValue - ), - terminal: .sparseWithFatalOnAXFailure - ) - } - - let capped = try capture(depth: 1) - - let quality = try XCTUnwrap(capped.snapshotQuality) - XCTAssertEqual(quality.backend, SnapshotBackendKind.privateAX.rawValue) - XCTAssertNotEqual(quality.state, "sparse") - let nodes = try XCTUnwrap(capped.nodes) - XCTAssertGreaterThan(nodes.count, 1) - XCTAssertEqual(nodes.map(\.depth).max(), 1) - XCTAssertNotEqual(nodes[0].rect, SnapshotRect(x: 0, y: 0, width: 0, height: 0)) - XCTAssertTrue(nodes.contains { $0.label == "Readable target" }) - - // The presented cut only ever narrows the unscoped capture from the same backend. - let unscoped = try XCTUnwrap(try capture(depth: nil).nodes) - XCTAssertLessThanOrEqual(nodes.count, unscoped.count) - } -#endif -} -#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotTiming.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotTiming.swift index 442aa6d1b9..288e4cca5a 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotTiming.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotTiming.swift @@ -73,7 +73,7 @@ extension RunnerTests { /// The penalty breaker observes only acquisition facts. Presentation is a separate phase and /// cannot arm the breaker, even when it is slower than the acquisition that produced the tree. - private static func snapshotXCTestPenaltyReason( + static func snapshotXCTestPenaltyReason( kind: SnapshotBackendKind, attempt: SnapshotBackendAttempt, slowThresholdMs: Double @@ -107,79 +107,3 @@ extension RunnerTests { ) } } - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS -extension RunnerTests { - func testXCTestPenaltyDecisionSeparatesAcquisitionAndPresentation() { - let slowPresentation = SnapshotBackendAttempt( - outcome: .noCapture, - timing: SnapshotCaptureTiming(acquisitionMs: 100, presentationMs: 4_000) - ) - XCTAssertNil( - Self.snapshotXCTestPenaltyReason( - kind: .recursiveTree, - attempt: slowPresentation, - slowThresholdMs: 3_000 - ) - ) - - let slowAcquisition = SnapshotBackendAttempt( - outcome: .noCapture, - timing: SnapshotCaptureTiming(acquisitionMs: 3_001, presentationMs: 100) - ) - XCTAssertEqual( - Self.snapshotXCTestPenaltyReason( - kind: .recursiveTree, - attempt: slowAcquisition, - slowThresholdMs: 3_000 - ), - "slow_tree_capture_3001ms" - ) - - let timeout = SnapshotCaptureFailure( - code: Self.xCTestSnapshotTimeoutCode, - message: "test timeout", - hint: "test" - ) - let acquisitionFailure = SnapshotBackendAttempt( - outcome: .failed(timeout, phase: .acquisition), - timing: SnapshotCaptureTiming(acquisitionMs: 100, presentationMs: 100) - ) - XCTAssertEqual( - Self.snapshotXCTestPenaltyReason( - kind: .recursiveTree, - attempt: acquisitionFailure, - slowThresholdMs: 3_000 - ), - "tree_backend_timeout" - ) - - let presentationFailure = SnapshotBackendAttempt( - outcome: .failed(timeout, phase: .presentation), - timing: SnapshotCaptureTiming(acquisitionMs: 100, presentationMs: 100) - ) - XCTAssertNil( - Self.snapshotXCTestPenaltyReason( - kind: .recursiveTree, - attempt: presentationFailure, - slowThresholdMs: 3_000 - ) - ) - } - - func testSnapshotPhaseTimerReportsAcquisitionAndPresentationSeparately() { - var now = Date(timeIntervalSinceReferenceDate: 100) - var timer = SnapshotPhaseTimer(now: { now }) - - _ = timer.measure(.acquisition) { - now = now.addingTimeInterval(2) - } - _ = timer.measure(.presentation) { - now = now.addingTimeInterval(5) - } - - XCTAssertEqual(timer.timing.acquisitionMs, 2_000, accuracy: 0.001) - XCTAssertEqual(timer.timing.presentationMs, 5_000, accuracy: 0.001) - } -} -#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift index a74bcd7908..74723a7f44 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift @@ -170,107 +170,3 @@ extension RunnerTests { + " fallbackAttempted=\(fallbackAttempted)" } } - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS) -extension RunnerTests { - func testSynthesizedGesturePolicyMarkerWritesOncePerKindUntilTheDecisionChanges() { - var written: [String] = [] - runnerMarkerWriter = { written.append($0) } - defer { - runnerMarkerWriter = { NSLog("%@", $0) } - invalidateCachedTarget(reason: "unit_test_cleanup") - } - logSynthesizedGesturePolicyDecision(kind: .coordinateTap, context: nil, fallbackAttempted: false) - logSynthesizedGesturePolicyDecision(kind: .coordinateTap, context: nil, fallbackAttempted: false) - XCTAssertEqual(written.count, 1, "a repeated decision writes no second line") - logSynthesizedGesturePolicyDecision(kind: .scroll, context: nil, fallbackAttempted: false) - XCTAssertEqual(written.count, 2, "each gesture kind states its own decision") - logSynthesizedGesturePolicyDecision(kind: .coordinateTap, context: nil, fallbackAttempted: true) - XCTAssertEqual(written.count, 3, "a changed decision writes a new line") - resetTargetBoundState() - logSynthesizedGesturePolicyDecision(kind: .coordinateTap, context: nil, fallbackAttempted: true) - XCTAssertEqual(written.count, 4, "a rebind states the same decision once more") - } -} -#endif - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS -extension RunnerTests { - func testSynthesizedFallbackPolicyRequiresPrivateSynthesisForScrollWhenAxUnavailableOrUnknown() { - XCTAssertFalse( - SynthesizedFallbackPolicy.privateSynthesisRequired - .allowsXCTestCoordinateFallback(accessibilityHealth: .unavailable) - ) - XCTAssertFalse( - SynthesizedFallbackPolicy.privateSynthesisRequired - .allowsXCTestCoordinateFallback(accessibilityHealth: .unknown) - ) - XCTAssertFalse( - SynthesizedFallbackPolicy.privateSynthesisRequired - .allowsXCTestCoordinateFallback(accessibilityHealth: .healthy) - ) - } - - func testSynthesizedDragCoordinateFallbackAllowsUnknownButNotUnavailableAccessibility() { - XCTAssertTrue( - SynthesizedFallbackPolicy.xctestCoordinateWhenAccessibilityAvailable - .allowsXCTestCoordinateFallback(accessibilityHealth: .healthy) - ) - XCTAssertFalse( - SynthesizedFallbackPolicy.xctestCoordinateWhenAccessibilityAvailable - .allowsXCTestCoordinateFallback(accessibilityHealth: .unavailable) - ) - XCTAssertTrue( - SynthesizedFallbackPolicy.xctestCoordinateWhenAccessibilityAvailable - .allowsXCTestCoordinateFallback(accessibilityHealth: .unknown) - ) - } - - /// Keyboard-policy semantics only. Which command gets which policy is the table below; a probe - /// that is merely permitted still costs a live AX fetch, so the two questions stay separate. - func testSynthesizedKeyboardPolicyAllowsProbeOnlyWhenAccessibilityPermitsIt() { - XCTAssertFalse( - SynthesizedKeyboardPolicy.whenAccessibilityHealthy - .allowsProbe(accessibilityHealth: .unknown) - ) - XCTAssertTrue( - SynthesizedKeyboardPolicy.requiredWhenAvailable - .allowsProbe(accessibilityHealth: .unknown) - ) - XCTAssertFalse( - SynthesizedKeyboardPolicy.requiredWhenAvailable - .allowsProbe(accessibilityHealth: .unavailable) - ) - } - - func testSynthesizedGesturePoliciesMatchCommandContracts() { - XCTAssertEqual( - synthesizedGesturePolicy(.coordinateTap), - SynthesizedGesturePolicy( - keyboardPolicy: .never, - fallbackPolicy: .xctestCoordinateAllowed - ) - ) - XCTAssertEqual( - synthesizedGesturePolicy(.scroll), - SynthesizedGesturePolicy( - keyboardPolicy: .requiredWhenAvailable, - fallbackPolicy: .privateSynthesisRequired - ) - ) - XCTAssertEqual( - synthesizedGesturePolicy(.synthesizedDrag), - SynthesizedGesturePolicy( - keyboardPolicy: .requiredWhenAvailable, - fallbackPolicy: .xctestCoordinateWhenAccessibilityAvailable - ) - ) - } - - func testCoordinateTapTextInputProbeSkipsPenalizedXCTestChannel() { - XCTAssertTrue(shouldProbeCoordinateTapTextInput(xCTestChannelPenalized: false)) - XCTAssertFalse(shouldProbeCoordinateTapTextInput(xCTestChannelPenalized: true)) - } - -} -#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputCandidatePolicy.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputCandidatePolicy.swift index a46e48e80f..e8dc68912b 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputCandidatePolicy.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputCandidatePolicy.swift @@ -14,34 +14,3 @@ func isCoordinateTextInputCandidate( && point.y >= frame.minY - tolerance && point.y <= frame.maxY + tolerance } - -extension RunnerTests { -#if AGENT_DEVICE_RUNNER_UNIT_TESTS - func testCoordinateTextInputCandidateMustBeEnabledAndContainTheTouchPoint() { - let frame = CGRect(x: 10, y: 20, width: 100, height: 40) - let point = CGPoint(x: 50, y: 40) - - XCTAssertTrue( - isCoordinateTextInputCandidate( - enabled: true, - frame: frame, - point: point - ) - ) - XCTAssertFalse( - isCoordinateTextInputCandidate( - enabled: false, - frame: frame, - point: point - ) - ) - XCTAssertFalse( - isCoordinateTextInputCandidate( - enabled: true, - frame: frame, - point: CGPoint(x: 200, y: 200) - ) - ) - } -#endif -} diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Transport.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Transport.swift index e694f41baa..9db9f0f7e7 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Transport.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Transport.swift @@ -200,7 +200,7 @@ extension RunnerTests { /// Returns true when this send duplicated a still-executing commandId and was attached as a /// waiter of the in-flight execution. Otherwise marks the commandId in flight and returns /// false so the caller enqueues the (single) execution. - private func attachToInFlightCommandIfNeeded( + func attachToInFlightCommandIfNeeded( command: Command, completion: @escaping ((data: Data, shouldFinish: Bool)) -> Void ) -> Bool { @@ -221,7 +221,7 @@ extension RunnerTests { return false } - private func deliverCommandResult( + func deliverCommandResult( command: Command, result: (data: Data, shouldFinish: Bool), completion: ((data: Data, shouldFinish: Bool)) -> Void @@ -239,45 +239,6 @@ extension RunnerTests { } } -#if AGENT_DEVICE_RUNNER_UNIT_TESTS - func testDuplicateCommandIdCoalescesOntoInFlightExecution() throws { - let command = try JSONDecoder().decode( - Command.self, - from: Data(#"{"command":"snapshot","commandId":"snapshot-coalesce"}"#.utf8) - ) - var primaryData: Data? - var waiterData: Data? - defer { - inFlightCommandIds.removeAll() - inFlightCommandWaiters.removeAll() - } - - XCTAssertFalse( - attachToInFlightCommandIfNeeded(command: command) { result in - primaryData = result.data - } - ) - XCTAssertTrue( - attachToInFlightCommandIfNeeded(command: command) { result in - waiterData = result.data - } - ) - - let delivered = Data("single-result".utf8) - deliverCommandResult( - command: command, - result: (delivered, false) - ) { result in - primaryData = result.data - } - - XCTAssertEqual(primaryData, delivered) - XCTAssertEqual(waiterData, delivered) - XCTAssertFalse(inFlightCommandIds.contains("snapshot-coalesce")) - XCTAssertNil(inFlightCommandWaiters["snapshot-coalesce"]) - } -#endif - // MARK: - Response Encoding private func jsonResponse(status: Int, response: Response) -> Data { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TvRemote.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TvRemote.swift index 252352baf0..b312933bb8 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TvRemote.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TvRemote.swift @@ -18,28 +18,6 @@ enum TvRemoteButton: String { } extension RunnerTests { -#if AGENT_DEVICE_RUNNER_UNIT_TESTS - func testTvRemoteButtonMappingAcceptsSupportedNamesAndRejectsUnknown() { - let supported = [ - ("select", "select"), - ("SELECT", "select"), - ("menu", "menu"), - ("home", "home"), - ("up", "up"), - ("down", "down"), - ("left", "left"), - ("right", "right"), - ] - for (raw, expected) in supported { - XCTAssertEqual(tvRemoteButton(from: raw)?.rawValue, expected) - } - - for raw in [String?(nil), "", "volumeUp", "select "] { - XCTAssertNil(tvRemoteButton(from: raw)) - } - } -#endif - func resolveTvRemoteDoublePressDelay() -> TimeInterval { guard let raw = ProcessInfo.processInfo.environment["AGENT_DEVICE_TV_REMOTE_DOUBLE_PRESS_DELAY_MS"], diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AXSnapshotFallbackTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AXSnapshotFallbackTests.swift new file mode 100644 index 0000000000..cbcad722a9 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AXSnapshotFallbackTests.swift @@ -0,0 +1,531 @@ +import XCTest +import AgentDeviceSnapshotPresentation + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +// MARK: - In-bundle unit tests + +extension RunnerTests { + func testPrivateAXAttemptDepthsAppliesRememberedDepth() { + XCTAssertEqual( + Self.privateAXAttemptDepths(requestedDepth: 64, rememberedDepth: nil), + [64, 56, 40, 24, 12] + ) + XCTAssertEqual( + Self.privateAXAttemptDepths(requestedDepth: 64, rememberedDepth: 56), + [56, 40, 24, 12] + ) + XCTAssertEqual(Self.privateAXAttemptDepths(requestedDepth: 64, rememberedDepth: 12), [12]) + // Remembered at/above the requested depth changes nothing. + XCTAssertEqual( + Self.privateAXAttemptDepths(requestedDepth: 64, rememberedDepth: 64), + [64, 56, 40, 24, 12] + ) + // A shallower explicit request keeps its own rungs; deeper stale memory is ignored. + XCTAssertEqual(Self.privateAXAttemptDepths(requestedDepth: 24, rememberedDepth: 56), [24, 12]) + } + + /// Executed producer contract for the #1627 review blocker: a frontier whose + /// live element vanished, and one whose re-rooted request fails, must BOTH + /// count as missed — an all-miss extension reporting itself drained would + /// present a capped capture as complete. Goes red if either miss-path + /// increment in extendSnapshotFrontiers is removed. + func testDeepExtensionCountsMissedFrontiers() { + // Element vanished (list churn between serialization and extension): the + // fabricated snapshot answers nil for accessibilityElement — missed, and + // no request call is consumed. (An explicit nil property: bare NSObject + // resolves the key through a UIKit category and would take the call path.) + let orphan = RunnerAXSnapshotFrontier() + orphan.snapshot = FrontierSnapshotWithoutElementForTesting() + orphan.node = NSMutableDictionary() + // Re-rooted request fails: the element resolves but the client cannot + // serve requestSnapshotForElement — one consumed call AND a miss. + let unreachable = RunnerAXSnapshotFrontier() + unreachable.snapshot = FrontierSnapshotWithElementForTesting() + unreachable.node = NSMutableDictionary() + + var nodeCount = 0 + var truncated = ObjCBool(false) + let outcome = RunnerAXSnapshotBridge.extend( + NSMutableArray(array: [orphan, unreachable]), + axClient: NSObject(), + attributes: [], + maxDepth: 56, + maxNodes: 5_000, + nodeCount: &nodeCount, + truncated: &truncated, + callsAllowed: 8, + mergedLeaves: nil, + deadline: nil + ) + + XCTAssertEqual(outcome?[RunnerAXSnapshotDeepExtensionMissedKey] as? Int, 2) + XCTAssertEqual(outcome?[RunnerAXSnapshotDeepExtensionCallsKey] as? Int, 1) + XCTAssertEqual(outcome?[RunnerAXSnapshotDeepExtensionPendingKey] as? Int, 0) + XCTAssertEqual(outcome?[RunnerAXSnapshotDeepExtensionNodesAddedKey] as? Int, 0) + XCTAssertFalse(truncated.boolValue) + // And the consumer verdict over exactly this outcome: still depth-limited. + XCTAssertTrue( + Self.privateAXDepthLimited( + effectiveDepth: 56, requestedDepth: 64, pendingFrontiers: 0, missedFrontiers: 2)) + } + + func testPrivateAXDepthLimitedRequiresEveryFrontierResolved() { + // Un-capped capture is never depth-limited, extension or not. + XCTAssertFalse( + Self.privateAXDepthLimited( + effectiveDepth: 64, requestedDepth: 64, pendingFrontiers: nil, missedFrontiers: nil)) + // Capped with no extension outcome (never ran) stays depth-limited. + XCTAssertTrue( + Self.privateAXDepthLimited( + effectiveDepth: 56, requestedDepth: 64, pendingFrontiers: nil, missedFrontiers: nil)) + // Fully drained extension clears the verdict. + XCTAssertFalse( + Self.privateAXDepthLimited( + effectiveDepth: 56, requestedDepth: 64, pendingFrontiers: 0, missedFrontiers: 0)) + // Budget exhaustion (pending frontiers) keeps it. + XCTAssertTrue( + Self.privateAXDepthLimited( + effectiveDepth: 56, requestedDepth: 64, pendingFrontiers: 2, missedFrontiers: 0)) + // The #1627 review blocker: an all-miss extension (elements vanished or + // re-rooted requests failed) resolved nothing — it must NOT present the + // capture as complete just because the queue emptied. + XCTAssertTrue( + Self.privateAXDepthLimited( + effectiveDepth: 56, requestedDepth: 64, pendingFrontiers: 0, missedFrontiers: 8)) + } + + func testPrivateAXAcceptedDepthMemoryMatchesBundleProcessAndExpires() { + defer { clearPrivateAXAcceptedDepth(reason: "test-cleanup") } + + rememberPrivateAXAcceptedDepth(bundleId: "xyz.blueskyweb.app", processIdentifier: 111, depth: 56) + XCTAssertEqual( + rememberedPrivateAXAcceptedDepth(bundleId: "xyz.blueskyweb.app", processIdentifier: 111), + 56 + ) + XCTAssertNil(rememberedPrivateAXAcceptedDepth(bundleId: "com.other.app", processIdentifier: 111)) + // A relaunch changes the PID; the new process must re-probe the full depth even inside the + // TTL, and an unknown current PID (post-invalidation) must never match. + XCTAssertNil(rememberedPrivateAXAcceptedDepth(bundleId: "xyz.blueskyweb.app", processIdentifier: 222)) + XCTAssertNil(rememberedPrivateAXAcceptedDepth(bundleId: "xyz.blueskyweb.app", processIdentifier: nil)) + + // Expired memory stops applying (the expiry re-probes the full requested depth). + privateAXAcceptedDepthUntil = Date(timeIntervalSinceNow: -1) + XCTAssertNil(rememberedPrivateAXAcceptedDepth(bundleId: "xyz.blueskyweb.app", processIdentifier: 111)) + } + + func testPrivateAXAcceptedDepthMemoryRequiresProcessIdentifierToRecord() { + defer { clearPrivateAXAcceptedDepth(reason: "test-cleanup") } + + rememberPrivateAXAcceptedDepth(bundleId: "xyz.blueskyweb.app", processIdentifier: nil, depth: 56) + XCTAssertNil( + rememberedPrivateAXAcceptedDepth(bundleId: "xyz.blueskyweb.app", processIdentifier: nil) + ) + } + + func testViewportReadSkippedWhileXCTestChannelPenalized() { + // Pins the viewport fast path (#1587 review): every penalized private AX capture used to burn + // the full 1s main-thread timeout on a doomed viewport read before falling back. + currentBundleId = "xyz.blueskyweb.app" + defer { + currentBundleId = nil + clearSnapshotXCTestChannelPenalty(reason: "test-cleanup") + abandonedMainThreadWorkCount = 0 + } + + XCTAssertTrue(shouldReadPrivateAXViewportViaXCTest()) + + penalizeSnapshotXCTestChannel(bundleId: "xyz.blueskyweb.app", reason: "test") + XCTAssertFalse(shouldReadPrivateAXViewportViaXCTest()) + + clearSnapshotXCTestChannelPenalty(reason: "test") + XCTAssertTrue(shouldReadPrivateAXViewportViaXCTest()) + + abandonedMainThreadWorkCount = 1 + XCTAssertFalse(shouldReadPrivateAXViewportViaXCTest()) + } + + /// The wire field must reach both capture options AND the backend pin: custom + /// actions are only readable through the private AX client, so a capture that + /// asked for them but planned the XCTest tree backend would return a payload + /// that structurally cannot carry them. + func testCustomActionsRequestPinsPrivateAXBackend() throws { + let asked = try JSONDecoder().decode( + Command.self, from: Data(#"{"command":"snapshot","customActions":true}"#.utf8)) + let options = Self.presentationOptions(from: asked) + XCTAssertTrue(options.customActions) + XCTAssertEqual(options.preferredBackend, SnapshotBackendKind.privateAX.rawValue) + XCTAssertTrue( + Self.snapshotXCTestChannelTreatedAsPenalized( + penalized: false, preferredBackend: options.preferredBackend)) + + // An explicit pin is never overwritten by the implied one. + let pinned = try JSONDecoder().decode( + Command.self, + from: Data(#"{"command":"snapshot","customActions":true,"preferredBackend":"tree"}"#.utf8)) + XCTAssertEqual(Self.presentationOptions(from: pinned).preferredBackend, "tree") + + // And the default capture neither asks nor pins. + let bare = try JSONDecoder().decode(Command.self, from: Data(#"{"command":"snapshot"}"#.utf8)) + XCTAssertFalse(Self.presentationOptions(from: bare).customActions) + XCTAssertNil(Self.presentationOptions(from: bare).preferredBackend) + } + + /// A request-pinned backend degraded nothing, so its verdict must not claim + /// slow accessibility work — that reason drives a user-facing warning. + func testRequestPinnedBackendReportsItsOwnReason() { + let requested = Self.xcTestChannelStateFirstFailure( + .deferredToIndependentBackend, requestPinnedBackend: true) + XCTAssertEqual(requested?.code, "requested-backend") + XCTAssertFalse(requested?.reason.contains("slow accessibility work") ?? true) + + // The circuit breaker's own deferral keeps its established code and wording. + let breaker = Self.xcTestChannelStateFirstFailure(.deferredToIndependentBackend) + XCTAssertEqual(breaker?.code, "deferred") + + // The bounded probe and the healthy plan are untouched by the new flag. + XCTAssertEqual( + Self.xcTestChannelStateFirstFailure(.boundedXCTestProbe, requestPinnedBackend: true)?.code, + "budget") + XCTAssertNil(Self.xcTestChannelStateFirstFailure(.normal, requestPinnedBackend: true)) + } + + /// The disclosure only exists if the counts survive the bridge boundary, and + /// "did not ask" must stay distinguishable from "read none". + func testCustomActionCoverageParsesOnlyCompletePairs() { + let complete: [String: Any] = [ + RunnerAXSnapshotCustomActionsReadKey: 12, + RunnerAXSnapshotCustomActionsCandidatesKey: 19, + RunnerAXSnapshotCustomActionsTruncatedKey: 2, + RunnerAXSnapshotCustomActionsBlockedKey: true, + ] + let coverage = Self.privateAXCustomActionCoverage(complete) + XCTAssertEqual(coverage?.read, 12) + XCTAssertEqual(coverage?.candidates, 19) + XCTAssertEqual(coverage?.truncated, 2) + XCTAssertEqual(coverage?.blocked, true) + + // Absent key = the capture never asked; it must not read as (0, 0), which + // would warn "0 of 0" on every default capture. + XCTAssertNil(Self.privateAXCustomActionCoverage(nil)) + // The bridge in this target always writes all four keys, so a partial + // dictionary is malformed and is dropped whole. + for key in complete.keys { + var partial = complete + partial.removeValue(forKey: key) + XCTAssertNil(Self.privateAXCustomActionCoverage(partial), "missing \(key)") + } + } + + /// The AX call cannot be cancelled once issued, so the read deadline frees + /// only the caller — the call keeps running. Without containment, repeating + /// `snapshot --actions` against a wedged element would stack orphaned reads, + /// all sharing one XCAXClient. This pins the containment: one serial queue and + /// a single-flight refusal that adds no work while a read is outstanding. + func testHungCustomActionReadIsContainedAndRecovers() { + let hung = HungAXClientForTesting() + let element = NSObject() + let dispatchesBefore = RunnerAXSnapshotBridge.customActionReadDispatchCount() + let blockedBefore = RunnerAXSnapshotBridge.customActionReadBlockedCount() + defer { hung.release() } + + // 1. First read wedges. The caller is freed by the deadline, but the call is + // still out there, so it stays counted in flight. + var completed = ObjCBool(true) + let firstStarted = Date() + let first = RunnerAXSnapshotBridge.customActionNames( + forElement: element, axClient: hung, completed: &completed) + XCTAssertNil(first) + XCTAssertFalse(completed.boolValue) + XCTAssertGreaterThanOrEqual(-firstStarted.timeIntervalSinceNow, 0.9) + XCTAssertEqual(RunnerAXSnapshotBridge.customActionReadsInFlight(), 1) + XCTAssertEqual( + RunnerAXSnapshotBridge.customActionReadDispatchCount(), dispatchesBefore + 1) + + // 2. Repeats do NOT accumulate: no new dispatch, still exactly one in + // flight, and every repeat is refused by single-flight admission. + for _ in 0..<5 { + XCTAssertNil( + RunnerAXSnapshotBridge.customActionNames( + forElement: element, axClient: hung, completed: &completed)) + XCTAssertFalse(completed.boolValue) + } + XCTAssertEqual(RunnerAXSnapshotBridge.customActionReadsInFlight(), 1) + XCTAssertEqual( + RunnerAXSnapshotBridge.customActionReadDispatchCount(), dispatchesBefore + 1) + XCTAssertEqual(RunnerAXSnapshotBridge.customActionReadBlockedCount(), blockedBefore + 5) + + // 3. A capture in that state discloses the skip rather than presenting the + // unread elements as action-free — and spends no read budget doing it. + let leaf = RunnerAXSnapshotFrontier() + leaf.snapshot = FrontierSnapshotWithElementForTesting() + leaf.node = NSMutableDictionary(dictionary: ["label": "feedItem", "children": []]) + let coverage = RunnerAXSnapshotBridge.annotateCustomActions( + onMergedLeaves: [leaf], axClient: hung, limit: 12, rootFrame: .zero, deadline: nil) + XCTAssertEqual(coverage[RunnerAXSnapshotCustomActionsBlockedKey] as? Bool, true) + XCTAssertEqual(coverage[RunnerAXSnapshotCustomActionsReadKey] as? Int, 0) + XCTAssertEqual(coverage[RunnerAXSnapshotCustomActionsCandidatesKey] as? Int, 1) + XCTAssertEqual( + RunnerAXSnapshotBridge.customActionReadDispatchCount(), dispatchesBefore + 1) + XCTAssertEqual( + Self.privateAXCustomActionCoverage(coverage), + SnapshotCustomActionCoverage(read: 0, candidates: 1, truncated: 0, blocked: true)) + + // 4. Recovery: once the wedged call returns, reads resume by themselves. + hung.release() + let recovered = expectation(description: "in-flight drains") + DispatchQueue.global().async { + while RunnerAXSnapshotBridge.customActionReadsInFlight() > 0 { + usleep(20_000) + } + recovered.fulfill() + } + wait(for: [recovered], timeout: 5) + + completed = ObjCBool(false) + XCTAssertNil( + RunnerAXSnapshotBridge.customActionNames( + forElement: element, axClient: hung, completed: &completed)) + // Completed (the fake answers nil actions), which is the point: the pass is + // live again rather than latched off. + XCTAssertTrue(completed.boolValue) + XCTAssertEqual( + RunnerAXSnapshotBridge.customActionReadDispatchCount(), dispatchesBefore + 2) + } + + /// The element budget bounds how many elements we read; these caps bound what + /// any ONE element can put in the response. Clipping must be reported, since + /// a clipped list looks exactly like a complete one. + func testActionNamesAreCappedPerElementAndReported() { + var truncated = ObjCBool(true) + + // Under both caps: untouched, nothing to report. + let small = ["Reply", "Repost"] + XCTAssertEqual( + RunnerAXSnapshotBridge.cappedActionNames(small, truncated: &truncated), small) + XCTAssertFalse(truncated.boolValue) + + // More actions than the per-element cap: clipped to the first 8, reported. + let many = (1...20).map { "Action \($0)" } + let cappedMany = RunnerAXSnapshotBridge.cappedActionNames(many, truncated: &truncated) + XCTAssertEqual(cappedMany.count, 8) + XCTAssertEqual(cappedMany.first, "Action 1") + XCTAssertTrue(truncated.boolValue) + + // A single very long name is shortened, reported, and stays one string. + let long = String(repeating: "a", count: 500) + let cappedLong = RunnerAXSnapshotBridge.cappedActionNames([long], truncated: &truncated) + XCTAssertEqual(cappedLong.count, 1) + XCTAssertTrue(truncated.boolValue) + XCTAssertLessThan(cappedLong[0].count, long.count) + XCTAssertTrue(cappedLong[0].hasSuffix("…")) + + // Empty input is not "truncated". + XCTAssertEqual(RunnerAXSnapshotBridge.cappedActionNames([], truncated: &truncated), []) + XCTAssertFalse(truncated.boolValue) + } + + /// Action names annotated by the bridge must survive into the emitted node — + /// the whole point of the capture is that the merged card names its hidden + /// affordances. + func testPrivateAXNodesCarryAnnotatedCustomActions() { + let tree: [String: Any] = [ + "type": Int(XCUIElement.ElementType.application.rawValue), + "label": "Blue Sky", + "frame": ["x": 0, "y": 0, "width": 390, "height": 844], + "children": [ + [ + "type": Int(XCUIElement.ElementType.link.rawValue), + "label": "feedItem-by-whiskers.test", + "frame": ["x": 0, "y": 100, "width": 390, "height": 200], + "actions": ["Reply", "Repost", "Open post options menu"], + "children": [], + ], + [ + "type": Int(XCUIElement.ElementType.button.rawValue), + "label": "Compose", + "frame": ["x": 300, "y": 700, "width": 60, "height": 60], + "children": [], + ], + ], + ] + let nodes = privateAXAcquisition( + rawRoot: tree, + hint: CaptureHint( + projection: .regular, depth: nil, regularPresentedDepth: nil, + interactiveOnly: false, customActions: false) + ) + + let card = nodes.first { $0.label == "feedItem-by-whiskers.test" } + XCTAssertEqual(card?.actions, ["Reply", "Repost", "Open post options menu"]) + // A node the bridge did not annotate stays absent, not empty. + XCTAssertNil(nodes.first { $0.label == "Compose" }?.actions) + } + + func testPrivateAXAcquisitionDoesNotInterpretScope() { + let tree: [String: Any] = [ + "type": 1, "label": "App", + "children": [ + [ + "type": 9, "identifier": "homeScreen", + "children": [ + ["type": 48, "label": "Post body without the scope text", "children": []] + ], + ], + ["type": 9, "label": "unrelated sibling", "children": []], + ], + ] + // Scope never reaches acquisition: the hint derived for a scoped request carries no scope, + // and the backend has no way to interpret one. + let nodes = privateAXAcquisition( + rawRoot: tree, + hint: SnapshotPresentation.captureHint( + for: PresentationOptions( + interactiveOnly: false, + depth: nil, + scope: "homeScreen", + raw: false + ) + ) + ) + + let labels = nodes.compactMap { $0.label ?? $0.identifier } + XCTAssertTrue(labels.contains("homeScreen")) + // Descendants of the matched scope are included even when they do not contain the text. + XCTAssertTrue(labels.contains("Post body without the scope text")) + XCTAssertTrue(labels.contains("unrelated sibling")) + } + + func testPrivateAXInteractiveFiltersLoginLikeHiddenDrawer() throws { + let tree: [String: Any] = [ + "type": Int(XCUIElement.ElementType.application.rawValue), + "label": "Blue Sky", + "frame": ["x": 0, "y": 0, "width": 390, "height": 844], + "children": [ + [ + "type": Int(XCUIElement.ElementType.scrollView.rawValue), + "frame": ["x": 0, "y": 0, "width": 390, "height": 844], + "children": [ + [ + "type": Int(XCUIElement.ElementType.image.rawValue), + "label": "Callstack", + "frame": ["x": 145, "y": 104, "width": 100, "height": 100], + "children": [], + ], + [ + "type": Int(XCUIElement.ElementType.staticText.rawValue), + "label": "Welcome back", + "frame": ["x": 32, "y": 260, "width": 326, "height": 32], + "children": [], + ], + [ + "type": Int(XCUIElement.ElementType.textField.rawValue), + "label": "Email", + "identifier": "login.email", + "frame": ["x": 32, "y": 348, "width": 326, "height": 48], + "children": [], + ], + [ + "type": Int(XCUIElement.ElementType.secureTextField.rawValue), + "label": "Password", + "identifier": "login.password", + "frame": ["x": 32, "y": 412, "width": 326, "height": 48], + "children": [], + ], + [ + "type": Int(XCUIElement.ElementType.button.rawValue), + "label": "Sign in", + "identifier": "login.submit", + "frame": ["x": 32, "y": 492, "width": 326, "height": 52], + "children": [], + ], + [ + "type": Int(XCUIElement.ElementType.link.rawValue), + "label": "Forgot password?", + "frame": ["x": 128, "y": 568, "width": 134, "height": 32], + "children": [], + ], + [ + "type": Int(XCUIElement.ElementType.button.rawValue), + "label": "Admin settings", + "frame": ["x": -260, "y": 184, "width": 220, "height": 44], + "children": [], + ], + [ + "type": Int(XCUIElement.ElementType.other.rawValue), + "frame": ["x": 16, "y": 184, "width": 220, "height": 44], + "children": [], + ], + ], + ] + ], + ] + let viewport = CGRect(x: 0, y: 0, width: 390, height: 844) + let hint = CaptureHint( + projection: .regular, depth: nil, regularPresentedDepth: nil, + interactiveOnly: true, customActions: false) + let acquired = SnapshotGeometrySpace.normalized( + nodes: privateAXAcquisition(rawRoot: tree, hint: hint), + viewport: viewport, + interfaceOrientation: RunnerInterfaceOrientation.portrait + ) + // Acquisition serializes the drawer too; the shared fold is what hides it (#1797). + XCTAssertTrue(acquired.compactMap(\.label).contains("Admin settings")) + + let capture = try SnapshotPresentation.presentRegular( + SnapshotAcquisition( + hint: hint, nodes: acquired, truncated: false, effectiveDepth: nil, viewport: viewport), + options: PresentationOptions(interactiveOnly: true, depth: nil, scope: nil, raw: false), + policy: .cursorProjected + ) + let labels = capture.nodes.compactMap { $0.label } + XCTAssertEqual( + labels, + ["Blue Sky", "Callstack", "Welcome back", "Email", "Password", "Sign in", "Forgot password?"] + ) + XCTAssertFalse(labels.contains("Admin settings")) + } +} + +/// Stands in for an AX client whose `attributesForElement:` never returns — +/// the wedged-server case the containment exists for. `release()` lets the +/// hung call finish so recovery is observable. +private final class HungAXClientForTesting: NSObject { + private let gate = DispatchSemaphore(value: 0) + private let releasedOnce = NSLock() + private var released = false + + @objc(attributesForElement:attributes:error:) + func attributes(forElement element: Any, attributes: Any, error: NSErrorPointer) -> Any? { + releasedOnce.lock() + let alreadyReleased = released + releasedOnce.unlock() + // Once the wedge clears, the server answers normally again — that is what + // makes the recovery leg a recovery rather than a second hang. + if alreadyReleased { + return nil + } + gate.wait() + return nil + } + + func release() { + releasedOnce.lock() + defer { releasedOnce.unlock() } + guard !released else { return } + released = true + gate.signal() + } +} + +/// Minimal snapshot stand-in whose accessibilityElement resolves (so the +/// extension proceeds to the request) while the paired fake client cannot +/// serve it — the failed-re-root miss path. +private final class FrontierSnapshotWithElementForTesting: NSObject { + @objc let accessibilityElement = NSObject() +} + +/// The vanished-element case: KVC resolves the property and gets nil. +private final class FrontierSnapshotWithoutElementForTesting: NSObject { + @objc let accessibilityElement: NSObject? = nil +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertTests.swift new file mode 100644 index 0000000000..1beb0bec9d --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertTests.swift @@ -0,0 +1,9 @@ +import XCTest + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +extension RunnerTests { + func testAlertAcceptTreatsOpenAsAffirmative() { + XCTAssertTrue(isAcceptButton("Open")) + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+BlockingSystemModalResolutionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+BlockingSystemModalResolutionTests.swift new file mode 100644 index 0000000000..8a124bb669 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+BlockingSystemModalResolutionTests.swift @@ -0,0 +1,37 @@ +import XCTest + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +extension RunnerTests { + func testRemoteHostProbeRunsOnlyWhenSpringboardModalHasNoActions() { + XCTAssertTrue(RemoteHostedSystemModalPolicy.shouldProbeRemoteHost(springboardActionCount: 0)) + XCTAssertFalse(RemoteHostedSystemModalPolicy.shouldProbeRemoteHost(springboardActionCount: 1)) + XCTAssertFalse(RemoteHostedSystemModalPolicy.shouldProbeRemoteHost(springboardActionCount: 3)) + } + + func testRemoteHostStateGateFailsClosedToForeground() { + XCTAssertTrue(RemoteHostedSystemModalPolicy.isEligibleHostState(.runningForeground)) + XCTAssertFalse(RemoteHostedSystemModalPolicy.isEligibleHostState(.runningBackground)) + XCTAssertFalse(RemoteHostedSystemModalPolicy.isEligibleHostState(.notRunning)) + XCTAssertFalse(RemoteHostedSystemModalPolicy.isEligibleHostState(.unknown)) + } + + // No SpringBoard host (`hasSpringBoardSystemModalHost`) means modal resolution must return + // `.absent` without probing com.apple.springboard (#1351). Written for tvOS, where no lane + // ever executed it; `resolveBlockingSystemModal` takes that decision at RUNTIME off the same + // flag on macOS, so the host lane runs the real branch on every PR. + // + // Its former sibling `testBlockingSystemAlertSnapshotIsNilOnTvOS` is deleted rather than + // widened: `blockingSystemAlertSnapshot` is `#if os(macOS) return nil`, so on the only lane + // that could run it the assertion would pin a compile-time literal — a green that no change + // to the runner could turn red. The runtime gate it meant to cover is this test's subject, + // and the nil it returns on macOS is the compiler's business, not a test's. + #if os(tvOS) || os(macOS) + func testResolveBlockingSystemModalIsAbsentWithoutSpringBoardHost() { + guard case .absent = resolveBlockingSystemModal(deadline: .distantFuture) else { + XCTFail("blocking system-modal resolution must be .absent without a SpringBoard host") + return + } + } + #endif +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandExecutionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandExecutionTests.swift new file mode 100644 index 0000000000..b763b9e07e --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandExecutionTests.swift @@ -0,0 +1,704 @@ +import XCTest +import AgentDeviceSnapshotPresentation + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS) +import ObjectiveC.runtime + +private final class RunnerSynthesizedSwipeFailureStub: NSObject { + @objc(synthesizeSwipeWithApplication:resolvedWindow:x:y:x2:y2:durationMs:) + class func synthesizeSwipe( + application: XCUIApplication, + resolvedWindow: Any?, + x: Double, + y: Double, + x2: Double, + y2: Double, + durationMs: Double + ) -> String? { + "forced private synthesis failure" + } +} + +private final class RunnerSynthesizedTapFailureStub: NSObject { + @objc(synthesizeTapWithApplication:resolvedWindow:x:y:) + class func synthesizeTap(application: XCUIApplication, resolvedWindow: Any?, x: Double, y: Double) -> String? { + "forced private synthesis failure" + } +} +#endif + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +extension RunnerTests { + func testGestureResponseIncludesSynthesizedTapFallbackDiagnostics() { + let response = gestureResponse( + message: "tapped", + timing: (gestureStartUptimeMs: 1, gestureEndUptimeMs: 2), + fallback: GestureFallback( + strategy: "xctest-coordinate-tap", + message: "Runner synthesized coordinate tap is unavailable", + hint: "Using XCTest coordinate tap fallback." + ) + ) + + XCTAssertEqual(response.ok, true) + XCTAssertEqual(response.data?.gestureFallback, "xctest-coordinate-tap") + XCTAssertEqual( + response.data?.gestureFallbackMessage, + "Runner synthesized coordinate tap is unavailable" + ) + XCTAssertEqual(response.data?.gestureFallbackHint, "Using XCTest coordinate tap fallback.") + } + + func testGestureResponseIncludesMaestroNonHittableFallbackUsage() { + let response = gestureResponse( + message: "tapped via non-hittable coordinate fallback", + timing: (gestureStartUptimeMs: 1, gestureEndUptimeMs: 2), + frame: .touch(nil), + maestroNonHittableCoordinateFallbackUsed: true + ) + + XCTAssertEqual(response.data?.maestroNonHittableCoordinateFallbackUsed, true) + } + + func testCanonicalPlannedGestureResponseOmitsDragFrameAndPreservesDiagnostics() { + let response = gestureResponse( + message: "fling", + timing: (gestureStartUptimeMs: 1, gestureEndUptimeMs: 2), + frame: .drag( + DragVisualizationFrame( + x: 160, + y: 150, + x2: 40, + y2: 150, + referenceWidth: 200, + referenceHeight: 300 + ) + ), + fallback: GestureFallback( + strategy: "xctest-coordinate-drag", + message: "Private synthesis unavailable", + hint: "Using XCTest coordinate fallback." + ) + ) + + let canonical = canonicalPlannedGestureResponse(response) + + XCTAssertEqual(canonical.data?.gestureStartUptimeMs, 1) + XCTAssertEqual(canonical.data?.gestureEndUptimeMs, 2) + XCTAssertEqual(canonical.data?.gestureFallback, "xctest-coordinate-drag") + XCTAssertEqual(canonical.data?.gestureFallbackMessage, "Private synthesis unavailable") + XCTAssertEqual(canonical.data?.gestureFallbackHint, "Using XCTest coordinate fallback.") + XCTAssertNil(canonical.data?.x) + XCTAssertNil(canonical.data?.y) + XCTAssertNil(canonical.data?.x2) + XCTAssertNil(canonical.data?.y2) + XCTAssertNil(canonical.data?.referenceWidth) + XCTAssertNil(canonical.data?.referenceHeight) + } + +#if os(iOS) + func testSinglePointerFlingFallsBackToXCTestCoordinateDragWhenPrivateSynthesisFails() throws { + let selector = NSSelectorFromString( + "synthesizeSwipeWithApplication:resolvedWindow:x:y:x2:y2:durationMs:" + ) + guard + let synthesizedSwipeMethod = class_getClassMethod(RunnerSynthesizedGesture.self, selector), + let failureStubMethod = class_getClassMethod(RunnerSynthesizedSwipeFailureStub.self, selector) + else { + XCTFail("unable to install synthesized swipe failure stub") + return + } + let originalImplementation = method_getImplementation(synthesizedSwipeMethod) + method_setImplementation( + synthesizedSwipeMethod, + method_getImplementation(failureStubMethod) + ) + app.launch() + runnerAccessibilityHealth = .healthy + defer { + method_setImplementation(synthesizedSwipeMethod, originalImplementation) + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } + let command = try runnerCommandFixture( + """ + {"command":"gesture","commandId":"gesture-fling-fallback","gesturePlan":{"topology":"single","intent":"fling","executionProfile":"endpoint-hold","durationMs":100,"viewport":{"x":0,"y":0,"width":200,"height":300},"pointers":[{"pointerId":0,"samples":[{"offsetMs":0,"point":{"x":160,"y":150}},{"offsetMs":100,"point":{"x":40,"y":150}}]}]}} + """ + ) + + let response = try executeOnMainPrepared(command: command, activeApp: app) + + XCTAssertTrue(response.ok) + XCTAssertEqual(response.data?.message, "fling") + XCTAssertEqual(response.data?.gestureFallback, "xctest-coordinate-drag") + XCTAssertEqual(response.data?.gestureFallbackMessage, "forced private synthesis failure") + XCTAssertEqual( + response.data?.gestureFallbackHint, + "Private XCTest event synthesis is required for AX-free coordinate drag on iOS; update Xcode if this persists." + ) + XCTAssertNil(response.data?.x) + XCTAssertNil(response.data?.y) + XCTAssertNil(response.data?.x2) + XCTAssertNil(response.data?.y2) + } + + func testSelectorTapFallsBackToXCTestCoordinateWhenPrivateSynthesisFails() throws { + let selector = NSSelectorFromString("synthesizeTapWithApplication:resolvedWindow:x:y:") + guard + let synthesizedTapMethod = class_getClassMethod(RunnerSynthesizedGesture.self, selector), + let failureStubMethod = class_getClassMethod(RunnerSynthesizedTapFailureStub.self, selector) + else { + XCTFail("unable to install synthesized tap failure stub") + return + } + let originalImplementation = method_getImplementation(synthesizedTapMethod) + method_setImplementation( + synthesizedTapMethod, + method_getImplementation(failureStubMethod) + ) + app.launch() + currentApp = app + runnerAccessibilityHealth = .healthy + defer { + method_setImplementation(synthesizedTapMethod, originalImplementation) + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } + let command = try runnerCommandFixture( + #"{"command":"tap","commandId":"selector-tap-fallback","selectorKey":"label","selectorValue":"Agent Device Runner","synthesized":true}"# + ) + + let response = try executeOnMainPrepared(command: command, activeApp: app) + + XCTAssertTrue(response.ok) + XCTAssertEqual(response.data?.message, "tapped") + XCTAssertEqual(response.data?.gestureFallback, "xctest-coordinate-tap") + XCTAssertEqual(response.data?.gestureFallbackMessage, "forced private synthesis failure") + XCTAssertEqual( + response.data?.gestureFallbackHint, + "Falling back to XCTest coordinate tap may be slower and can still need a healthy accessibility tree." + ) + } +#endif + +#if os(iOS) + // `waitForTextEntryReadiness`'s hardware-keyboard fallback returns early only on confirmed + // focus (#1874), and `keyboardFocusConfirmed` reads that from the app-wide focus predicate this + // bundle otherwise refuses to trust. Two XCTest facts it rests on, neither a repository + // invariant: the predicate reports a responder that shows NO software keyboard at all, and it + // names the element well enough to tell the tapped field from another one. The fixture field is + // the exact shape the fallback exists for — a real responder with an empty `inputView` — so this + // is where both are observable. If either regressed, readiness would silently stop taking the + // fallback and spend the full readinessTimeout on every hardware-keyboard field, which no other + // assertion would notice. + func testHardwareKeyboardResponderConfirmsItsOwnKeyboardFocus() throws { + app.launchArguments = ["--agent-device-text-entry-regression"] + app.launch() + defer { + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } + XCTAssertTrue(app.waitForExistence(timeout: appExistenceTimeout)) + + let textField = app.textFields["agent-device-hardware-keyboard-input"] + XCTAssertTrue(textField.waitForExistence(timeout: appExistenceTimeout)) + let otherElement = app.staticTexts["Agent Device Runner"] + XCTAssertTrue(otherElement.waitForExistence(timeout: appExistenceTimeout)) + XCTAssertFalse( + keyboardFocusConfirmed(app: app, element: textField), + "an untapped field must not confirm focus, or the fallback would fire immediately" + ) + + let tapCommand = try runnerCommandFixture( + #"{"command":"tap","commandId":"tap-focus-confirmation","selectorKey":"id","selectorValue":"agent-device-hardware-keyboard-input"}"# + ) + let tapResponse = try executeOnMainPrepared(command: tapCommand, activeApp: app) + XCTAssertTrue(tapResponse.ok, String(describing: tapResponse.error)) + try XCTSkipIf( + isKeyboardVisible(app: app), + "software keyboard is up: this simulator cannot exercise the hidden-keyboard responder path" + ) + + let deadline = Date().addingTimeInterval(TextEntryTiming.readinessTimeout) + var confirmed = keyboardFocusConfirmed(app: app, element: textField) + while !confirmed && Date() < deadline { + sleepFor(TextEntryTiming.pollInterval) + confirmed = keyboardFocusConfirmed(app: app, element: textField) + } + XCTAssertTrue(confirmed, "a tapped responder must confirm its own keyboard focus") + XCTAssertFalse( + keyboardFocusConfirmed(app: app, element: otherElement), + "focus held by another element must read as a refusal, never as this element's focus" + ) + } +#endif + + func testInjectedTapRecordedFailureGateIsTapOnlyAndCountGated() { + // The seam's recording side cannot run in-bundle (a real XCTIssue would + // fail this very test run — same constraint the record(_:) suppression + // tests document); the live daemon proof covers it. This pins the gate. + XCTAssertFalse(RunnerTests.shouldInjectTapRecordedFailure(command: .tap, remaining: 0)) + XCTAssertTrue(RunnerTests.shouldInjectTapRecordedFailure(command: .tap, remaining: 1)) + XCTAssertFalse(RunnerTests.shouldInjectTapRecordedFailure(command: .type, remaining: 1)) + XCTAssertFalse(RunnerTests.shouldInjectTapRecordedFailure(command: .snapshot, remaining: 1)) + } + + func testXCTestRecordedFailureResponseFailsMutatingSuccesses() throws { + let command = try runnerCommandFixture(#"{"command":"tap","commandId":"tap-1"}"#) + let response = Response(ok: true, data: DataPayload(message: "tapped")) + + let failureResponse = xctestRecordedFailureResponse(command: command, response: response) + + XCTAssertEqual(failureResponse?.ok, false) + XCTAssertEqual(failureResponse?.error?.code, "XCTEST_RECORDED_FAILURE") + XCTAssertEqual( + failureResponse?.error?.message, + "XCTest recorded a failure while executing tap; the action may not have been performed." + ) + } + + func testXCTestRecordedFailureResponseFailsActionButtonSuccess() throws { + // The Action Button press carries no settle and no post-action observation, so this conversion is + // the only evidence the press landed. That is why the press is not classified runner-lifecycle: + // `isLifecycle` would silence the conversion here (#2699, #2702 review). + let command = try runnerCommandFixture(#"{"command":"actionButton","commandId":"action-button-1"}"#) + let response = Response(ok: true, data: DataPayload(message: "actionButton")) + + let failureResponse = xctestRecordedFailureResponse(command: command, response: response) + + XCTAssertEqual(failureResponse?.ok, false) + XCTAssertEqual(failureResponse?.error?.code, "XCTEST_RECORDED_FAILURE") + XCTAssertEqual( + failureResponse?.error?.message, + "XCTest recorded a failure while executing actionButton; the action may not have been performed." + ) + } + + func testXCTestRecordedFailureResponseDoesNotWrapReadOnlyOrRunnerFatalResponses() throws { + let snapshotCommand = try runnerCommandFixture(#"{"command":"snapshot","commandId":"snapshot-1"}"#) + let tapCommand = try runnerCommandFixture(#"{"command":"tap","commandId":"tap-1"}"#) + let runnerFatalResponse = Response( + ok: true, + data: DataPayload(runnerFatal: true, runnerFatalReason: "ax_snapshot_unavailable") + ) + + XCTAssertNil( + xctestRecordedFailureResponse( + command: snapshotCommand, + response: Response(ok: true, data: DataPayload(nodes: [], truncated: false)) + ) + ) + XCTAssertNil(xctestRecordedFailureResponse(command: tapCommand, response: runnerFatalResponse)) + } + + // Simulator-only from here to the matching #endif: these launch the host app, route through + // SpringBoard, or assert the iOS-only alert/system-modal branches. Tests outside the + // `os(iOS)` regions in this file are pure runner decisions and also run on the macOS host + // lane (ci.yml) — see the classification convention in RunnerTests.swift. +#if os(iOS) + func testMissingBundleCommandInvalidatesCompleteCachedTargetState() throws { + app.launch() + currentApp = app + currentBundleId = "com.example.stale-target" + currentAppProcessIdentifier = 42 + snapshotXCTestPenaltyWarmupExemptionPending = true + defer { + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } + let command = try runnerCommandFixture( + #"{"command":"snapshot","commandId":"snapshot-without-bundle"}"# + ) + + _ = prepareActiveCommandContext(command: command) + + XCTAssertNil(currentApp) + XCTAssertNil(currentBundleId) + XCTAssertNil(currentAppProcessIdentifier) + XCTAssertFalse(snapshotXCTestPenaltyWarmupExemptionPending) + } + + func testSkipAppActivationPreflightIncludesForegroundCachedCoordinateOnlyTaps() throws { + app.launch() + currentApp = app + currentBundleId = nil + defer { + currentApp = nil + currentBundleId = nil + app.terminate() + } + let tap = try runnerCommandFixture( + #"{"command":"tap","commandId":"tap-1","x":10,"y":20}"# + ) + + XCTAssertTrue(shouldSkipAppActivationPreflight(tap)) + } + + func testSkipAppActivationPreflightRejectsMissingChangedAndBackgroundTargets() throws { + let coordinateTap = try runnerCommandFixture( + #"{"command":"tap","commandId":"tap-1","x":10,"y":20}"# + ) + currentApp = nil + currentBundleId = nil + XCTAssertFalse(shouldSkipAppActivationPreflight(coordinateTap)) + + app.launch() + currentApp = app + currentBundleId = "com.example.current" + defer { + currentApp = nil + currentBundleId = nil + app.terminate() + } + let changedBundleTap = try runnerCommandFixture( + #"{"command":"tap","commandId":"tap-2","appBundleId":"com.example.other","x":10,"y":20}"# + ) + + XCTAssertFalse(shouldSkipAppActivationPreflight(changedBundleTap)) + + app.terminate() + currentApp = app + currentBundleId = nil + + XCTAssertFalse(shouldSkipAppActivationPreflight(coordinateTap)) + } + + func testActionButtonPressSkipsAppActivationPreflightWithoutBeingRunnerLifecycle() throws { + currentApp = nil + currentBundleId = nil + let press = try runnerCommandFixture(#"{"command":"actionButton","commandId":"action-button-1"}"#) + + // The skip is its own decision, reached without the lifecycle flag that would also drop the + // recorded-failure conversion; no cached target and no foreground app is required for it. + XCTAssertFalse(isRunnerLifecycleCommand(.actionButton)) + XCTAssertTrue(shouldSkipAppActivationPreflight(press)) + } + + func testPrepareActiveCommandContextRoutesBlockingSystemModalToSpringboard() throws { + blockingSystemModalPresenceOverrideForTesting = true + currentApp = nil + currentBundleId = nil + defer { + blockingSystemModalPresenceOverrideForTesting = nil + currentApp = nil + currentBundleId = nil + } + let tap = try runnerCommandFixture( + #"{"command":"tap","commandId":"tap-1","x":10,"y":20}"# + ) + + let preparation = prepareActiveCommandContext( + command: tap, + routeToSpringboard: shouldRouteToSpringboardBlockingSystemModal(tap) + ) + + guard case .context(let context) = preparation else { + XCTFail("expected command context") + return + } + XCTAssertTrue(context.app === springboard) + } + + func testExecuteDispatchedReturnsBusyBeforeBlockingSystemModalProbeDrains() throws { + app.launch() + currentApp = app + currentBundleId = nil + defer { + currentApp = nil + currentBundleId = nil + systemModalProbeOverrideForTesting = nil + clearSnapshotXCTestChannelPenalty(reason: "test-cleanup") + app.terminate() + } + + final class ResultBox { + var response: Response? + var error: Error? + var commandRecoveredBeforeRelease = false + var wasBusyBeforeRelease = false + var hadAbandonedProbeBeforeRelease = false + var drained = false + } + let box = ResultBox() + let probeStarted = expectation(description: "system-modal routing probe started") + let verificationFinished = expectation(description: "command recovery and modal probe drain verified") + let probeReleaseGate = DispatchSemaphore(value: 0) + let commandFinishedGate = DispatchSemaphore(value: 0) + systemModalProbeOverrideForTesting = { _ in + probeStarted.fulfill() + _ = probeReleaseGate.wait(timeout: .now() + 15) + return DataPayload(message: "late system modal") + } + + let command = try runnerCommandFixture( + #"{"command":"tap","commandId":"bounded-modal-routing","x":10,"y":20}"# + ) + DispatchQueue(label: "agent-device.runner.tests.modal-routing-probe").async { + do { + box.response = try self.executeDispatched(command: command) + } catch { + box.error = error + } + commandFinishedGate.signal() + } + DispatchQueue(label: "agent-device.runner.tests.modal-routing-probe-verifier").async { + let commandWait = commandFinishedGate.wait( + timeout: .now() + self.systemModalProbeBudget + 3 + ) + box.commandRecoveredBeforeRelease = commandWait == .success + && box.error == nil + && box.response?.error?.code == "RUNNER_BUSY" + if case .busy = self.currentMainThreadBusyState() { + box.wasBusyBeforeRelease = true + } + box.hadAbandonedProbeBeforeRelease = self.hasAbandonedMainThreadWork() + + // The XCTest main thread is blocked inside the injected probe, so this verifier owns the + // ordered release after recording the command result and abandoned-work state above. + probeReleaseGate.signal() + let deadline = Date().addingTimeInterval(5) + while self.hasAbandonedMainThreadWork(), Date() < deadline { + self.sleepFor(0.002) + } + box.drained = !self.hasAbandonedMainThreadWork() + verificationFinished.fulfill() + } + + wait(for: [probeStarted, verificationFinished], timeout: 15) + XCTAssertTrue( + box.commandRecoveredBeforeRelease, + "the public coordinate tap must return RUNNER_BUSY before the blocked modal probe drains" + ) + XCTAssertTrue(box.wasBusyBeforeRelease) + XCTAssertTrue(box.hadAbandonedProbeBeforeRelease) + XCTAssertTrue(box.drained) + guard case .idle = currentMainThreadBusyState() else { + return XCTFail("expected the runner to become idle after the routing probe drained") + } + XCTAssertFalse(hasAbandonedMainThreadWork()) + } + + func testSkipAppActivationPreflightRejectsSelectorAndMixedSequenceGestures() throws { + app.launch() + currentApp = app + currentBundleId = nil + defer { + currentApp = nil + currentBundleId = nil + app.terminate() + } + let selectorTap = try runnerCommandFixture( + #"{"command":"tap","commandId":"tap-1","selectorKey":"label","selectorValue":"Search","synthesized":true}"# + ) + let standardDrag = try runnerCommandFixture( + #"{"command":"drag","commandId":"drag-1","x":10,"y":20,"x2":30,"y2":40}"# + ) + let mixedSequence = try runnerCommandFixture( + """ + {"command":"sequence","commandId":"seq-1","steps":[ + {"kind":"tap","x":10,"y":20,"synthesized":true}, + {"kind":"doubleTap","x":30,"y":40} + ]} + """ + ) + + XCTAssertFalse(shouldSkipAppActivationPreflight(selectorTap)) + XCTAssertFalse(shouldSkipAppActivationPreflight(standardDrag)) + XCTAssertFalse(shouldSkipAppActivationPreflight(mixedSequence)) + } + + // Launches nothing, but still simulator-only: `shouldSkipAppActivationPreflight` is + // `#if os(iOS) …guards… #else return false #endif`, so on macOS this asserts a compile-time + // literal and no edit to the iOS body could make it red. Its five siblings above and below + // are gated for the same reason. + func testSkipAppActivationPreflightRequiresCachedForegroundTarget() throws { + currentApp = nil + currentBundleId = nil + let scroll = try runnerCommandFixture( + #"{"command":"scroll","commandId":"scroll-1","direction":"down","pixels":400}"# + ) + + XCTAssertFalse(shouldSkipAppActivationPreflight(scroll)) + } + + func testSkipAppActivationPreflightKeepsDragScrollAndSequenceOnForegroundGuard() throws { + app.launch() + currentApp = app + currentBundleId = nil + defer { + currentApp = nil + currentBundleId = nil + app.terminate() + } + let drag = try runnerCommandFixture( + #"{"command":"drag","commandId":"drag-1","x":10,"y":20,"x2":30,"y2":40}"# + ) + let scroll = try runnerCommandFixture( + #"{"command":"scroll","commandId":"scroll-1","direction":"down","pixels":400}"# + ) + let sequence = try runnerCommandFixture( + """ + {"command":"sequence","commandId":"seq-1","steps":[ + {"kind":"tap","x":10,"y":20,"synthesized":true}, + {"kind":"longPress","x":10,"y":200,"durationMs":300} + ]} + """ + ) + + XCTAssertFalse(shouldSkipAppActivationPreflight(drag)) + XCTAssertFalse(shouldSkipAppActivationPreflight(scroll)) + XCTAssertFalse(shouldSkipAppActivationPreflight(sequence)) + } + + func testSkipAppActivationPreflightIncludesAlertCommands() throws { + let alert = try runnerCommandFixture( + #"{"command":"alert","commandId":"alert-1","action":"get"}"# + ) + + XCTAssertTrue(shouldSkipAppActivationPreflight(alert)) + } +#endif + + func testDispatchReturnsBusyBeforeQueueingMainThreadWork() throws { + let command = try runnerCommandFixture(#"{"command":"snapshot","commandId":"snapshot-busy"}"#) + abandonedMainThreadWorkCount = 1 + abandonedMainThreadWorkSince = Date(timeIntervalSinceNow: -2) + defer { + abandonedMainThreadWorkCount = 0 + abandonedMainThreadWorkSince = nil + } + + let response = try execute(command: command) + + XCTAssertFalse(response.ok) + XCTAssertEqual(response.error?.code, "RUNNER_BUSY") + XCTAssertTrue(response.error?.message.contains("previous command") == true) + } + + func testDispatchReturnsWedgedBeforeQueueingMainThreadWork() throws { + let command = try runnerCommandFixture(#"{"command":"snapshot","commandId":"snapshot-wedged"}"#) + abandonedMainThreadWorkCount = 1 + abandonedMainThreadWorkSince = Date(timeIntervalSinceNow: -(mainThreadWedgeThreshold + 1)) + defer { + abandonedMainThreadWorkCount = 0 + abandonedMainThreadWorkSince = nil + } + + let response = try execute(command: command) + + XCTAssertFalse(response.ok) + XCTAssertEqual(response.error?.code, "RUNNER_WEDGED") + XCTAssertTrue(response.error?.hint?.contains("runner session will be restarted") == true) + } + + func testPostSnapshotDelayMarkDoesNotQueueBehindAbandonedMainThreadWork() { + abandonedMainThreadWorkCount = 1 + defer { + abandonedMainThreadWorkCount = 0 + needsPostSnapshotInteractionDelay = false + } + + let finished = expectation(description: "off-main caller finished") + DispatchQueue(label: "agent-device.runner.tests.post-snapshot-delay").async { + self.setNeedsPostSnapshotInteractionDelay() + finished.fulfill() + } + + wait(for: [finished], timeout: 1) + mainThreadWorkLock.lock() + let abandonedWorkCount = abandonedMainThreadWorkCount + mainThreadWorkLock.unlock() + XCTAssertEqual(abandonedWorkCount, 1, "the skipped mark must not add an abandoned unit") + XCTAssertFalse(needsPostSnapshotInteractionDelay) + } + + func testSnapshotFailureInvalidationQueuesBehindAbandonedMainThreadWorkWithoutWaiting() { + currentBundleId = "com.example.stale-target" + defer { currentBundleId = nil } + + final class ResultBox { + var elapsed: TimeInterval? + var bundleStillCachedWhileBlocked: Bool? + var abandonedWhileBlocked: Int? + } + let box = ResultBox() + let mainBlocked = DispatchSemaphore(value: 0) + let releaseMain = DispatchSemaphore(value: 0) + let finished = expectation(description: "invalidation returned while main was blocked") + + DispatchQueue(label: "agent-device.runner.tests.snapshot-invalidation").async { + _ = try? self.runMainThreadWork( + "command_execution", + timeout: 0, + timeoutError: self.mainThreadExecutionTimeoutError + ) { + mainBlocked.signal() + _ = releaseMain.wait(timeout: .now() + 5) + return true + } + _ = mainBlocked.wait(timeout: .now() + 2) + let startedAt = Date() + self.invalidateCachedTargetAfterSnapshotFailure() + box.elapsed = Date().timeIntervalSince(startedAt) + box.bundleStillCachedWhileBlocked = self.currentBundleId != nil + self.mainThreadWorkLock.lock() + box.abandonedWhileBlocked = self.abandonedMainThreadWorkCount + self.mainThreadWorkLock.unlock() + releaseMain.signal() + finished.fulfill() + } + + wait(for: [finished], timeout: 8) + let drainDeadline = Date().addingTimeInterval(2) + while hasAbandonedMainThreadWork() || currentBundleId != nil, Date() < drainDeadline { + sleepFor(0.005) + } + + XCTAssertLessThan( + box.elapsed ?? .infinity, + 0.5, + "the failed capture must not wait behind abandoned main-thread work" + ) + XCTAssertEqual( + box.bundleStillCachedWhileBlocked, + true, + "the drop must queue behind the blocked main thread, not run early" + ) + XCTAssertEqual(box.abandonedWhileBlocked, 1, "the deferred drop must not add an abandoned unit") + XCTAssertFalse(hasAbandonedMainThreadWork()) + XCTAssertNil(currentBundleId, "the drop must run once the main thread frees") + } + + /// Routes `command` through the transport's inline and queued paths. The calling test's main + /// thread serves the command's main-thread work while it waits. + func execute(command: Command) throws -> Response { + dispatchPrecondition(condition: .onQueue(.main)) + if let response = inlineResponse(for: command) { + return response + } + final class ResultBox { + var result: Result? + } + let box = ResultBox() + let executed = XCTestExpectation(description: "\(command.command.rawValue) executed off main") + enqueueAccepted(command: command) { result in + box.result = result + executed.fulfill() + } + guard XCTWaiter.wait(for: [executed], timeout: mainThreadExecutionTimeout + 5) == .completed, + let result = box.result + else { + throw NSError( + domain: RunnerErrorDomain.general, + code: RunnerErrorCode.commandReturnedNoResponse, + userInfo: [NSLocalizedDescriptionKey: "command did not finish on the command queue"] + ) + } + return try result.get() + } + + func runnerCommandFixture(_ json: String) throws -> Command { + try JSONDecoder().decode(Command.self, from: Data(json.utf8)) + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandJournalTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandJournalTests.swift new file mode 100644 index 0000000000..44555b0b76 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandJournalTests.swift @@ -0,0 +1,340 @@ +import Foundation +import XCTest +import AgentDeviceSnapshotPresentation + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +extension RunnerTests { + func testUptimeBypassesCommandJournal() throws { + let command = runnerJournalCommand("uptime", id: "uptime-probe") + + let response = try execute(command: command) + let status = commandJournal.status(normalizedCommandId: "uptime-probe") + + XCTAssertEqual(response.ok, true) + XCTAssertNotNil(response.data?.currentUptimeMs) + XCTAssertEqual(status.lifecycleState, RunnerCommandLifecycleState.notAccepted.rawValue) + } + + func testStampingCurrentUptimePreservesPayload() { + let stamped = Response(ok: true, data: DataPayload(message: "recording started")) + .stampingCurrentUptimeMs(123.5) + + XCTAssertEqual(stamped.ok, true) + XCTAssertEqual(stamped.data?.message, "recording started") + XCTAssertEqual(stamped.data?.currentUptimeMs, 123.5) + } + + func testStampingCurrentUptimeCreatesPayloadWhenNil() { + let stamped = Response(ok: true).stampingCurrentUptimeMs(456.0) + + XCTAssertEqual(stamped.ok, true) + XCTAssertEqual(stamped.data?.currentUptimeMs, 456.0) + } + + func testStampingCurrentUptimeSkipsErrorResponses() { + let response = Response(ok: false, error: ErrorPayload(message: "boom")) + let stamped = response.stampingCurrentUptimeMs(789.0) + + XCTAssertEqual(stamped.ok, false) + XCTAssertNil(stamped.data) + XCTAssertEqual(stamped.error?.message, "boom") + } + + func testStampingCurrentMainThreadBusyPreservesPayload() { + let stamped = Response(ok: true, data: DataPayload(nodes: [], truncated: false)) + .stampingCurrentMainThreadBusy(true) + + XCTAssertEqual(stamped.ok, true) + XCTAssertEqual(stamped.data?.runnerMainThreadBusy, true) + } + + func testStampingCurrentMainThreadBusySkipsErrorResponses() { + let response = Response(ok: false, error: ErrorPayload(code: "RUNNER_BUSY", message: "busy")) + let stamped = response.stampingCurrentMainThreadBusy(true) + + XCTAssertEqual(stamped.ok, false) + XCTAssertNil(stamped.data) + XCTAssertEqual(stamped.error?.code, "RUNNER_BUSY") + } + + func testMainThreadBusyStateReportsOccupancy() { + XCTAssertFalse(MainThreadBusyState.idle.reportsMainThreadBusy) + XCTAssertTrue(MainThreadBusyState.busy(abandonedForSeconds: 5).reportsMainThreadBusy) + XCTAssertTrue(MainThreadBusyState.wedged(abandonedForSeconds: 200).reportsMainThreadBusy) + XCTAssertEqual( + Response(ok: true).stampingCurrentMainThreadBusy(false).data?.runnerMainThreadBusy, false) + } + + func testCommandFailedResponseTagsMainThreadTimeoutWithTypedCode() { + let timeout = NSError( + domain: RunnerErrorDomain.general, + code: RunnerErrorCode.mainThreadExecutionTimedOut, + userInfo: [NSLocalizedDescriptionKey: "main thread execution timed out"] + ) + + let response = commandFailedResponse(from: timeout) + + XCTAssertEqual(response.ok, false) + XCTAssertEqual(response.error?.code, RunnerWireErrorCode.mainThreadTimeout) + } + + func testCommandFailedResponseKeepsGenericCodeForOtherErrors() { + let other = NSError(domain: "SomeOtherDomain", code: 99, userInfo: nil) + + let response = commandFailedResponse(from: other) + + XCTAssertEqual(response.error?.code, "COMMAND_FAILED") + } + + func testJournalStoredResponseStaysUnstamped() throws { + let journal = RunnerCommandJournal() + let recordStart = runnerJournalCommand("recordStart", id: "record-start-anchor") + + journal.accept(command: recordStart) + journal.finish( + command: recordStart, + response: Response(ok: true, data: DataPayload(message: "recording started")) + ) + + let status = journal.status(normalizedCommandId: "record-start-anchor") + let responseJson = try XCTUnwrap(status.lifecycleResponseJson) + XCTAssertFalse(responseJson.contains("currentUptimeMs")) + } + + func testCommandJournalRetentionPolicy() throws { + let journal = RunnerCommandJournal() + + let uptime = runnerJournalCommand("uptime", id: "small-scalar") + journal.accept(command: uptime) + journal.finish( + command: uptime, + response: Response(ok: true, data: DataPayload(currentUptimeMs: 12.5)) + ) + + let scalarStatus = journal.status(normalizedCommandId: "small-scalar") + XCTAssertEqual(scalarStatus.lifecycleState, RunnerCommandLifecycleState.completed.rawValue) + XCTAssertEqual(scalarStatus.lifecycleResponseOk, true) + XCTAssertNotNil(scalarStatus.lifecycleResponseJson) + let scalarResponse = try decodeRunnerJournalResponse(scalarStatus.lifecycleResponseJson) + XCTAssertEqual(scalarResponse.data?.currentUptimeMs, 12.5) + + let querySelector = runnerJournalCommand("querySelector", id: "small-object") + journal.accept(command: querySelector) + journal.finish( + command: querySelector, + response: Response(ok: true, data: DataPayload(found: true, nodes: [runnerJournalNode()])) + ) + + let objectStatus = journal.status(normalizedCommandId: "small-object") + XCTAssertNotNil(objectStatus.lifecycleResponseJson) + let objectResponse = try decodeRunnerJournalResponse(objectStatus.lifecycleResponseJson) + XCTAssertEqual(objectResponse.data?.found, true) + XCTAssertEqual(objectResponse.data?.nodes?.count, 1) + + let snapshot = runnerJournalCommand("snapshot", id: "snapshot-tree") + journal.accept(command: snapshot) + journal.finish( + command: snapshot, + response: Response(ok: true, data: DataPayload(nodes: [runnerJournalNode()], truncated: false)) + ) + + let snapshotStatus = journal.status(normalizedCommandId: "snapshot-tree") + XCTAssertEqual(snapshotStatus.lifecycleState, RunnerCommandLifecycleState.completed.rawValue) + XCTAssertEqual(snapshotStatus.lifecycleResponseOk, true) + XCTAssertNil(snapshotStatus.lifecycleResponseJson) + + let screenshot = runnerJournalCommand("screenshot", id: "screenshot-artifact") + journal.accept(command: screenshot) + journal.finish( + command: screenshot, + response: Response(ok: true, data: DataPayload(message: "tmp/screenshot-1.png")) + ) + + let screenshotStatus = journal.status(normalizedCommandId: "screenshot-artifact") + XCTAssertEqual(screenshotStatus.lifecycleState, RunnerCommandLifecycleState.completed.rawValue) + XCTAssertEqual(screenshotStatus.lifecycleResponseOk, true) + XCTAssertNil(screenshotStatus.lifecycleResponseJson) + + let scroll = runnerJournalCommand("scroll", id: "scroll-drag") + journal.accept(command: scroll) + journal.finish( + command: scroll, + response: Response( + ok: true, + data: DataPayload( + message: "scrolled", + gestureStartUptimeMs: 1, + gestureEndUptimeMs: 2, + x: 155, + y: 420, + x2: 155, + y2: 301, + referenceWidth: 300, + referenceHeight: 600 + ) + ) + ) + + let scrollStatus = journal.status(normalizedCommandId: "scroll-drag") + XCTAssertEqual(scrollStatus.lifecycleState, RunnerCommandLifecycleState.completed.rawValue) + XCTAssertEqual(scrollStatus.lifecycleResponseOk, true) + XCTAssertNotNil(scrollStatus.lifecycleResponseJson) + let scrollResponse = try decodeRunnerJournalResponse(scrollStatus.lifecycleResponseJson) + XCTAssertEqual(scrollResponse.data?.x, 155) + XCTAssertEqual(scrollResponse.data?.y, 420) + XCTAssertEqual(scrollResponse.data?.x2, 155) + XCTAssertEqual(scrollResponse.data?.y2, 301) + XCTAssertEqual(scrollResponse.data?.referenceWidth, 300) + XCTAssertEqual(scrollResponse.data?.referenceHeight, 600) + + let largeRead = runnerJournalCommand("readText", id: "large-read") + journal.accept(command: largeRead) + journal.finish( + command: largeRead, + response: Response(ok: true, data: DataPayload(text: String(repeating: "x", count: 17 * 1024))) + ) + + let largeReadStatus = journal.status(normalizedCommandId: "large-read") + XCTAssertEqual(largeReadStatus.lifecycleState, RunnerCommandLifecycleState.completed.rawValue) + XCTAssertEqual(largeReadStatus.lifecycleResponseOk, true) + XCTAssertNil(largeReadStatus.lifecycleResponseJson) + } + + func testCommandJournalKeepsErrorMetadataWhenResponseJsonIsDropped() { + let journal = RunnerCommandJournal() + let snapshot = runnerJournalCommand("snapshot", id: "snapshot-error") + let hint = "Try a smaller read such as snapshot -s -d 8." + + journal.accept(command: snapshot) + journal.finish( + command: snapshot, + response: Response( + ok: false, + error: ErrorPayload( + code: "IOS_AX_SNAPSHOT_FAILED", + message: "iOS XCTest snapshot failed while serializing the accessibility tree.", + hint: hint + ) + ) + ) + + let status = journal.status(normalizedCommandId: "snapshot-error") + XCTAssertEqual(status.lifecycleState, RunnerCommandLifecycleState.failed.rawValue) + XCTAssertEqual(status.lifecycleResponseOk, false) + XCTAssertNil(status.lifecycleResponseJson) + XCTAssertEqual(status.lifecycleErrorCode, "IOS_AX_SNAPSHOT_FAILED") + XCTAssertEqual( + status.lifecycleErrorMessage, + "iOS XCTest snapshot failed while serializing the accessibility tree." + ) + XCTAssertEqual(status.lifecycleErrorHint, hint) + } + + func testCommandJournalRetainsCompletedSequenceResults() throws { + let journal = RunnerCommandJournal() + let sequence = runnerJournalCommand("sequence", id: "sequence-completed") + let results = (0..<20).map { _ in + SequenceStepResult( + ok: true, + kind: "tap", + errorCode: nil, + errorMessage: nil, + gestureStartUptimeMs: 100, + gestureEndUptimeMs: 120 + ) + } + + journal.accept(command: sequence) + journal.finish( + command: sequence, + response: Response( + ok: true, + data: DataPayload( + message: "sequence", + completedSteps: 20, + failedStepIndex: nil, + sequenceResults: results + ) + ) + ) + + let status = journal.status(normalizedCommandId: "sequence-completed") + XCTAssertEqual(status.lifecycleState, RunnerCommandLifecycleState.completed.rawValue) + XCTAssertEqual(status.lifecycleResponseOk, true) + let json = try XCTUnwrap(status.lifecycleResponseJson) + // Worst-case 20-step response must stay under the 16KB journal retention cap. + XCTAssertLessThan(json.utf8.count, 16 * 1024) + let decoded = try decodeRunnerJournalResponse(status.lifecycleResponseJson) + XCTAssertEqual(decoded.data?.completedSteps, 20) + XCTAssertEqual(decoded.data?.sequenceResults?.count, 20) + } + + func testCommandJournalRetainsFailedSequenceResults() throws { + let journal = RunnerCommandJournal() + let sequence = runnerJournalCommand("sequence", id: "sequence-failed") + let longError = String(repeating: "z", count: 200) + let results: [SequenceStepResult] = [ + SequenceStepResult(ok: true, kind: "tap", errorCode: nil, errorMessage: nil, + gestureStartUptimeMs: 100, gestureEndUptimeMs: 120), + SequenceStepResult(ok: true, kind: "tap", errorCode: nil, errorMessage: nil, + gestureStartUptimeMs: 130, gestureEndUptimeMs: 150), + SequenceStepResult(ok: false, kind: "longPress", errorCode: "UNSUPPORTED_OPERATION", + errorMessage: longError, gestureStartUptimeMs: 160, gestureEndUptimeMs: 180), + ] + + journal.accept(command: sequence) + journal.finish( + command: sequence, + response: Response( + ok: true, + data: DataPayload( + message: "sequence", + completedSteps: 2, + failedStepIndex: 2, + sequenceResults: results + ) + ) + ) + + let status = journal.status(normalizedCommandId: "sequence-failed") + XCTAssertEqual(status.lifecycleState, RunnerCommandLifecycleState.completed.rawValue) + let decoded = try decodeRunnerJournalResponse(status.lifecycleResponseJson) + XCTAssertEqual(decoded.data?.completedSteps, 2) + XCTAssertEqual(decoded.data?.failedStepIndex, 2) + XCTAssertEqual(decoded.data?.sequenceResults?.count, 3) + XCTAssertEqual(decoded.data?.sequenceResults?[2].ok, false) + XCTAssertEqual(decoded.data?.sequenceResults?[2].errorCode, "UNSUPPORTED_OPERATION") + } + + private func runnerJournalCommand(_ command: String, id: String) -> Command { + let json = #"{"command":"\#(command)","commandId":"\#(id)"}"# + return try! JSONDecoder().decode(Command.self, from: Data(json.utf8)) + } + + private func runnerJournalNode() -> PresentedNode { + SnapshotPresentation.singleElementRead( + RawAXNode( + index: 0, + type: "button", + label: "Continue", + identifier: "continue", + value: nil, + rect: SnapshotRect(x: 10, y: 20, width: 100, height: 44), + enabled: true, + focused: nil, + selected: nil, + hittable: true, + depth: 0, + parentIndex: nil, + hiddenContentAbove: nil, + hiddenContentBelow: nil + ) + ) + } + + private func decodeRunnerJournalResponse(_ responseJson: String?) throws -> Response { + let responseJson = try XCTUnwrap(responseJson) + return try JSONDecoder().decode(Response.self, from: Data(responseJson.utf8)) + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+InteractionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+InteractionTests.swift new file mode 100644 index 0000000000..659dbd9c8b --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+InteractionTests.swift @@ -0,0 +1,27 @@ +import XCTest +import AgentDeviceSnapshotPresentation + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +extension RunnerTests { + func testDesktopScrollWheelDeltasMapDirections() { + XCTAssertEqual(desktopScrollWheelDeltas(direction: .up, pixels: 120).vertical, 120) + XCTAssertEqual(desktopScrollWheelDeltas(direction: .down, pixels: 120).vertical, -120) + XCTAssertEqual(desktopScrollWheelDeltas(direction: .left, pixels: 120).horizontal, 120) + XCTAssertEqual(desktopScrollWheelDeltas(direction: .right, pixels: 120).horizontal, -120) + } + + func testDesktopScrollWheelDeltaEventsHonorDurationAndPreservePixels() { + let events = desktopScrollWheelDeltaEvents(direction: .down, pixels: 200, durationMs: 50) + XCTAssertEqual(events.count, 4) + XCTAssertEqual(events.map(\.vertical).reduce(0, +), -200) + XCTAssertEqual(events.map(\.horizontal).reduce(0, +), 0) + XCTAssertEqual(desktopScrollEventIntervalSeconds(durationMs: 50, eventCount: events.count), 0.05 / 3.0) + } + + func testDesktopScrollWheelDeltaEventsKeepInstantScrollSingleEvent() { + let events = desktopScrollWheelDeltaEvents(direction: .down, pixels: 200, durationMs: 0) + XCTAssertEqual(events.count, 1) + XCTAssertEqual(events.first?.vertical, -200) + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+KeyboardTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+KeyboardTests.swift new file mode 100644 index 0000000000..8264903617 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+KeyboardTests.swift @@ -0,0 +1,144 @@ +import XCTest +import AgentDeviceSnapshotPresentation + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +extension RunnerTests { + func testKeyboardBandFactReadFailureIsStatedAsUnmeasurableNotAbsence() { + // Absence would let a consumer claim the screen is clear of a keyboard on the strength of a read + // that never answered. + let fact = runnerKeyboardBandFact( + readSucceeded: false, + exists: false, + frame: CGRect(x: 0, y: 198, width: 402, height: 204) + ) + XCTAssertEqual(fact, .unmeasurable(RunnerKeyboardBandReason.queryFailed)) + XCTAssertEqual(fact.payload.kind, "unmeasurable") + XCTAssertEqual(fact.payload.reason, RunnerKeyboardBandReason.queryFailed) + XCTAssertNil(fact.payload.frame) + } + + func testKeyboardBandFactPublishesAbsenceWhenTheQueryFindsNoKeyboard() { + let fact = runnerKeyboardBandFact(readSucceeded: true, exists: false, frame: .zero) + XCTAssertEqual(fact, .absent) + XCTAssertEqual(fact.payload.kind, "absent") + XCTAssertNil(fact.payload.frame) + XCTAssertNil(fact.payload.reason) + } + + func testKeyboardBandFactPublishesTheMeasuredBandInAppOrientationSpace() { + // The landscape band measured on iPhone 17 Pro (iOS 26.2) after #2653: full width across the + // bottom of a 402 pt-tall app, which is what the tree reported as a strip down the left edge. + let frame = CGRect(x: 0, y: 198, width: 874, height: 204) + let fact = runnerKeyboardBandFact(readSucceeded: true, exists: true, frame: frame) + XCTAssertEqual(fact, .visible(frame)) + let payload = fact.payload + XCTAssertEqual(payload.kind, "visible") + XCTAssertEqual(payload.frame, SnapshotRect(x: 0, y: 198, width: 874, height: 204)) + XCTAssertNil(payload.reason) + } + + func testKeyboardBandFactRefusesUnusableGeometryInsteadOfClaimingAbsence() { + let nan = CGFloat(Double.nan) + let infinite = CGFloat.infinity + let unusable: [CGRect] = [ + .zero, + CGRect(x: 0, y: 198, width: 0, height: 204), + CGRect(x: 0, y: 198, width: 874, height: -1), + CGRect(x: 0, y: 198, width: -874, height: 204), + CGRect(x: 0, y: nan, width: 874, height: 204), + CGRect(x: 0, y: 198, width: nan, height: 204), + CGRect(x: infinite, y: 198, width: 874, height: 204) + ] + for frame in unusable { + let fact = runnerKeyboardBandFact(readSucceeded: true, exists: true, frame: frame) + XCTAssertEqual( + fact, + .unmeasurable(RunnerKeyboardBandReason.unusableFrame), + "expected \(frame) to be refused as a band" + ) + } + } + + func testKeyboardBandFactPayloadRoundTripsThroughTheWireShape() throws { + let cases: [RunnerKeyboardBandFact] = [ + .visible(CGRect(x: 0, y: 583, width: 402, height: 291)), + .absent, + .unmeasurable(RunnerKeyboardBandReason.queryTimeout) + ] + for fact in cases { + let data = try JSONEncoder().encode(fact.payload) + // `encodeIfPresent` for the two optional fields: a fact carries its own evidence and nothing + // else, so the daemon never has to distinguish a null from an absent key. + let object = try XCTUnwrap(JSONSerialization.jsonObject(with: data) as? [String: Any]) + switch fact { + case .visible: + XCTAssertNil(object["reason"]) + XCTAssertNotNil(object["frame"]) + case .absent: + XCTAssertNil(object["frame"]) + XCTAssertNil(object["reason"]) + case .unmeasurable: + XCTAssertNil(object["frame"]) + XCTAssertNotNil(object["reason"]) + } + XCTAssertEqual(try JSONDecoder().decode(KeyboardBandFactPayload.self, from: data), fact.payload) + } + } + + func testRunnerScreenshotStabilitySettledNeedsEnoughSamples() { + XCTAssertFalse(runnerScreenshotStabilitySettled([], requiredConsecutiveMatches: 3)) + XCTAssertFalse(runnerScreenshotStabilitySettled([Data([1])], requiredConsecutiveMatches: 3)) + let frame = Data([1, 2, 3]) + XCTAssertFalse( + runnerScreenshotStabilitySettled([frame, frame], requiredConsecutiveMatches: 3) + ) + } + + func testRunnerScreenshotStabilitySettledTrueWhenWindowMatches() { + let frame = Data([1, 2, 3]) + XCTAssertTrue( + runnerScreenshotStabilitySettled([Data([9]), frame, frame, frame], requiredConsecutiveMatches: 3) + ) + } + + func testRunnerScreenshotStabilitySettledFalseOnMidWindowMismatch() { + // A momentary pause (two matching samples) followed by resumed movement + // must not read as settled: the 3-sample window still spans the mismatch. + let frame = Data([1, 2, 3]) + let moved = Data([4, 5, 6]) + XCTAssertFalse( + runnerScreenshotStabilitySettled([frame, frame, moved], requiredConsecutiveMatches: 3) + ) + } + + func testRunnerScreenshotStabilitySettledFalseOnFailedCapture() { + // A nil sample (failed screenshot) never counts as a match, even against + // other nils — an unverifiable run must not look "stable". + XCTAssertFalse(runnerScreenshotStabilitySettled([nil, nil, nil], requiredConsecutiveMatches: 3)) + let frame = Data([1, 2, 3]) + XCTAssertFalse( + runnerScreenshotStabilitySettled([frame, frame, nil], requiredConsecutiveMatches: 3) + ) + } + + func testRunnerScreenshotStabilitySettledOnlyLooksAtTheTrailingWindow() { + // An older mismatch before the trailing window must not block settlement + // once the required run of most-recent samples agrees. + let frame = Data([9]) + XCTAssertTrue( + runnerScreenshotStabilitySettled( + [Data([1]), Data([2]), frame, frame, frame], + requiredConsecutiveMatches: 3 + ) + ) + } + + func testRunnerScreenshotStabilitySettledRejectsDegenerateRequirement() { + // Fewer than 2 required matches would make any single sample "settled" — + // guard against a misconfigured caller rather than silently no-op the wait. + XCTAssertFalse( + runnerScreenshotStabilitySettled([Data([1])], requiredConsecutiveMatches: 1) + ) + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LifecycleTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LifecycleTests.swift index cba532f03b..559fa9021b 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LifecycleTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LifecycleTests.swift @@ -104,3 +104,33 @@ extension RunnerTests { } } #endif + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +extension RunnerTests { + func testResettingTargetBoundStateForgetsTheLastWrittenMarkers() { + defer { invalidateCachedTarget(reason: "unit_test_cleanup") } + lastLoggedFastAppGuardLine = "AGENT_DEVICE_RUNNER_FAST_APP_GUARD bundle=app state=4" + lastLoggedGesturePolicyLines[.scroll] = "AGENT_DEVICE_RUNNER_SYNTHESIZED_GESTURE_POLICY kind=scroll" + resetTargetBoundState() + XCTAssertNil(lastLoggedFastAppGuardLine, "a rebind must state the guard once more") + XCTAssertTrue(lastLoggedGesturePolicyLines.isEmpty, "a rebind must state the policy once more") + } + + func testFastAppGuardMarkerWritesOnceUntilTheFactChanges() { + var written: [String] = [] + runnerMarkerWriter = { written.append($0) } + defer { + runnerMarkerWriter = { NSLog("%@", $0) } + invalidateCachedTarget(reason: "unit_test_cleanup") + } + writeFastAppGuardMarker(bundleId: "com.example.app", state: .runningForeground) + writeFastAppGuardMarker(bundleId: "com.example.app", state: .runningForeground) + XCTAssertEqual(written.count, 1, "a repeated fact writes no second line") + writeFastAppGuardMarker(bundleId: "com.example.other", state: .runningForeground) + XCTAssertEqual(written.count, 2, "a changed fact writes a new line") + resetTargetBoundState() + writeFastAppGuardMarker(bundleId: "com.example.other", state: .runningForeground) + XCTAssertEqual(written.count, 3, "a rebind states the same fact once more") + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+MainThreadWorkTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+MainThreadWorkTests.swift new file mode 100644 index 0000000000..f98e37e7a8 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+MainThreadWorkTests.swift @@ -0,0 +1,93 @@ +import XCTest + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +extension RunnerTests { + func testRunMainThreadWorkExecutesOffMainCallerOnMainThread() { + final class ResultBox { + var observedMainThread: Bool? + var error: Error? + } + let box = ResultBox() + let finished = expectation(description: "off-main caller finished") + + DispatchQueue(label: "agent-device.runner.tests.off-main").async { + do { + box.observedMainThread = try self.runMainThreadWork( + "command_execution", + timeout: 1, + timeoutError: self.mainThreadExecutionTimeoutError + ) { + Thread.isMainThread + } + } catch { + box.error = error + } + finished.fulfill() + } + + wait(for: [finished], timeout: 2) + XCTAssertNil(box.error) + XCTAssertEqual(box.observedMainThread, true) + } + + func testRunMainThreadWorkTimeoutMarksAbandonedUntilDrained() { + final class ResultBox { + var error: Error? + var abandonedCount: Int? + var abandonedSinceSet: Bool? + var busyWhileAbandoned = false + } + let box = ResultBox() + let releaseWork = DispatchSemaphore(value: 0) + let observedAbandoned = DispatchSemaphore(value: 0) + let timedOut = expectation(description: "off-main caller timed out") + + DispatchQueue(label: "agent-device.runner.tests.timeout").async { + do { + _ = try self.runMainThreadWork( + "command_execution", + timeout: 0, + timeoutError: self.mainThreadExecutionTimeoutError + ) { + _ = releaseWork.wait(timeout: .now() + 2) + return true + } + } catch { + box.error = error + } + self.mainThreadWorkLock.lock() + box.abandonedCount = self.abandonedMainThreadWorkCount + box.abandonedSinceSet = self.abandonedMainThreadWorkSince != nil + self.mainThreadWorkLock.unlock() + if case .busy = self.currentMainThreadBusyState() { + box.busyWhileAbandoned = true + } + observedAbandoned.signal() + timedOut.fulfill() + } + DispatchQueue(label: "agent-device.runner.tests.release-timeout").async { + _ = observedAbandoned.wait(timeout: .now() + 2) + releaseWork.signal() + } + + wait(for: [timedOut], timeout: 3) + let drainDeadline = Date().addingTimeInterval(2) + while hasAbandonedMainThreadWork(), Date() < drainDeadline { + sleepFor(0.005) + } + + XCTAssertEqual((box.error as NSError?)?.code, RunnerErrorCode.mainThreadExecutionTimedOut) + XCTAssertEqual(box.abandonedCount, 1) + XCTAssertEqual(box.abandonedSinceSet, true) + XCTAssertTrue(box.busyWhileAbandoned) + XCTAssertFalse(hasAbandonedMainThreadWork(), "drained work must release the main thread") + mainThreadWorkLock.lock() + let sinceCleared = abandonedMainThreadWorkSince == nil + mainThreadWorkLock.unlock() + XCTAssertTrue(sinceCleared) + guard case .idle = currentMainThreadBusyState() else { + return XCTFail("expected the runner idle once the abandoned work drained") + } + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift new file mode 100644 index 0000000000..e18e91241f --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift @@ -0,0 +1,169 @@ +import XCTest + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +extension RunnerTests { + func testTopLeadingNavigationFallbackPointTargetsHeaderControlBand() throws { + let point = try XCTUnwrap( + Self.topLeadingNavigationFallbackPoint( + in: CGRect(x: 0, y: 0, width: 430, height: 932) + ) + ) + + XCTAssertEqual(point.x, 34.4, accuracy: 0.01) + XCTAssertEqual(point.y, 132, accuracy: 0.01) + } + + func testTopLeadingNavigationFallbackPointRejectsInvalidFrame() { + XCTAssertNil(Self.topLeadingNavigationFallbackPoint(in: .infinite)) + XCTAssertNil(Self.topLeadingNavigationFallbackPoint(in: .zero)) + } + + func testNavigationBackControlRankPrefersBackThenCloseThenCancel() { + XCTAssertEqual(Self.navigationBackControlRank(label: "Back", identifier: ""), 0) + XCTAssertEqual(Self.navigationBackControlRank(label: "Close", identifier: ""), 1) + XCTAssertEqual(Self.navigationBackControlRank(label: "Cancel search", identifier: ""), 2) + XCTAssertNil(Self.navigationBackControlRank(label: "Search for more feeds", identifier: "")) + } + + func testNavigationBackPredicateUsesTheSharedKeywordTable() { + let predicate = Self.navigationBackPredicate() + + XCTAssertTrue(predicate.evaluate(with: ["label": "Back", "identifier": ""])) + XCTAssertTrue(predicate.evaluate(with: ["label": "", "identifier": "close-button"])) + XCTAssertFalse(predicate.evaluate(with: ["label": "Search for more feeds", "identifier": ""])) + } + + func testTopNavigationControlFrameAcceptsOnlyHeaderBand() { + let window = CGRect(x: 0, y: 0, width: 430, height: 932) + + XCTAssertTrue( + Self.isTopNavigationControlFrame( + CGRect(x: 340, y: 84, width: 72, height: 44), + in: window + ) + ) + XCTAssertFalse( + Self.isTopNavigationControlFrame( + CGRect(x: 20, y: 760, width: 72, height: 44), + in: window + ) + ) + XCTAssertFalse(Self.isTopNavigationControlFrame(.infinite, in: window)) + } + + func testNavigationVisualVerificationSeparatesNoChangeFromNoSample() { + XCTAssertEqual( + Self.navigationVisualObservation(before: Data([1, 2, 3]), after: Data([1, 2, 4])), + .changed + ) + XCTAssertEqual( + Self.navigationVisualObservation(before: Data([1, 2, 3]), after: Data([1, 2, 3])), + .unchanged + ) + // A missing sample is neither a change nor a no-change; treating it as "unchanged" would let a + // capture that refused become the reason the `back` command claims no control exists (#2728). + XCTAssertEqual(Self.navigationVisualObservation(before: nil, after: Data([1])), .unobserved) + XCTAssertEqual(Self.navigationVisualObservation(before: Data([1]), after: nil), .unobserved) + XCTAssertEqual(Self.navigationVisualObservation(before: nil, after: nil), .unobserved) + } + + func testNavigationFallbackReportsTheRefusalItHitNotADefaultCode() { + // The refusal from the most recent sample wins, so the code names what the fallback last looked at + // before giving up; an earlier refusal is reported only when the later sample carried none (#2728). + let after = NavigationVisualSample( + data: nil, + refusalCode: "APP_SCREEN_WINDOW_UNRESOLVED", + refusalHint: "after hint" + ) + let before = NavigationVisualSample( + data: nil, + refusalCode: "APP_SCREEN_UNRESOLVED", + refusalHint: "before hint" + ) + let laterWins = Self.navigationFallbackErrorPayload(after: after, before: before) + XCTAssertEqual(laterWins.code, "APP_SCREEN_WINDOW_UNRESOLVED") + XCTAssertEqual(laterWins.hint, "after hint") + + let onlyBefore = Self.navigationFallbackErrorPayload( + after: NavigationVisualSample(data: nil), + before: before + ) + XCTAssertEqual(onlyBefore.code, "APP_SCREEN_UNRESOLVED") + XCTAssertEqual(onlyBefore.hint, "before hint") + + // Neither side named a reason (unreachable on iOS): a real capture code, never a bare failure. + let unnamed = Self.navigationFallbackErrorPayload( + after: NavigationVisualSample(data: nil), + before: NavigationVisualSample(data: nil) + ) + XCTAssertEqual(unnamed.code, "APP_SCREEN_UNRESOLVED") + XCTAssertTrue(unnamed.message.contains("unknown outcome")) + } + + func testVerifyNavigationFallbackOutcomeReportsUnresolvedWindowWithoutSystemSurface() { + // The in-app `back` fallback ran its tap but the app resolved no window. It must name that refusal + // as an unknown outcome AND must not have sampled the system surface: capturing SpringBoard's home + // screen twice reads as "unchanged" and launders a wrong-process frame into "no back control + // exists" (#2728). This drives the SAME `navigationFallbackSample` production calls, handing it a + // system surface that fails the test if consulted — so reverting the fallback to sample SpringBoard + // (or dropping the refusal code) turns this red, which an inline `.never` re-creation could not. + var askedSystemSurface = false + let sample = Self.navigationFallbackSample( + resolvingApp: { .failure(.unresolvedWindow) }, + systemSurface: { + askedSystemSurface = true + return .failure(.unresolvedWindow) + }, + encoding: { _ in Data([1, 2, 3]) } + ) + XCTAssertFalse(askedSystemSurface, "the in-app fallback samples the app only, never SpringBoard") + + XCTAssertNil(sample.data) + XCTAssertEqual(sample.refusalCode, "APP_SCREEN_WINDOW_UNRESOLVED") + + let observation = Self.navigationVisualObservation(before: sample.data, after: sample.data) + XCTAssertEqual(observation, .unobserved) + + switch Self.inAppBackOutcome(observation: observation, before: sample, after: sample) { + case .unverified(let payload): + XCTAssertEqual(payload.code, "APP_SCREEN_WINDOW_UNRESOLVED") + case .performed, .unavailable: + XCTFail("a refused capture must report an unknown outcome, not 'no back control'") + } + } + + func testNavigationVisualSampleDistinguishesEncodedFrameFromRefusal() { + // The same capture entry point yields three different samples, and only the refusal ones may carry + // a code: an encoded frame is evidence, a resolved-but-unencodable image and a refusal are not + // (#2728). Reverting the mapping to a plain no-sample loses the reason a host keys on. + let captured = CapturedAppScreen( + image: RunnerImage(), + displayID: 3, + pixelWidth: 12, + pixelHeight: 24, + pixelsPerPoint: 3 + ) + + let encoded = Self.navigationVisualSample( + from: .success(captured), + encoding: { _ in Data([7, 7]) } + ) + XCTAssertEqual(encoded.data, Data([7, 7])) + XCTAssertNil(encoded.refusalCode) + + let unencodable = Self.navigationVisualSample( + from: .success(captured), + encoding: { _ in nil } + ) + XCTAssertNil(unencodable.data) + XCTAssertEqual(unencodable.refusalCode, "APP_SCREEN_CAPTURE_UNRENDERABLE") + + let refused = Self.navigationVisualSample( + from: .failure(.unresolvedScreen), + encoding: { _ in Data([7, 7]) } + ) + XCTAssertNil(refused.data) + XCTAssertEqual(refused.refusalCode, "APP_SCREEN_UNRESOLVED") + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+PrivateAXPresentationTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+PrivateAXPresentationTests.swift new file mode 100644 index 0000000000..23ee5db993 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+PrivateAXPresentationTests.swift @@ -0,0 +1,211 @@ +import XCTest +import AgentDeviceSnapshotPresentation + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +extension RunnerTests { + fileprivate static func privateAXFrame( + _ x: Double, _ y: Double, _ width: Double, _ height: Double + ) -> [String: Any] { + ["x": x, "y": y, "width": width, "height": height] + } + + /// A scroll container whose second row is scrolled out of the viewport, plus an unlabeled + /// decoration: the shapes the regular projection folds away and the raw projection must keep. + fileprivate static var privateAXScrolledFixture: [String: Any] { + let frame = privateAXFrame + return ["type": Int(XCUIElement.ElementType.application.rawValue), + "label": "Element", "frame": frame(0, 0, 402, 874), "children": [[ + "type": Int(XCUIElement.ElementType.scrollView.rawValue), "frame": frame(0, 96, 402, 700), + "actions": ["Scroll down"], + "children": [ + ["type": Int(XCUIElement.ElementType.button.rawValue), "label": "Profile picture", + "frame": frame(16, 120, 44, 44), + "children": [["type": Int(XCUIElement.ElementType.image.rawValue), + "frame": frame(16, 120, 1, 1)]]], + ["type": Int(XCUIElement.ElementType.button.rawValue), "label": "Theme", + "frame": frame(16, 900, 360, 44)]]]]] + } + + /// Acquire with the private-AX serializer, run the one normalization pass the production capture + /// plan runs (`captureWithBackend`), then present through the shared regular fold -- the production + /// route for this backend since the fold moved into presentation (#1797, #2661). + fileprivate func privateAXRegularPresentation( + rawRoot: [String: Any], + viewport: CGRect, + interactiveOnly: Bool = false + ) throws -> [PresentedNode] { + let hint = CaptureHint( + projection: .regular, depth: nil, regularPresentedDepth: nil, + interactiveOnly: interactiveOnly, customActions: false) + let nodes = privateAXNormalizedAcquisition( + rawRoot: rawRoot, hint: hint, viewport: viewport, + interfaceOrientation: RunnerInterfaceOrientation.portrait) + return try SnapshotPresentation.presentRegular( + SnapshotAcquisition( + hint: hint, nodes: nodes, truncated: false, effectiveDepth: nil, viewport: viewport, + interfaceOrientation: RunnerInterfaceOrientation.portrait), + options: PresentationOptions( + interactiveOnly: interactiveOnly, depth: nil, scope: nil, raw: false), + policy: .cursorProjected + ).nodes + } + + /// Acquire, then normalize once -- the exact pair `captureWithBackend` runs for this backend. + fileprivate func privateAXNormalizedAcquisition( + rawRoot: [String: Any], + hint: CaptureHint, + viewport: CGRect, + interfaceOrientation: Int + ) -> [RawAXNode] { + SnapshotGeometrySpace.normalized( + nodes: privateAXAcquisition(rawRoot: rawRoot, hint: hint), + viewport: viewport, + interfaceOrientation: interfaceOrientation + ) + } + + /// The one normalization pass has to reach the node it publishes: this asserts the rotated rect of + /// a key under a turned surface host, and an untouched sibling under the app's own window, so a + /// pass that drops the space a subtree declared fails here. + func testPrivateAXAcquisitionPublishesATurnedSurfaceHostInAppOrientationSpace() { + let frame = Self.privateAXFrame + let appWindow: [String: Any] = [ + "type": Int(XCUIElement.ElementType.window.rawValue), + "frame": frame(0, 0, 874, 402), + "children": [ + ["type": Int(XCUIElement.ElementType.button.rawValue), "label": "Home", + "frame": frame(204, 323, 91, 55), "children": []] + ] + ] + let keyboardWindow: [String: Any] = [ + "type": Int(XCUIElement.ElementType.window.rawValue), + "frame": frame(0, 0, 874, 402), + "children": [ + ["type": Int(XCUIElement.ElementType.other.rawValue), + "frame": frame(0, 0, 402, 874), + "children": [ + ["type": Int(XCUIElement.ElementType.key.rawValue), "label": "q", + "frame": frame(154, 77, 45, 72), "children": []] + ]] + ] + ] + let hint = CaptureHint( + projection: .raw, depth: nil, regularPresentedDepth: nil, + interactiveOnly: false, customActions: false) + let nodes = privateAXNormalizedAcquisition( + rawRoot: [ + "type": Int(XCUIElement.ElementType.application.rawValue), + "label": "Element", "frame": frame(0, 0, 874, 402), + "children": [appWindow, keyboardWindow] + ], + hint: hint, + viewport: CGRect(x: 0, y: 0, width: 874, height: 402), + interfaceOrientation: RunnerInterfaceOrientation.landscapeRight + ) + + // Measured on iPhone 17 Pro (26.2): the key plane's left column arrives 154 pt along the device's + // long axis and comes back 203 pt down the app's short one. + XCTAssertEqual( + nodes.first { $0.label == "q" }?.rect, + SnapshotRect(x: 77, y: 203, width: 72, height: 45) + ) + XCTAssertEqual( + nodes.first { $0.label == "Home" }?.rect, + SnapshotRect(x: 204, y: 323, width: 91, height: 55) + ) + } + + func testPrivateAXRegularPresentationProjectsToViewportAndKeepsScrollHint() throws { + let nodes = try privateAXRegularPresentation( + rawRoot: Self.privateAXScrolledFixture, + viewport: CGRect(x: 0, y: 0, width: 402, height: 874)) + XCTAssertEqual(nodes.compactMap(\.label), ["Element", "Profile picture"]) + let scrollView = nodes.first { $0.type == "ScrollView" } + XCTAssertEqual(scrollView?.hiddenContentBelow, true) + XCTAssertEqual(scrollView?.actions, ["Scroll down"]) + } + + /// #1797 D4: the raw projection is the acquired tree. The offscreen row and the sub-pixel + /// decoration the regular projection folds away are both present, at traversal depth, and every + /// regular node still appears -- `regular ⊆ raw` on the same capture. + func testPrivateAXRawProjectionKeepsEveryAcquiredNode() throws { + let viewport = CGRect(x: 0, y: 0, width: 402, height: 874) + let root = Self.privateAXScrolledFixture + let regular = try privateAXRegularPresentation(rawRoot: root, viewport: viewport, + interactiveOnly: true) + let raw = privateAXNormalizedAcquisition(rawRoot: root, + hint: CaptureHint( + projection: .raw, depth: nil, regularPresentedDepth: nil, + interactiveOnly: false, customActions: false), + viewport: viewport, + interfaceOrientation: RunnerInterfaceOrientation.portrait) + + XCTAssertEqual(raw.map(\.type), ["Application", "ScrollView", "Button", "Image", "Button"]) + XCTAssertEqual(raw.map(\.depth), [0, 1, 2, 3, 2]) + XCTAssertEqual(raw.map(\.parentIndex), [nil, 0, 1, 2, 1]) + XCTAssertEqual(raw.compactMap(\.label), ["Element", "Profile picture", "Theme"]) + // The offscreen row is a reported fact in raw, and reported facts do not become hittable + // just because the projection kept them. + XCTAssertEqual(raw.last?.hittable, false) + XCTAssertTrue(raw.allSatisfy { $0.hiddenContentAbove == nil && $0.hiddenContentBelow == nil }) + + let rawKeys = Set(raw.map { "\($0.type)-\($0.label ?? "")-\($0.rect.y)" }) + for node in regular { + XCTAssertTrue( + rawKeys.contains("\(node.type)-\(node.label ?? "")-\(node.rect.y)"), + "regular node \(node.type)/\(node.label ?? "") is missing from the raw projection" + ) + } + XCTAssertGreaterThan(raw.count, regular.count) + } + + /// Raw depth is traversal depth, so a raw `--depth` request is the one narrowing this backend + /// can prove complete. + func testPrivateAXRawProjectionAppliesRequestedTraversalDepth() { + let raw = privateAXNormalizedAcquisition(rawRoot: Self.privateAXScrolledFixture, + hint: CaptureHint( + projection: .raw, depth: 2, regularPresentedDepth: nil, + interactiveOnly: false, customActions: false), + viewport: CGRect(x: 0, y: 0, width: 402, height: 874), + interfaceOrientation: RunnerInterfaceOrientation.portrait) + XCTAssertEqual(raw.map(\.type), ["Application", "ScrollView", "Button", "Button"]) + XCTAssertEqual(raw.map(\.depth), [0, 1, 2, 2]) + } + + func testPrivateAXPresentationKeepsOffscreenSubtreeExcludedWhenChildFramesAreClamped() throws { + let frame = Self.privateAXFrame + let root: [String: Any] = ["type": Int(XCUIElement.ElementType.application.rawValue), + "label": "Element", "frame": frame(0, 0, 402, 874), "children": [[ + "type": Int(XCUIElement.ElementType.table.rawValue), "frame": frame(0, 96, 402, 700), + "children": [["type": Int(XCUIElement.ElementType.cell.rawValue), + "label": "Theme", "frame": frame(0, 900, 402, 44), "children": [[ + "type": Int(XCUIElement.ElementType.staticText.rawValue), + "label": "Theme", "frame": frame(16, 96, 120, 44)], + ["type": Int(XCUIElement.ElementType.switch.rawValue), + "label": "Theme", "frame": frame(340, 96, 46, 44)]]]]]]] + + let nodes = try privateAXRegularPresentation( + rawRoot: root, viewport: CGRect(x: 0, y: 0, width: 402, height: 874)) + + XCTAssertEqual(nodes.compactMap(\.label), ["Element"]) + XCTAssertEqual(nodes.first { $0.type == "Table" }?.hiddenContentBelow, true) + } + + func testPrivateAXGeometrylessSemanticsAreNeverActionableOrScrollContexts() throws { + let zero = ["x": 0, "y": 0, "width": 0, "height": 0] + let root: [String: Any] = ["type": Int(XCUIElement.ElementType.application.rawValue), + "label": "Element", "frame": ["x": 0, "y": 0, "width": 402, "height": 874], + "children": [["type": Int(XCUIElement.ElementType.scrollView.rawValue), + "label": "Settings semantics", "frame": zero, "children": [[ + "type": Int(XCUIElement.ElementType.button.rawValue), "label": "Theme", "frame": zero], + ["type": Int(XCUIElement.ElementType.other.rawValue), "frame": zero]]]]] + let nodes = try privateAXRegularPresentation( + rawRoot: root, viewport: CGRect(x: 0, y: 0, width: 402, height: 874), + interactiveOnly: true) + XCTAssertEqual(nodes.compactMap(\.label), ["Element", "Settings semantics", "Theme"]) + XCTAssertEqual(nodes.filter { $0.index != 0 }.map(\.hittable), [false, false]) + XCTAssertFalse(nodes.contains { $0.type == "Other" }) + XCTAssertTrue(nodes.allSatisfy { $0.hiddenContentAbove == nil && $0.hiddenContentBelow == nil }) + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScrollGestureTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScrollGestureTests.swift new file mode 100644 index 0000000000..9e9b9d26fc --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScrollGestureTests.swift @@ -0,0 +1,197 @@ +import XCTest + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +private struct ScrollGestureFixture: Decodable { + struct Constants: Decodable { + let defaultIosScrollAmount: Double + let defaultMobileScrollDurationMs: Double + let defaultIosScrollDurationMs: Double + let defaultScrollAmount: Double + let defaultEdgePaddingFraction: Double + let ordinaryScrollReleaseBehavior: String + let edgeScrollReleaseBehavior: String + } + struct Expected: Decodable { + let x1: Double + let y1: Double + let x2: Double + let y2: Double + let pixels: Double + } + struct Case: Decodable { + let name: String + let direction: String + let amount: Double? + let pixels: Double? + let referenceWidth: Double + let referenceHeight: Double + let expected: Expected + } + + let constants: Constants + let cases: [Case] +} + +extension RunnerTests { + // Cross-language parity table: every case in contracts/fixtures/scroll-gesture.json must agree + // with the vitest twin (packages/contracts/src/scroll-gesture.test.ts). Add vectors there, + // never fork the math. + private func loadScrollGestureFixture() throws -> ScrollGestureFixture { + let fixtureURL = URL(fileURLWithPath: #filePath) + .deletingLastPathComponent() // UnitTests + .deletingLastPathComponent() // AgentDeviceRunnerUITests + .deletingLastPathComponent() // AgentDeviceRunner + .deletingLastPathComponent() // runner + .deletingLastPathComponent() // apple + .deletingLastPathComponent() // repo root + .appendingPathComponent("contracts") + .appendingPathComponent("fixtures") + .appendingPathComponent("scroll-gesture.json") + return try JSONDecoder().decode(ScrollGestureFixture.self, from: Data(contentsOf: fixtureURL)) + } + + func testRunnerScrollGesturePlanMatchesParityTable() throws { + let fixture = try loadScrollGestureFixture() + XCTAssertFalse(fixture.cases.isEmpty, "parity table must not be empty") + for testCase in fixture.cases { + let plan = try XCTUnwrap( + runnerScrollGesturePlan( + direction: try XCTUnwrap(RunnerScrollDirection(rawValue: testCase.direction)), + amount: testCase.amount, + pixels: testCase.pixels, + referenceWidth: testCase.referenceWidth, + referenceHeight: testCase.referenceHeight + ), + testCase.name + ) + XCTAssertEqual(plan.x1, testCase.expected.x1, testCase.name) + XCTAssertEqual(plan.y1, testCase.expected.y1, testCase.name) + XCTAssertEqual(plan.x2, testCase.expected.x2, testCase.name) + XCTAssertEqual(plan.y2, testCase.expected.y2, testCase.name) + XCTAssertEqual(plan.travelPixels, testCase.expected.pixels, testCase.name) + } + } + + // The planner constants are private on both sides; the table pins them behaviourally on a + // 1000px axis where every rounding step is exact. + func testRunnerScrollGesturePlanUsesParityTableConstants() throws { + let constants = try loadScrollGestureFixture().constants + let defaultScroll = try JSONDecoder().decode( + Command.self, from: Data(#"{"command":"scroll"}"#.utf8)) + let defaults = runnerDragCommandDefaults(defaultScroll) + XCTAssertEqual(defaults.durationMs, constants.defaultIosScrollDurationMs) + XCTAssertEqual(defaults.scrollAmount, constants.defaultIosScrollAmount) + let defaulted = try XCTUnwrap( + runnerScrollGesturePlan( + direction: .down, amount: nil, pixels: nil, referenceWidth: 1000, referenceHeight: 1000 + ) + ) + XCTAssertEqual(defaulted.travelPixels, 1000 * constants.defaultScrollAmount) + let saturated = try XCTUnwrap( + runnerScrollGesturePlan( + direction: .down, amount: 10, pixels: nil, referenceWidth: 1000, referenceHeight: 1000 + ) + ) + XCTAssertEqual( + saturated.travelPixels, 1000 - 2 * 1000 * constants.defaultEdgePaddingFraction) + } + + func testRunnerScrollAndDragCommandDefaultsStayDistinct() throws { + func command(_ json: String) throws -> Command { + try JSONDecoder().decode(Command.self, from: Data(json.utf8)) + } + + let pixelScroll = runnerDragCommandDefaults( + try command(#"{"command":"scroll","pixels":120}"#)) + XCTAssertNil(pixelScroll.scrollAmount) + XCTAssertEqual(pixelScroll.durationMs, 400) + + let drag = runnerDragCommandDefaults(try command(#"{"command":"drag"}"#)) + XCTAssertNil(drag.scrollAmount) + XCTAssertEqual(drag.durationMs, 250) + + let explicitScroll = runnerDragCommandDefaults( + try command(#"{"command":"scroll","amount":0.5,"durationMs":125}"#)) + XCTAssertEqual(explicitScroll.scrollAmount, 0.5) + XCTAssertEqual(explicitScroll.durationMs, 125) + + let explicitDrag = runnerDragCommandDefaults( + try command(#"{"command":"drag","durationMs":125}"#)) + XCTAssertEqual(explicitDrag.durationMs, 125) + } + + func testRunnerScrollReleaseBehaviorSelectsTheDragProfile() throws { + let constants = try loadScrollGestureFixture().constants + let controlled = try XCTUnwrap(ScrollReleaseBehavior(rawValue: constants.ordinaryScrollReleaseBehavior)) + let inertial = try XCTUnwrap(ScrollReleaseBehavior(rawValue: constants.edgeScrollReleaseBehavior)) + XCTAssertEqual( + scrollDragProfile(releaseBehavior: nil), + .controlledScroll + ) + XCTAssertEqual( + scrollDragProfile(releaseBehavior: controlled), + .controlledScroll + ) + XCTAssertEqual( + scrollDragProfile(releaseBehavior: inertial), + .fastSwipe + ) + } + + func testControlledScrollProfileUsesReliableCadenceAndMonotonicDeceleration() { + XCTAssertEqual(RunnerControlledScrollFrameCount(350), 21) + XCTAssertEqual(RunnerControlledScrollFrameCount(400), 24) + XCTAssertEqual(RunnerControlledScrollFrameCount(500), 30) + XCTAssertEqual(RunnerControlledScrollFrameCount(1_000), 30) + XCTAssertEqual(RunnerControlledScrollFrameCount(10_000), 30) + + let frameCount = RunnerControlledScrollFrameCount(400) + let progress = (0...frameCount).map { + RunnerControlledScrollProgress(Double($0) / Double(frameCount)) + } + let deltas = zip(progress.dropFirst(), progress).map { $0.0 - $0.1 } + XCTAssertEqual(progress.first, 0) + XCTAssertEqual(progress.last, 1) + XCTAssertTrue(zip(deltas, deltas.dropFirst()).allSatisfy { $0.1 <= $0.0 }) + + let iPhoneViewportPoints = 874.0 + let defaultFingerTravel = iPhoneViewportPoints * 0.65 + let finalSampleTravel = (1 - progress[progress.count - 2]) * defaultFingerTravel + XCTAssertLessThan(finalSampleTravel, 0.1) + } + + func testRunnerScrollGesturePlanRejectsUnknownDirection() { + XCTAssertNil(RunnerScrollDirection(rawValue: "sideways")) + } + + func testRunnerScrollGesturePlanRejectsInvalidAmountAndPixels() { + XCTAssertNil( + runnerScrollGesturePlan( + direction: .down, + amount: 0, + pixels: nil, + referenceWidth: 300, + referenceHeight: 600 + ) + ) + XCTAssertNil( + runnerScrollGesturePlan( + direction: .down, + amount: nil, + pixels: -10, + referenceWidth: 300, + referenceHeight: 600 + ) + ) + XCTAssertNil( + runnerScrollGesturePlan( + direction: .down, + amount: .infinity, + pixels: nil, + referenceWidth: 300, + referenceHeight: 600 + ) + ) + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScrollViewportPolicyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScrollViewportPolicyTests.swift new file mode 100644 index 0000000000..208397cc95 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScrollViewportPolicyTests.swift @@ -0,0 +1,165 @@ +import XCTest +import AgentDeviceSnapshotPresentation + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +private struct ScrollViewportPolicyFixture: Decodable { + struct Frame: Decodable { + let x: Double + let y: Double + let width: Double + let height: Double + + var cgRect: CGRect { + CGRect(x: x, y: y, width: width, height: height) + } + } + + struct Constants: Decodable { + let minVisibleFraction: Double + let accessoryAllowance: Double + } + + struct Expected: Decodable { + let kind: String + let viewport: Frame? + let keyboardMinY: Double? + let visibleHeight: Double? + } + + struct TestCase: Decodable { + let name: String + let viewport: Frame + let keyboard: Frame + let expected: Expected + } + + let constants: Constants + let cases: [TestCase] +} + +extension RunnerTests { + /// Golden parity table (#2500): every case in contracts/fixtures/scroll-keyboard-policy.json must + /// agree with the vitest twin. Add cases there, never fork the rule. + func testScrollViewportKeyboardClipMatchesGoldenParityTable() throws { + let fixture = try loadScrollViewportPolicyFixture() + XCTAssertFalse(fixture.cases.isEmpty, "parity table must not be empty") + for testCase in fixture.cases { + let clip = ScrollViewportPolicy.clip( + viewport: testCase.viewport.cgRect, + keyboard: testCase.keyboard.cgRect + ) + switch testCase.expected.kind { + case "unobstructed": + XCTAssertEqual(clip, .unobstructed, testCase.name) + case "avoided": + let expectedFrame = try XCTUnwrap(testCase.expected.viewport, testCase.name).cgRect + let expectedMinY = try XCTUnwrap(testCase.expected.keyboardMinY, testCase.name) + XCTAssertEqual( + clip, + .avoided(frame: expectedFrame, keyboardMinY: expectedMinY), + testCase.name + ) + case "occluded": + let expectedMinY = try XCTUnwrap(testCase.expected.keyboardMinY, testCase.name) + let expectedVisibleHeight = try XCTUnwrap(testCase.expected.visibleHeight, testCase.name) + XCTAssertEqual( + clip, + .occluded(keyboardMinY: expectedMinY, visibleHeight: expectedVisibleHeight), + testCase.name + ) + default: + XCTFail("unknown expected kind `\(testCase.expected.kind)` in \(testCase.name)") + } + } + } + + /// The thresholds are the table's, not RunnerScrollViewportPolicy.swift's. The refusal reason + /// and the runner code are each one side's own vocabulary: the reason is what the host publishes, + /// the code is what this runner answers with, and neither is a shared clip constant. + func testScrollViewportPolicyUsesParityTableConstants() throws { + let constants = try loadScrollViewportPolicyFixture().constants + XCTAssertEqual(constants.minVisibleFraction, ScrollViewportPolicy.minVisibleFraction) + XCTAssertEqual(constants.accessoryAllowance, ScrollViewportPolicy.accessoryAllowance) + } + + /// A clipped landscape band shortens the frame, and `CoordinateSpaceRotation.native(point:)` derives a + /// `landscapeRight` native x from the frame's HEIGHT. Rotating inside the band therefore moves the + /// dispatched path sideways by exactly what the keyboard took, off the lane the plan was built for, + /// so the plan band and the coordinate basis stay separate values through dispatch (#2500). + func testScrollViewportDispatchKeepsTheUnclippedFrameAsItsCoordinateRotationBasis() throws { + let viewport = CGRect(x: 0, y: 0, width: 1210, height: 834) + let keyboard = CGRect(x: 0, y: 588, width: 1210, height: 246) + let clip = ScrollViewportPolicy.clip(viewport: viewport, keyboard: keyboard) + guard case .avoided(let band, let keyboardMinY) = clip else { + return XCTFail("expected a landscape keyboard to be avoided, got \(clip)") + } + XCTAssertEqual(band.height, 576) + + guard case .gesture(let gesture) = ScrollViewportPolicy.frames( + referenceFrame: viewport, + clip: clip + ).gestureDispatch(direction: .up, amount: nil, pixels: nil) else { + return XCTFail("expected a gesture inside the clipped band") + } + XCTAssertEqual(gesture.planFrame, band) + XCTAssertEqual(gesture.keyboardMinY, keyboardMinY) + XCTAssertEqual(gesture.coordinateFrame, viewport, "the rotation basis must survive the clip") + XCTAssertLessThanOrEqual( + max(gesture.plan.y1, gesture.plan.y2), + keyboard.minY - ScrollViewportPolicy.accessoryAllowance, + "a landscape swipe must stay clear of the keys" + ) + + let reported = gesture.attachingEvidence( + to: Response( + ok: true, + data: DataPayload(referenceWidth: viewport.width, referenceHeight: viewport.height), + error: nil + ) + ) + XCTAssertEqual( + reported.data?.referenceHeight, + band.height, + "the payload names the band the plan ran inside, not the synthesis frame" + ) + XCTAssertEqual(reported.data?.referenceWidth, viewport.width) + XCTAssertEqual(reported.data?.keyboardMinY, keyboardMinY) + XCTAssertEqual(reported.data?.keyboardAvoided, true) + + let orientedStartY = gesture.planFrame.minY + gesture.plan.y1 + let dispatchedFromViewport = CoordinateSpaceRotation.native( + point: CGPoint(x: gesture.planFrame.minX + gesture.plan.x1, y: orientedStartY), + in: gesture.coordinateFrame, + interfaceOrientation: RunnerInterfaceOrientation.landscapeRight + ) + let dispatchedFromBand = CoordinateSpaceRotation.native( + point: CGPoint(x: gesture.planFrame.minX + gesture.plan.x1, y: orientedStartY), + in: gesture.planFrame, + interfaceOrientation: RunnerInterfaceOrientation.landscapeRight + ) + XCTAssertEqual( + dispatchedFromViewport.x - dispatchedFromBand.x, + viewport.height - band.height, + accuracy: 0.001, + "rotating inside the clipped band would shift native x by what the keyboard took" + ) + } + + private func loadScrollViewportPolicyFixture() throws -> ScrollViewportPolicyFixture { + let fixtureURL = URL(fileURLWithPath: #filePath) + .deletingLastPathComponent() // UnitTests + .deletingLastPathComponent() // AgentDeviceRunnerUITests + .deletingLastPathComponent() // AgentDeviceRunner + .deletingLastPathComponent() // runner + .deletingLastPathComponent() // apple + .deletingLastPathComponent() // repo root + .appendingPathComponent("contracts") + .appendingPathComponent("fixtures") + .appendingPathComponent("scroll-keyboard-policy.json") + return try JSONDecoder().decode( + ScrollViewportPolicyFixture.self, + from: Data(contentsOf: fixtureURL) + ) + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SequenceExecutionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SequenceExecutionTests.swift new file mode 100644 index 0000000000..a2e0327df6 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SequenceExecutionTests.swift @@ -0,0 +1,186 @@ +import XCTest + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +// MARK: - In-bundle unit tests (device-free) + +extension RunnerTests { + func testSequenceDecodesStepsFromWire() throws { + let json = """ + {"command":"sequence","commandId":"seq-1","steps":[ + {"kind":"tap","x":100,"y":200}, + {"kind":"doubleTap","x":101,"y":200}, + {"kind":"longPress","x":102,"y":200,"durationMs":300,"pauseMs":50} + ]} + """ + let command = try JSONDecoder().decode(Command.self, from: Data(json.utf8)) + XCTAssertEqual(command.command, .sequence) + XCTAssertEqual(command.steps?.count, 3) + XCTAssertEqual(command.steps?[0].kind, "tap") + XCTAssertEqual(command.steps?[1].kind, "doubleTap") + XCTAssertEqual(command.steps?[2].pauseMs, 50) + } + + func testSequenceAcceptsDoubleTapKind() { + // A doubleTap step missing coords must fail on the coords check, not the kind allowlist — + // proving "doubleTap" passes validateSequenceStep without needing a device to execute on. + let response = executeSequenceForTest(steps: [ + sequenceStep(kind: "doubleTap", x: nil) + ]) + XCTAssertEqual(response.ok, false) + XCTAssertEqual(response.error?.code, "INVALID_ARGS") + XCTAssertTrue(response.error?.message.contains("requires finite x and y") ?? false) + XCTAssertFalse(response.error?.message.contains("unsupported kind") ?? true) + } + + func testSequenceRejectsUnknownKind() throws { + let response = executeSequenceForTest(steps: [ + sequenceStep(kind: "tap", x: 1, y: 2), + sequenceStep(kind: "pinch", x: 3, y: 4), + ]) + XCTAssertEqual(response.ok, false) + XCTAssertEqual(response.error?.code, "INVALID_ARGS") + XCTAssertTrue(response.error?.message.contains("step 1") ?? false) + XCTAssertTrue(response.error?.message.contains("pinch") ?? false) + } + + func testSequenceRejectsEmpty() { + let response = executeSequenceForTest(steps: []) + XCTAssertEqual(response.ok, false) + XCTAssertEqual(response.error?.code, "INVALID_ARGS") + } + + func testSequenceRejectsTooManySteps() { + let steps = (0..<21).map { _ in sequenceStep(kind: "tap", x: 1, y: 2) } + let response = executeSequenceForTest(steps: steps) + XCTAssertEqual(response.ok, false) + XCTAssertEqual(response.error?.code, "INVALID_ARGS") + XCTAssertTrue(response.error?.message.contains("at most 20") ?? false) + } + + func testSequenceHasSynthesizedCoordinateStep() { + XCTAssertTrue( + sequenceHasSynthesizedCoordinateStep([ + sequenceStep(kind: "tap", x: 1, y: 2, synthesized: true), + ]) + ) + XCTAssertFalse( + sequenceHasSynthesizedCoordinateStep([ + sequenceStep(kind: "tap", x: 1, y: 2), + sequenceStep(kind: "doubleTap", x: 1, y: 2, synthesized: true), + ]) + ) + } + + func testAssembleSequencePreservesOrderOnSuccess() { + let steps = [ + sequenceStep(kind: "tap", x: 1, y: 1), + sequenceStep(kind: "longPress", x: 2, y: 2), + sequenceStep(kind: "tap", x: 3, y: 3), + ] + var calls: [Int] = [] + let execution = assembleSequenceExecution(steps: steps) { index, _ in + calls.append(index) + return SequenceStepOutcome( + outcome: .performed, + gestureStartUptimeMs: Double(index * 10), + gestureEndUptimeMs: Double(index * 10 + 5) + ) + } + XCTAssertEqual(calls, [0, 1, 2]) + XCTAssertEqual(execution.completedSteps, 3) + XCTAssertNil(execution.failedStepIndex) + XCTAssertEqual(execution.results.map { $0.kind }, ["tap", "longPress", "tap"]) + XCTAssertEqual(execution.gestureStartUptimeMs, 0) + XCTAssertEqual(execution.gestureEndUptimeMs, 25) + } + + func testAssembleSequenceStopsAtFirstFailure() { + let steps = [ + sequenceStep(kind: "tap", x: 1, y: 1), + sequenceStep(kind: "longPress", x: 2, y: 2), + sequenceStep(kind: "tap", x: 3, y: 3), + ] + var calls: [Int] = [] + let execution = assembleSequenceExecution(steps: steps) { index, _ in + calls.append(index) + if index == 1 { + return SequenceStepOutcome( + outcome: .unsupported(message: "long press unsupported", hint: nil), + gestureStartUptimeMs: 10, + gestureEndUptimeMs: 15 + ) + } + return SequenceStepOutcome(outcome: .performed, gestureStartUptimeMs: 0, gestureEndUptimeMs: 5) + } + // Step 2 is never invoked. + XCTAssertEqual(calls, [0, 1]) + XCTAssertEqual(execution.completedSteps, 1) + XCTAssertEqual(execution.failedStepIndex, 1) + // results.count == completedSteps + 1 (the failed step). + XCTAssertEqual(execution.results.count, 2) + XCTAssertEqual(execution.results[1].ok, false) + XCTAssertEqual(execution.results[1].errorCode, "UNSUPPORTED_OPERATION") + XCTAssertEqual(execution.results[1].errorMessage, "long press unsupported") + } + + func testSequenceWorstCaseResponseStaysUnderJournalCap() throws { + let longMessage = String(repeating: "e", count: 200) + let results = (0..<20).map { index in + SequenceStepResult( + ok: index < 19, + kind: "longPress", + errorCode: index < 19 ? nil : "UNSUPPORTED_OPERATION", + errorMessage: index < 19 ? nil : longMessage, + gestureStartUptimeMs: 123456.789, + gestureEndUptimeMs: 123466.789 + ) + } + let response = Response( + ok: true, + data: DataPayload( + message: "sequence", + completedSteps: 19, + failedStepIndex: 19, + sequenceResults: results + ) + ) + let encoded = try JSONEncoder().encode(response) + XCTAssertLessThan(encoded.count, 16 * 1024) + } + + private func sequenceStep( + kind: String, + x: Double?, + y: Double? = nil, + synthesized: Bool? = nil + ) -> SequenceStep { + SequenceStep( + kind: kind, + x: x, + y: y, + durationMs: nil, + pauseMs: nil, + synthesized: synthesized + ) + } + + /// Validation runs before any executor call, so the INVALID_ARGS paths are exercised without + /// reaching the device executor (which is never invoked when validation rejects). + private func executeSequenceForTest(steps: [SequenceStep]) -> Response { + let command = makeSequenceCommand(steps: steps) + return executeSequence(command: command, activeApp: app) + } + + /// Build a sequence Command via JSON so the test does not depend on the memberwise init's + /// parameter order. + private func makeSequenceCommand(steps: [SequenceStep]) -> Command { + struct SequenceCommandFixture: Encodable { + let command = "sequence" + let commandId = "seq-test" + let steps: [SequenceStep] + } + let data = try! JSONEncoder().encode(SequenceCommandFixture(steps: steps)) + return try! JSONDecoder().decode(Command.self, from: data) + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotBackendCapabilitiesTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotBackendCapabilitiesTests.swift new file mode 100644 index 0000000000..a56627eed9 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotBackendCapabilitiesTests.swift @@ -0,0 +1,80 @@ +import XCTest + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +private struct SnapshotBackendParityFixture: Decodable { + struct Availability: Decodable { + let simulator: Bool + let physicalDevice: Bool + } + + struct Backend: Decodable { + let name: String + let forceable: Bool + let supportsRawProjection: Bool + let regularDepth: String + let hittable: String + let availability: Availability + } + + let backends: [Backend] +} + +extension RunnerTests { + private func loadSnapshotBackendParityFixture() throws -> SnapshotBackendParityFixture { + let fixtureURL = URL(fileURLWithPath: #filePath) + .deletingLastPathComponent() // UnitTests + .deletingLastPathComponent() // AgentDeviceRunnerUITests + .deletingLastPathComponent() // AgentDeviceRunner + .deletingLastPathComponent() // runner + .deletingLastPathComponent() // apple + .deletingLastPathComponent() // repo root + .appendingPathComponent("contracts") + .appendingPathComponent("fixtures") + .appendingPathComponent("ios-snapshot-backends.json") + return try JSONDecoder().decode( + SnapshotBackendParityFixture.self, + from: Data(contentsOf: fixtureURL) + ) + } + + /// The JSON table is the cross-runtime declaration used by the TypeScript capability registry + /// and this runner. A backend case, forceability branch, projection/depth claim, or availability + /// change that is not classified in both implementations fails before an iOS smoke can drift. + func testSnapshotBackendDeclarationsMatchCapabilityFixture() throws { + let fixture = try loadSnapshotBackendParityFixture() + XCTAssertEqual( + fixture.backends.map(\.name), + SnapshotBackendKind.allCases.map(\.rawValue) + ) + + for expected in fixture.backends { + guard let backend = SnapshotBackendKind(rawValue: expected.name) else { + XCTFail("fixture contains an unknown snapshot backend: \(expected.name)") + continue + } + XCTAssertEqual(backend.isForceable, expected.forceable, expected.name) + XCTAssertEqual(backend.supportsRawProjection, expected.supportsRawProjection, expected.name) + XCTAssertEqual( + backend.regularDepthCapability.rawValue, + expected.regularDepth, + "regular depth capability: \(expected.name)" + ) + XCTAssertEqual( + backend.hittableSemantics, + expected.hittable, + "hittable semantics: \(expected.name)" + ) + XCTAssertEqual( + backend.isAvailable(on: .simulator), + expected.availability.simulator, + "simulator availability: \(expected.name)" + ) + XCTAssertEqual( + backend.isAvailable(on: .physicalDevice), + expected.availability.physicalDevice, + "physical-device availability: \(expected.name)" + ) + } + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanTests.swift new file mode 100644 index 0000000000..f0eae25e74 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanTests.swift @@ -0,0 +1,492 @@ +import XCTest +import AgentDeviceSnapshotPresentation + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +// MARK: - In-bundle unit tests + +extension RunnerTests { + private func planTestNode( + index: Int, + type: String, + label: String? = nil, + identifier: String? = nil, + hittable: Bool = false, + parentIndex: Int? = nil + ) -> PresentedNode { + SnapshotPresentation.singleElementRead( + RawAXNode( + index: index, + type: type, + label: label, + identifier: identifier, + value: nil, + rect: SnapshotRect(.zero), + enabled: true, + focused: nil, + selected: nil, + hittable: hittable, + depth: parentIndex == nil ? 0 : 1, + parentIndex: parentIndex, + hiddenContentAbove: nil, + hiddenContentBelow: nil + ) + ) + } + + func testSparsePayloadReasonMatrix() { + let root = planTestNode(index: 0, type: "Application", label: "Example App", hittable: true) + let window = planTestNode(index: 1, type: "Window", parentIndex: 0) + let button = planTestNode(index: 1, type: "Button", label: "Ok", hittable: true, parentIndex: 0) + let shell = planTestNode( + index: 1, + type: "Other", + identifier: "appShell", + parentIndex: 0 + ) + let serializationPlaceholder = planTestNode( + index: 2, + type: "Other", + label: "[object Object]", + parentIndex: 1 + ) + + // Labeled, hittable root over a bare window is still sparse. + XCTAssertNotNil(Self.sparsePayloadReason(DataPayload(nodes: [root, window], truncated: false))) + // Deadline-truncated near-empty sweep needs recovery even with one real control. + XCTAssertNotNil(Self.sparsePayloadReason(DataPayload(nodes: [root, button], truncated: true))) + // The same tiny tree from a completed sweep is a legitimately minimal screen. + XCTAssertNil(Self.sparsePayloadReason(DataPayload(nodes: [root, button], truncated: false))) + // Container metadata plus a stringified serialization placeholder is not readable UI. + XCTAssertNotNil( + Self.sparsePayloadReason( + DataPayload(nodes: [root, shell, serializationPlaceholder], truncated: false) + ) + ) + let actionableShell = planTestNode( + index: 1, + type: "Other", + identifier: "checkout", + hittable: true, + parentIndex: 0 + ) + XCTAssertNil( + Self.sparsePayloadReason(DataPayload(nodes: [root, actionableShell], truncated: false)) + ) + // Empty payloads are degraded. + XCTAssertNotNil(Self.sparsePayloadReason(DataPayload(nodes: [], truncated: false))) + } + + func testCollapsedLeafIndexesFlagsMergedContainersOnly() { + let root = planTestNode(index: 0, type: "Application", label: "App") + let merged = planTestNode( + index: 1, + type: "Other", + label: (0...30).map { "Row \($0), Tap" }.joined(separator: ", "), + parentIndex: 0 + ) + let prose = planTestNode( + index: 2, + type: "StaticText", + label: (0...30).map { "clause \($0)" }.joined(separator: ", "), + parentIndex: 0 + ) + XCTAssertEqual(Self.collapsedLeafIndexes([root, merged, prose]), [1]) + XCTAssertNil(Self.collapsedLeafIndexes([root, prose])) + } + + func testTerminalFailsClosedOnInteractiveAxFailureRegardlessOfSparseBest() { + // Interactive AX failure must invalidate + fail closed; a later tier's sparse synthetic-root + // "best" must never downgrade this to a returned-sparse payload (regression: best == nil guard). + XCTAssertEqual( + Self.resolveSnapshotPlanTerminal(terminal: .sparseWithFatalOnAXFailure, interactiveOnly: true), + .failClosed + ) + XCTAssertEqual( + Self.resolveSnapshotPlanTerminal(terminal: .sparseWithFatalOnAXFailure, interactiveOnly: false), + .sparseBest + ) + XCTAssertEqual( + Self.resolveSnapshotPlanTerminal(terminal: .throwOnAXFailure, interactiveOnly: true), + .throwAxFailure + ) + } + + func testXCTestChannelStateFirstFailureStampsDeferredCodeOnlyForDeferral() { + XCTAssertNil(Self.xcTestChannelStateFirstFailure(.normal)) + XCTAssertEqual(Self.xcTestChannelStateFirstFailure(.deferredToIndependentBackend)?.code, "deferred") + XCTAssertEqual(Self.xcTestChannelStateFirstFailure(.boundedXCTestProbe)?.code, "budget") + } + + func testSnapshotQualityCarriesPhaseTimingAtResponseLevel() { + let timing = SnapshotCaptureTiming(acquisitionMs: 12, presentationMs: 34) + let capture = SnapshotBackendCapture( + payload: DataPayload( + nodes: [planTestNode(index: 0, type: "Application", label: "App")], + truncated: false + ), + effectiveDepth: nil, + timing: timing + ) + + let payload = stampedSnapshotPayload( + capture, + backend: .recursiveTree, + state: "healthy", + reason: nil + ) + + XCTAssertEqual(payload.snapshotQuality?.timing, timing) + XCTAssertEqual(payload.nodes?.count, 1) + } + + func testStampedPayloadCarriesDisclosuresOnlyInTheVerdict() { + let root = planTestNode(index: 0, type: "Application", label: "App") + let merged = planTestNode( + index: 1, + type: "Other", + label: (0...30).map { "Tab \($0)" }.joined(separator: ", "), + parentIndex: 0 + ) + let coverage = SnapshotCustomActionCoverage( + read: 12, candidates: 19, truncated: 0, blocked: false) + let silent = stampedSnapshotPayload( + SnapshotBackendCapture( + payload: DataPayload(nodes: [root, merged], truncated: false), + effectiveDepth: nil, + customActions: coverage + ), + backend: .recursiveTree, + state: "healthy", + reason: nil + ) + XCTAssertNil(silent.message) + XCTAssertEqual(silent.snapshotQuality?.customActions, coverage) + XCTAssertEqual(silent.snapshotQuality?.collapsedLeafIndexes, [1]) + + let underlying = stampedSnapshotPayload( + SnapshotBackendCapture( + payload: DataPayload(message: "underlying", nodes: [root], truncated: false), + effectiveDepth: 4 + ), + backend: .privateAX, + state: "recovered", + reason: (reason: "tree capture timed out", code: "budget") + ) + XCTAssertEqual(underlying.message, "underlying") + } + + func testStampedPayloadTruncationTracksCompletenessNotRecoveryProvenance() { + let complete = SnapshotBackendCapture( + payload: DataPayload( + nodes: [ + planTestNode(index: 0, type: "Application", label: "App"), + planTestNode(index: 1, type: "Button", label: "Open", parentIndex: 0), + ], + truncated: false + ), + effectiveDepth: nil + ) + let deferred: (reason: String, code: String) = ( + "XCTest-backed snapshot tiers were deferred after recent slow accessibility work", "deferred" + ) + + // The CI signature behind `is absent ... capture was truncated`: a complete private AX + // tree selected while the XCTest channel is penalized is whole, and must say so. + let recovered = stampedSnapshotPayload( + complete, backend: .privateAX, state: "recovered", reason: deferred) + XCTAssertEqual(recovered.snapshotQuality?.state, "recovered") + XCTAssertEqual(recovered.truncated, false) + + let depthLimited = stampedSnapshotPayload( + SnapshotBackendCapture(payload: complete.payload, effectiveDepth: 56), + backend: .privateAX, state: "recovered", reason: deferred) + XCTAssertEqual(depthLimited.truncated, true) + + let cappedPayload = stampedSnapshotPayload( + SnapshotBackendCapture( + payload: DataPayload(nodes: complete.payload.nodes ?? [], truncated: true), + effectiveDepth: nil), + backend: .recursiveTree, state: "healthy", reason: nil) + XCTAssertEqual(cappedPayload.truncated, true) + + let sparse = stampedSnapshotPayload( + complete, backend: .querySweep, state: "sparse", + reason: ("snapshot returned no semantic controls or content", "sparse-tree")) + XCTAssertEqual(sparse.truncated, true) + } + + func testSnapshotQualityCarriesUnscopedQualityPayload() { + let quality = DataPayload( + nodes: [planTestNode(index: 0, type: "Application", label: "App")], + truncated: false + ) + let capture = SnapshotBackendCapture( + payload: quality, + effectiveDepth: nil, + qualityPayload: quality + ) + + let payload = stampedSnapshotPayload( + capture, + backend: .recursiveTree, + state: "healthy", + reason: nil + ) + + XCTAssertEqual(payload.qualityPayload?.nodes.count, 1) + XCTAssertEqual(payload.qualityPayload?.truncated, false) + XCTAssertNil(payload.qualityPayload?.scope) + } + + func testDirectPresentationDoesNotClaimPlanTiming() { + let options = PresentationOptions( + interactiveOnly: false, + depth: nil, + scope: nil, + raw: true + ) + let result = SnapshotPresentation.presentRaw( + SnapshotAcquisition( + hint: SnapshotPresentation.captureHint(for: options), + nodes: [], + truncated: false, + effectiveDepth: nil, + viewport: .infinite + ), + options: options + ) + let capture = Self.makeSnapshotBackendCapture(from: result) + + let payload = stampedSnapshotPayload( + capture, + backend: .recursiveTree, + state: "healthy", + reason: nil + ) + + XCTAssertNil(payload.snapshotQuality?.timing) + } + + /// The raw plan is derived from what each backend can actually serve, not from a second + /// hand-maintained list. Non-vacuity: flipping `querySweep.supportsRawProjection` to true adds it + /// to the plan and fails the first two assertions — which is exactly the shape of #1797 D4, a + /// `--raw` request answered by a backend that has no hierarchy to return. + func testRawDiagnosticPlanCarriesOnlyBackendsThatCanServeRaw() { + XCTAssertEqual(Self.rawDiagnosticPlan, [.recursiveTree, .privateAX]) + XCTAssertEqual( + SnapshotBackendKind.allCases.filter { !$0.supportsRawProjection }, [.querySweep]) + XCTAssertTrue(Self.rawDiagnosticPlan.allSatisfy(\.supportsRawProjection)) + // Tree-first error propagation is the raw plan's other contract (ADR 0004). + XCTAssertEqual(Self.rawDiagnosticPlan.first, .recursiveTree) + } + + /// A projection mismatch is a runner bug, not an accessibility failure: it must not take the + /// AX-failure terminal route (rethrow / fail-closed), just drop its tier with a named reason. + func testProjectionMismatchFailureIsStructuredAndNotAnAxFailure() { + let failure = Self.snapshotProjectionMismatchFailure( + .querySweep, requested: .raw, acquired: .regular) + XCTAssertEqual(failure.code, "IOS_SNAPSHOT_PROJECTION_MISMATCH") + XCTAssertTrue(failure.message.contains("queries")) + XCTAssertTrue(failure.message.contains("raw")) + XCTAssertFalse(Self.isAxSnapshotFailure(failure)) + } + + /// #1634 P2: the decoded wire field must reach capture options and its + /// applicable plan. A pinned REGULAR capture defers to privateAX-first; the + /// RAW diagnostic plan is never rerouted by the pin — raw keeps tree-first + /// error propagation, which is exactly why raw baselines are excluded from + /// corroboration daemon-side. + func testDecodedPreferredBackendReachesOptionsAndApplicablePlan() throws { + let json = #"{"command":"snapshot","preferredBackend":"private-ax"}"# + let command = try JSONDecoder().decode(Command.self, from: Data(json.utf8)) + let options = Self.presentationOptions(from: command) + XCTAssertEqual(options.preferredBackend, "private-ax") + XCTAssertFalse(options.raw) + + let treated = Self.snapshotXCTestChannelTreatedAsPenalized( + penalized: false, preferredBackend: options.preferredBackend) + let pinned = Self.effectiveSnapshotCapturePlan( + Self.regularVisiblePlan, + xCTestChannelPenalized: treated, + preferredBackend: options.preferredBackend + ) + XCTAssertEqual(pinned.plan, [.privateAX]) + XCTAssertEqual(pinned.xCTestChannelState, .deferredToIndependentBackend) + + let raw = Self.effectiveSnapshotCapturePlan( + Self.rawDiagnosticPlan, + xCTestChannelPenalized: treated, + preferredBackend: options.preferredBackend + ) + XCTAssertEqual(raw.plan, Self.rawDiagnosticPlan) + + // A command without the field decodes to no pin and a normal plan. + let bare = try JSONDecoder().decode( + Command.self, from: Data(#"{"command":"snapshot"}"#.utf8)) + XCTAssertNil(Self.presentationOptions(from: bare).preferredBackend) + } + + /// #1635: the force seam must select the recursive tree even when the XCTest + /// channel is currently penalized. Without the preferred-backend argument, + /// this call returns the independent private-AX recovery plan instead. + func testPreferredTreeBackendPinsRegularPlanAndLeavesStructuredEvidence() { + let forced = Self.effectiveSnapshotCapturePlan( + Self.regularVisiblePlan, + xCTestChannelPenalized: true, + preferredBackend: SnapshotBackendKind.recursiveTree.rawValue + ) + XCTAssertEqual(forced.plan, [.recursiveTree]) + XCTAssertEqual(forced.xCTestChannelState, .normal) + XCTAssertEqual( + Self.xcTestChannelStateFirstFailure( + forced.xCTestChannelState, + preferredBackend: forced.preferredBackend?.rawValue + )?.code, + "requested-backend" + ) + } + + /// Same-backend evidence probes: a daemon-pinned private-AX capture takes the + /// penalized route even with a healthy channel, so tap-outcome corroboration + /// baselines and probes are always captured by the same backend (backends are + /// never comparable views of a screen). Composed with the plan rule, the pin + /// yields the privateAX-first deferred plan. + func testPreferredPrivateAXBackendPlansAsPenalized() { + XCTAssertTrue( + Self.snapshotXCTestChannelTreatedAsPenalized(penalized: false, preferredBackend: "private-ax")) + XCTAssertTrue( + Self.snapshotXCTestChannelTreatedAsPenalized(penalized: true, preferredBackend: nil)) + XCTAssertFalse( + Self.snapshotXCTestChannelTreatedAsPenalized(penalized: false, preferredBackend: nil)) + XCTAssertFalse( + Self.snapshotXCTestChannelTreatedAsPenalized(penalized: false, preferredBackend: "tree")) + + let pinned = Self.effectiveSnapshotCapturePlan( + Self.regularVisiblePlan, + xCTestChannelPenalized: Self.snapshotXCTestChannelTreatedAsPenalized( + penalized: false, preferredBackend: "private-ax" + ), + preferredBackend: "private-ax" + ) + XCTAssertEqual(pinned.plan, [.privateAX]) + XCTAssertEqual(pinned.xCTestChannelState, .deferredToIndependentBackend) + } + + func testEffectiveSnapshotCapturePlanDefersXCTestBackedTiersOnlyWhenPenalizedRegularPlan() { + let regular = Self.effectiveSnapshotCapturePlan( + Self.regularVisiblePlan, + xCTestChannelPenalized: true + ) + XCTAssertEqual(regular.plan, [.privateAX]) + XCTAssertEqual(regular.xCTestChannelState, .deferredToIndependentBackend) + XCTAssertNil(regular.treeCaptureSliceBudgetOverride) + + let unpenalized = Self.effectiveSnapshotCapturePlan( + Self.regularVisiblePlan, + xCTestChannelPenalized: false + ) + XCTAssertEqual(unpenalized.plan, Self.regularVisiblePlan) + XCTAssertEqual(unpenalized.xCTestChannelState, .normal) + XCTAssertNil(unpenalized.treeCaptureSliceBudgetOverride) + + // The raw diagnostic plan preserves tree-first error propagation even under penalty. + let raw = Self.effectiveSnapshotCapturePlan( + Self.rawDiagnosticPlan, + xCTestChannelPenalized: true + ) + XCTAssertEqual(raw.plan, Self.rawDiagnosticPlan) + XCTAssertEqual(raw.xCTestChannelState, .normal) + XCTAssertNil(raw.treeCaptureSliceBudgetOverride) + } + + func testEffectiveSnapshotCapturePlanUsesBoundedXCTestProbeWhenNoIndependentBackendRuns() { + let physicalDevicePlan = Self.effectiveSnapshotCapturePlan( + Self.regularVisiblePlan, + xCTestChannelPenalized: true, + availableBackends: [.recursiveTree, .querySweep] + ) + + XCTAssertEqual(physicalDevicePlan.plan, [.recursiveTree, .querySweep]) + XCTAssertEqual(physicalDevicePlan.xCTestChannelState, .boundedXCTestProbe) + XCTAssertEqual( + physicalDevicePlan.treeCaptureSliceBudgetOverride, + Self.penalizedXCTestProbeTreeSliceBudget + ) + } + + func testSnapshotXCTestChannelPenaltyMatchesBundleAndExpires() { + defer { + snapshotXCTestChannelPenaltyBundleId = nil + snapshotXCTestChannelPenaltyUntil = .distantPast + } + + penalizeSnapshotXCTestChannel(bundleId: "xyz.blueskyweb.app", reason: "test") + XCTAssertTrue(isSnapshotXCTestChannelPenalized(bundleId: "xyz.blueskyweb.app")) + XCTAssertFalse(isSnapshotXCTestChannelPenalized(bundleId: "com.other.app")) + + // A penalty recorded without a bundle applies to any current target. + penalizeSnapshotXCTestChannel(bundleId: nil, reason: "test") + XCTAssertTrue(isSnapshotXCTestChannelPenalized(bundleId: "com.other.app")) + + // Expired penalties stop applying. + snapshotXCTestChannelPenaltyUntil = Date(timeIntervalSinceNow: -1) + XCTAssertFalse(isSnapshotXCTestChannelPenalized(bundleId: "com.other.app")) + } + + func testAbandonedMainThreadWorkSkipsOnlyXCTestBackedSnapshotTiers() { + abandonedMainThreadWorkCount = 1 + defer { abandonedMainThreadWorkCount = 0 } + + XCTAssertTrue(shouldSkipSnapshotBackendForAbandonedMainThreadWork(.recursiveTree)) + XCTAssertTrue(shouldSkipSnapshotBackendForAbandonedMainThreadWork(.querySweep)) + XCTAssertFalse(shouldSkipSnapshotBackendForAbandonedMainThreadWork(.privateAX)) + } + +#if os(iOS) + /// #2403: a plan pinned to private AX serves a regular `--depth` request through acquisition + /// and presentation. With a backend depth gate in `captureWithBackend`, private AX returns no + /// capture, the plan falls through to the synthetic sparse root, and the daemon rejects that + /// zero-rect root as a missing viewport. + func testPrivateAXPinnedRegularDepthReachesAcquisitionAndPresentation() throws { + app.launchArguments = ["--agent-device-selector-read-regression"] + app.launch() + currentApp = app + currentBundleId = nil + defer { + currentApp = nil + clearPrivateAXAcceptedDepth(reason: "test-cleanup") + app.terminate() + } + func capture(depth: Int?) throws -> DataPayload { + try runSnapshotCapturePlan( + Self.regularVisiblePlan, + app: app, + options: PresentationOptions( + interactiveOnly: false, + depth: depth, + scope: nil, + raw: false, + preferredBackend: SnapshotBackendKind.privateAX.rawValue + ), + terminal: .sparseWithFatalOnAXFailure + ) + } + + let capped = try capture(depth: 1) + + let quality = try XCTUnwrap(capped.snapshotQuality) + XCTAssertEqual(quality.backend, SnapshotBackendKind.privateAX.rawValue) + XCTAssertNotEqual(quality.state, "sparse") + let nodes = try XCTUnwrap(capped.nodes) + XCTAssertGreaterThan(nodes.count, 1) + XCTAssertEqual(nodes.map(\.depth).max(), 1) + XCTAssertNotEqual(nodes[0].rect, SnapshotRect(x: 0, y: 0, width: 0, height: 0)) + XCTAssertTrue(nodes.contains { $0.label == "Readable target" }) + + // The presented cut only ever narrows the unscoped capture from the same backend. + let unscoped = try XCTUnwrap(try capture(depth: nil).nodes) + XCTAssertLessThanOrEqual(nodes.count, unscoped.count) + } +#endif +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotTests.swift new file mode 100644 index 0000000000..527da828fb --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotTests.swift @@ -0,0 +1,245 @@ +import XCTest +import AgentDeviceSnapshotPresentation + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +extension RunnerTests { + func testSnapshotAccessibilityUnavailableMarksSparseSnapshotRunnerFatal() { + currentApp = app + currentBundleId = "com.example.app" + + let payload = snapshotAccessibilityUnavailable( + failure: SnapshotCaptureFailure( + code: Self.axSnapshotErrorCode, + message: Self.axSnapshotFailureMessage, + hint: Self.axSnapshotHint + ) + ) + + XCTAssertEqual(payload.message, "\(Self.axSnapshotFailureMessage) Hint: \(Self.axSnapshotHint)") + XCTAssertEqual(payload.nodes?.count, 1) + XCTAssertEqual(payload.nodes?.first?.type, "Application") + XCTAssertEqual(payload.truncated, true) + XCTAssertEqual(payload.runnerFatal, true) + XCTAssertEqual(payload.runnerFatalReason, Self.axSnapshotUnavailableReason) + // The planned terminal result carries the structured verdict like every other planned + // snapshot — downstream sparse handling keys off it, not off node shapes. + XCTAssertEqual(payload.snapshotQuality?.state, "sparse") + XCTAssertEqual(payload.snapshotQuality?.reasonCode, "ax-rejected") + XCTAssertEqual(payload.snapshotQuality?.reason, Self.axSnapshotFailureMessage) + XCTAssertNil(currentApp) + XCTAssertNil(currentBundleId) + } + + func testRecoveredSnapshotMessagePreservesHint() { + let message = recoveredSnapshotMessage( + SnapshotCaptureFailure( + code: Self.axSnapshotErrorCode, + message: Self.axSnapshotFailureMessage, + hint: Self.axSnapshotHint + ) + ) + + XCTAssertTrue(message.contains(Self.axSnapshotFailureMessage)) + XCTAssertTrue(message.contains(Self.axSnapshotHint)) + } + + func testRawSnapshotTooLargeFailureIsStructured() { + let failure = rawSnapshotTooLargeFailure(nodeCount: Self.rawSnapshotMaxNodes + 1) + + XCTAssertEqual(failure.code, Self.rawSnapshotTooLargeCode) + XCTAssertTrue(failure.message.contains("\(Self.rawSnapshotMaxNodes) nodes")) + XCTAssertEqual(failure.hint, Self.rawSnapshotTooLargeHint) + } + + func testSystemModalProbeSliceSharesAndClampsToPlanDeadline() { + // Fresh plan deadline: the probe gets its full dedicated budget. + XCTAssertEqual(Self.systemModalProbeSlice(budget: 4, deadlineRemaining: 20), 4) + // Nearly-spent plan deadline: the probe is clamped so it can't run past the shared budget. + XCTAssertEqual(Self.systemModalProbeSlice(budget: 4, deadlineRemaining: 1.5), 1.5) + // Exactly/already exhausted deadline: skip the probe entirely (0), never a negative timeout. + XCTAssertEqual(Self.systemModalProbeSlice(budget: 4, deadlineRemaining: 0), 0) + XCTAssertEqual(Self.systemModalProbeSlice(budget: 4, deadlineRemaining: -5), 0) + } + + // Simulator-only: the bounded probe body returns nil on macOS (no SpringBoard host), so the + // timeout/penalty/drain machinery below only exists on the iOS branch. +#if os(iOS) + /// Regression for #1244/#1248: drives the bounded system-modal probe through a real, + /// production-only command entry point (`snapshotFast` or `snapshotRaw` -- see the two test + /// methods below), not `boundedBlockingSystemAlertSnapshot` directly, with + /// `systemModalProbeOverrideForTesting` set to a closure that blocks past the probe's real + /// slice, forcing a real `runMainThreadWork` timeout. This is revert-sensitive on both halves + /// of the fix, for either entry point: + /// - if the entry point reverted to calling the unbounded `blockingSystemAlertSnapshot` + /// directly (or dropped the `runMainThreadWork` wrap), nothing here would ever time out, + /// so the mid-flight busy/penalty assertions below would never be met; + /// - if the `onAbandoned` penalty hook or the abandoned-work accounting were dropped, the + /// timeout would still fire, but the busy/penalty and drain assertions would not hold. + /// + /// The drain assertion is synchronized on the *real* release rather than raced: after + /// signaling the probe to finish, the background queue polls `hasAbandonedMainThreadWork()` + /// (bounded) and only then fulfills `drained`, which the test `wait(for:timeout:)`s on before + /// asserting `.idle`/`hasAbandonedMainThreadWork() == false` below -- so a slow drain fails that + /// assertion instead of racing a fixed-timing guess. + private func assertBoundedSystemModalProbeTimeoutRecoversThenReleasesOnDrain( + entryPointName: String, + callEntryPoint: @escaping (XCUIApplication, PresentationOptions) throws -> DataPayload + ) { + let targetBundleId = "com.callstack.agentdevice.runner.missing.snapshot-timeout-test" + let snapshotTarget = XCUIApplication(bundleIdentifier: targetBundleId) + let probeReleaseGate = DispatchSemaphore(value: 0) + currentApp = snapshotTarget + currentBundleId = targetBundleId + defer { + probeReleaseGate.signal() + currentApp = nil + currentBundleId = nil + systemModalProbeOverrideForTesting = nil + clearSnapshotXCTestChannelPenalty(reason: "test-cleanup") + } + + final class ResultBox { + var payload: DataPayload? + var wasBusyBeforeDrain = false + var hadAbandonedCaptureBeforeDrain = false + var wasPenalizedBeforeDrain = false + } + let box = ResultBox() + // The test owns release of the injected probe. A fixed timeout races the capture plan's + // independent fallback tiers on loaded CI hosts and can drain before the test records the + // abandoned-work state. The defer above still releases the probe if an earlier assertion or + // expectation fails. + systemModalProbeOverrideForTesting = { _ in + probeReleaseGate.wait() + return nil + } + + let completion = expectation( + description: "\(entryPointName) recovered while the probe was abandoned, then released it" + ) + let drained = expectation(description: "\(entryPointName) modal probe drained") + DispatchQueue(label: "agent-device.runner.tests.modal-probe-timeout").async { + box.payload = try? callEntryPoint( + snapshotTarget, + PresentationOptions(interactiveOnly: false, depth: nil, scope: nil, raw: false) + ) + + // 1) Penalty/busy accounting: must already be in place by the time the entry point + // returns, well before we release the still-blocked probe below. + if case .busy = self.currentMainThreadBusyState() { + box.wasBusyBeforeDrain = true + } + box.hadAbandonedCaptureBeforeDrain = self.hasAbandonedMainThreadWork() + box.wasPenalizedBeforeDrain = self.isSnapshotXCTestChannelPenalized(bundleId: self.currentBundleId) + + // 2) `box.payload` above was already produced -- through the capture plan's recovery + // tiers -- while the probe is still blocked on `probeReleaseGate`, i.e. recovered before + // drain, not queued behind it. + completion.fulfill() + + // 3) Only now let the abandoned probe finish, then block this queue (never the test's + // main-thread wait) on the *real* drain signal -- the abandoned-work count reaching zero + // -- bounded so a revert that never drains fulfills `drained` anyway and lets the + // assertions below report the regression explicitly instead of just timing out. + probeReleaseGate.signal() + let drainDeadline = Date().addingTimeInterval(5) + while self.hasAbandonedMainThreadWork(), Date() < drainDeadline { + self.sleepFor(0.002) + } + drained.fulfill() + } + + wait(for: [completion], timeout: 15) + + // 1) Penalty/busy accounting. + XCTAssertTrue( + box.wasBusyBeforeDrain, + "expected RUNNER_BUSY while the \(entryPointName) modal probe timeout is outstanding" + ) + XCTAssertTrue( + box.hadAbandonedCaptureBeforeDrain, + "onAbandoned must retain the abandoned XCTest channel work for \(entryPointName)" + ) + XCTAssertTrue( + box.wasPenalizedBeforeDrain, + "a timed-out modal probe must penalize the XCTest snapshot channel for \(entryPointName)" + ) + + // 2) Recovered response before drain. + XCTAssertNotNil( + box.payload, + "\(entryPointName) must recover a payload through the capture plan while the probe drains" + ) + + // 3) Bounded, deterministic drain barrier, then release assertions. + wait(for: [drained], timeout: 6) + guard case .idle = currentMainThreadBusyState() else { + return XCTFail("expected the runner to be idle once the abandoned \(entryPointName) probe drained") + } + XCTAssertFalse( + hasAbandonedMainThreadWork(), + "the drained probe must release the main thread for \(entryPointName)" + ) + } + + func testBoundedSystemModalProbeTimeoutRecoversThenReleasesOnDrain() { + assertBoundedSystemModalProbeTimeoutRecoversThenReleasesOnDrain(entryPointName: "snapshotFast") { + target, options in + try self.snapshotFast(app: target, options: options) + } + } + + func testBoundedSystemModalProbeTimeoutRecoversThenReleasesOnDrainForSnapshotRaw() { + assertBoundedSystemModalProbeTimeoutRecoversThenReleasesOnDrain(entryPointName: "snapshotRaw") { + target, options in + try self.snapshotRaw(app: target, options: options) + } + } +#endif + + func testDispatchRecoverySkipsBookkeepingWhileXCTestChannelOccupied() { + // The #1244 recovery shape: the modal probe abandoned an XCTest query that is still grinding on + // main, the capture recovered independently, and its response is ready. The recovery loop must + // return it without re-entering the main queue for recorded-failure/retry bookkeeping (that hop + // would block behind the abandoned query and re-stall the command), and a later command must + // still see the runner busy until the abandoned work drains. Removing the guard regresses this. + let command = try! JSONDecoder().decode( + Command.self, + from: Data(#"{"command":"snapshot","commandId":"recovery-guard"}"#.utf8) + ) + let recovered = Response(ok: false, error: .targetAppUnavailable(bundleId: nil)) + + setAbandonedMainThreadWork(1) + defer { setAbandonedMainThreadWork(0) } + guard case .busy = currentMainThreadBusyState() else { + return XCTFail("expected RUNNER_BUSY while abandoned XCTest work is outstanding") + } + + var occupiedCalls = 0 + let occupied = try! executeDispatchedWithRecovery(command: command) { + occupiedCalls += 1 + return recovered + } + XCTAssertEqual(occupiedCalls, 1, "recovered response must not retry behind abandoned XCTest work") + XCTAssertEqual(occupied.ok, false) + + setAbandonedMainThreadWork(0) + guard case .idle = currentMainThreadBusyState() else { + return XCTFail("runner should be idle once the abandoned work drained") + } + var drainedCalls = 0 + _ = try! executeDispatchedWithRecovery(command: command) { + drainedCalls += 1 + return recovered + } + XCTAssertEqual(drainedCalls, 2, "with the channel free the read-only retry runs once") + } + + private func setAbandonedMainThreadWork(_ count: Int) { + mainThreadWorkLock.lock() + abandonedMainThreadWorkCount = count + abandonedMainThreadWorkSince = count > 0 ? Date(timeIntervalSinceNow: -1) : nil + mainThreadWorkLock.unlock() + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotTimingTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotTimingTests.swift new file mode 100644 index 0000000000..24fdcfaa46 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotTimingTests.swift @@ -0,0 +1,78 @@ +import Foundation +import XCTest + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +extension RunnerTests { + func testXCTestPenaltyDecisionSeparatesAcquisitionAndPresentation() { + let slowPresentation = SnapshotBackendAttempt( + outcome: .noCapture, + timing: SnapshotCaptureTiming(acquisitionMs: 100, presentationMs: 4_000) + ) + XCTAssertNil( + Self.snapshotXCTestPenaltyReason( + kind: .recursiveTree, + attempt: slowPresentation, + slowThresholdMs: 3_000 + ) + ) + + let slowAcquisition = SnapshotBackendAttempt( + outcome: .noCapture, + timing: SnapshotCaptureTiming(acquisitionMs: 3_001, presentationMs: 100) + ) + XCTAssertEqual( + Self.snapshotXCTestPenaltyReason( + kind: .recursiveTree, + attempt: slowAcquisition, + slowThresholdMs: 3_000 + ), + "slow_tree_capture_3001ms" + ) + + let timeout = SnapshotCaptureFailure( + code: Self.xCTestSnapshotTimeoutCode, + message: "test timeout", + hint: "test" + ) + let acquisitionFailure = SnapshotBackendAttempt( + outcome: .failed(timeout, phase: .acquisition), + timing: SnapshotCaptureTiming(acquisitionMs: 100, presentationMs: 100) + ) + XCTAssertEqual( + Self.snapshotXCTestPenaltyReason( + kind: .recursiveTree, + attempt: acquisitionFailure, + slowThresholdMs: 3_000 + ), + "tree_backend_timeout" + ) + + let presentationFailure = SnapshotBackendAttempt( + outcome: .failed(timeout, phase: .presentation), + timing: SnapshotCaptureTiming(acquisitionMs: 100, presentationMs: 100) + ) + XCTAssertNil( + Self.snapshotXCTestPenaltyReason( + kind: .recursiveTree, + attempt: presentationFailure, + slowThresholdMs: 3_000 + ) + ) + } + + func testSnapshotPhaseTimerReportsAcquisitionAndPresentationSeparately() { + var now = Date(timeIntervalSinceReferenceDate: 100) + var timer = SnapshotPhaseTimer(now: { now }) + + _ = timer.measure(.acquisition) { + now = now.addingTimeInterval(2) + } + _ = timer.measure(.presentation) { + now = now.addingTimeInterval(5) + } + + XCTAssertEqual(timer.timing.acquisitionMs, 2_000, accuracy: 0.001) + XCTAssertEqual(timer.timing.presentationMs, 5_000, accuracy: 0.001) + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGesturePolicyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGesturePolicyTests.swift new file mode 100644 index 0000000000..acb3d9b032 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGesturePolicyTests.swift @@ -0,0 +1,105 @@ +import XCTest + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS) +extension RunnerTests { + func testSynthesizedGesturePolicyMarkerWritesOncePerKindUntilTheDecisionChanges() { + var written: [String] = [] + runnerMarkerWriter = { written.append($0) } + defer { + runnerMarkerWriter = { NSLog("%@", $0) } + invalidateCachedTarget(reason: "unit_test_cleanup") + } + logSynthesizedGesturePolicyDecision(kind: .coordinateTap, context: nil, fallbackAttempted: false) + logSynthesizedGesturePolicyDecision(kind: .coordinateTap, context: nil, fallbackAttempted: false) + XCTAssertEqual(written.count, 1, "a repeated decision writes no second line") + logSynthesizedGesturePolicyDecision(kind: .scroll, context: nil, fallbackAttempted: false) + XCTAssertEqual(written.count, 2, "each gesture kind states its own decision") + logSynthesizedGesturePolicyDecision(kind: .coordinateTap, context: nil, fallbackAttempted: true) + XCTAssertEqual(written.count, 3, "a changed decision writes a new line") + resetTargetBoundState() + logSynthesizedGesturePolicyDecision(kind: .coordinateTap, context: nil, fallbackAttempted: true) + XCTAssertEqual(written.count, 4, "a rebind states the same decision once more") + } +} +#endif + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +extension RunnerTests { + func testSynthesizedFallbackPolicyRequiresPrivateSynthesisForScrollWhenAxUnavailableOrUnknown() { + XCTAssertFalse( + SynthesizedFallbackPolicy.privateSynthesisRequired + .allowsXCTestCoordinateFallback(accessibilityHealth: .unavailable) + ) + XCTAssertFalse( + SynthesizedFallbackPolicy.privateSynthesisRequired + .allowsXCTestCoordinateFallback(accessibilityHealth: .unknown) + ) + XCTAssertFalse( + SynthesizedFallbackPolicy.privateSynthesisRequired + .allowsXCTestCoordinateFallback(accessibilityHealth: .healthy) + ) + } + + func testSynthesizedDragCoordinateFallbackAllowsUnknownButNotUnavailableAccessibility() { + XCTAssertTrue( + SynthesizedFallbackPolicy.xctestCoordinateWhenAccessibilityAvailable + .allowsXCTestCoordinateFallback(accessibilityHealth: .healthy) + ) + XCTAssertFalse( + SynthesizedFallbackPolicy.xctestCoordinateWhenAccessibilityAvailable + .allowsXCTestCoordinateFallback(accessibilityHealth: .unavailable) + ) + XCTAssertTrue( + SynthesizedFallbackPolicy.xctestCoordinateWhenAccessibilityAvailable + .allowsXCTestCoordinateFallback(accessibilityHealth: .unknown) + ) + } + + /// Keyboard-policy semantics only. Which command gets which policy is the table below; a probe + /// that is merely permitted still costs a live AX fetch, so the two questions stay separate. + func testSynthesizedKeyboardPolicyAllowsProbeOnlyWhenAccessibilityPermitsIt() { + XCTAssertFalse( + SynthesizedKeyboardPolicy.whenAccessibilityHealthy + .allowsProbe(accessibilityHealth: .unknown) + ) + XCTAssertTrue( + SynthesizedKeyboardPolicy.requiredWhenAvailable + .allowsProbe(accessibilityHealth: .unknown) + ) + XCTAssertFalse( + SynthesizedKeyboardPolicy.requiredWhenAvailable + .allowsProbe(accessibilityHealth: .unavailable) + ) + } + + func testSynthesizedGesturePoliciesMatchCommandContracts() { + XCTAssertEqual( + synthesizedGesturePolicy(.coordinateTap), + SynthesizedGesturePolicy( + keyboardPolicy: .never, + fallbackPolicy: .xctestCoordinateAllowed + ) + ) + XCTAssertEqual( + synthesizedGesturePolicy(.scroll), + SynthesizedGesturePolicy( + keyboardPolicy: .requiredWhenAvailable, + fallbackPolicy: .privateSynthesisRequired + ) + ) + XCTAssertEqual( + synthesizedGesturePolicy(.synthesizedDrag), + SynthesizedGesturePolicy( + keyboardPolicy: .requiredWhenAvailable, + fallbackPolicy: .xctestCoordinateWhenAccessibilityAvailable + ) + ) + } + + func testCoordinateTapTextInputProbeSkipsPenalizedXCTestChannel() { + XCTAssertTrue(shouldProbeCoordinateTapTextInput(xCTestChannelPenalized: false)) + XCTAssertFalse(shouldProbeCoordinateTapTextInput(xCTestChannelPenalized: true)) + } + +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SystemSurfaceHostPolicyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SystemSurfaceHostPolicyTests.swift new file mode 100644 index 0000000000..e16aac6e3b --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SystemSurfaceHostPolicyTests.swift @@ -0,0 +1,50 @@ +import Foundation + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +import XCTest + +private struct SystemSurfaceHostFixture: Decodable { + struct Host: Decodable { + let bundleId: String + let kind: String + } + let hosts: [Host] +} + +extension RunnerTests { + func testSystemSurfaceHostRegistryMirrorsGoldenFixture() throws { + let fixtureURL = URL(fileURLWithPath: #filePath) + .deletingLastPathComponent() // UnitTests + .deletingLastPathComponent() // AgentDeviceRunnerUITests + .deletingLastPathComponent() // AgentDeviceRunner + .deletingLastPathComponent() // runner + .deletingLastPathComponent() // apple + .deletingLastPathComponent() // repo root + .appendingPathComponent("contracts") + .appendingPathComponent("fixtures") + .appendingPathComponent("ios-system-surface-hosts.json") + let fixture = try JSONDecoder().decode( + SystemSurfaceHostFixture.self, + from: Data(contentsOf: fixtureURL) + ) + let registry = SystemSurfaceHostRegistry.hosts.map { [$0.bundleId, $0.kind.rawValue] } + let golden = fixture.hosts.map { [$0.bundleId, $0.kind] } + XCTAssertEqual(registry, golden, "SystemSurfaceHostRegistry drifted from the golden fixture") + } + + func testSystemSurfaceHostRegistryRecognizesRegisteredHosts() { + XCTAssertTrue(SystemSurfaceHostRegistry.isSystemSurfaceHost("com.apple.SafariViewService")) + XCTAssertTrue(SystemSurfaceHostRegistry.isSystemSurfaceHost("com.apple.PassbookUIService")) + XCTAssertFalse(SystemSurfaceHostRegistry.isSystemSurfaceHost("com.example.app")) + XCTAssertFalse(SystemSurfaceHostRegistry.isSystemSurfaceHost(nil)) + XCTAssertEqual( + SystemSurfaceHostRegistry.host(forBundleId: "com.apple.SafariViewService")?.kind, + .webAuth + ) + XCTAssertEqual( + SystemSurfaceHostRegistry.host(forBundleId: "com.apple.PassbookUIService")?.kind, + .payment + ) + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TapPointPolicyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TapPointPolicyTests.swift new file mode 100644 index 0000000000..791b1b2530 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TapPointPolicyTests.swift @@ -0,0 +1,52 @@ +import XCTest + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +private struct TapPointPolicyFixture: Decodable { + struct Frame: Decodable { + let x: Double + let y: Double + let width: Double + let height: Double + + var cgRect: CGRect { + CGRect(x: x, y: y, width: width, height: height) + } + } + + let name: String + let elementFrame: Frame + let windowFrame: Frame + let allowed: Bool +} + +extension RunnerTests { + // Golden parity table (ADR 0011 Layer 2): every case in + // contracts/fixtures/tap-point-policy.json must agree with the vitest twin + // (tap-point-policy-parity.test.ts). Add cases there, never fork the rule. + func testTapPointPolicyMatchesGoldenParityTable() throws { + let fixtureURL = URL(fileURLWithPath: #filePath) + .deletingLastPathComponent() // UnitTests + .deletingLastPathComponent() // AgentDeviceRunnerUITests + .deletingLastPathComponent() // AgentDeviceRunner + .deletingLastPathComponent() // runner + .deletingLastPathComponent() // apple + .deletingLastPathComponent() // repo root + .appendingPathComponent("contracts") + .appendingPathComponent("fixtures") + .appendingPathComponent("tap-point-policy.json") + let data = try Data(contentsOf: fixtureURL) + let cases = try JSONDecoder().decode([TapPointPolicyFixture].self, from: data) + XCTAssertFalse(cases.isEmpty, "parity table must not be empty") + for fixture in cases { + XCTAssertEqual( + TapPointPolicy.isAllowed( + elementFrame: fixture.elementFrame.cgRect, + windowFrame: fixture.windowFrame.cgRect + ), + fixture.allowed, + fixture.name + ) + } + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextInputCandidatePolicyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextInputCandidatePolicyTests.swift new file mode 100644 index 0000000000..ec3201275b --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextInputCandidatePolicyTests.swift @@ -0,0 +1,33 @@ +import CoreGraphics +import XCTest + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +extension RunnerTests { + func testCoordinateTextInputCandidateMustBeEnabledAndContainTheTouchPoint() { + let frame = CGRect(x: 10, y: 20, width: 100, height: 40) + let point = CGPoint(x: 50, y: 40) + + XCTAssertTrue( + isCoordinateTextInputCandidate( + enabled: true, + frame: frame, + point: point + ) + ) + XCTAssertFalse( + isCoordinateTextInputCandidate( + enabled: false, + frame: frame, + point: point + ) + ) + XCTAssertFalse( + isCoordinateTextInputCandidate( + enabled: true, + frame: frame, + point: CGPoint(x: 200, y: 200) + ) + ) + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TransportTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TransportTests.swift new file mode 100644 index 0000000000..42c0f53875 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TransportTests.swift @@ -0,0 +1,43 @@ +import XCTest +import Network + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +extension RunnerTests { + func testDuplicateCommandIdCoalescesOntoInFlightExecution() throws { + let command = try JSONDecoder().decode( + Command.self, + from: Data(#"{"command":"snapshot","commandId":"snapshot-coalesce"}"#.utf8) + ) + var primaryData: Data? + var waiterData: Data? + defer { + inFlightCommandIds.removeAll() + inFlightCommandWaiters.removeAll() + } + + XCTAssertFalse( + attachToInFlightCommandIfNeeded(command: command) { result in + primaryData = result.data + } + ) + XCTAssertTrue( + attachToInFlightCommandIfNeeded(command: command) { result in + waiterData = result.data + } + ) + + let delivered = Data("single-result".utf8) + deliverCommandResult( + command: command, + result: (delivered, false) + ) { result in + primaryData = result.data + } + + XCTAssertEqual(primaryData, delivered) + XCTAssertEqual(waiterData, delivered) + XCTAssertFalse(inFlightCommandIds.contains("snapshot-coalesce")) + XCTAssertNil(inFlightCommandWaiters["snapshot-coalesce"]) + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TvRemoteTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TvRemoteTests.swift new file mode 100644 index 0000000000..3cdcc89651 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TvRemoteTests.swift @@ -0,0 +1,25 @@ +import XCTest + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +extension RunnerTests { + func testTvRemoteButtonMappingAcceptsSupportedNamesAndRejectsUnknown() { + let supported = [ + ("select", "select"), + ("SELECT", "select"), + ("menu", "menu"), + ("home", "home"), + ("up", "up"), + ("down", "down"), + ("left", "left"), + ("right", "right"), + ] + for (raw, expected) in supported { + XCTAssertEqual(tvRemoteButton(from: raw)?.rawValue, expected) + } + + for raw in [String?(nil), "", "volumeUp", "select "] { + XCTAssertNil(tvRemoteButton(from: raw)) + } + } +} +#endif diff --git a/apple/runner/README.md b/apple/runner/README.md index 4a657b6ce8..d899f9cfce 100644 --- a/apple/runner/README.md +++ b/apple/runner/README.md @@ -33,6 +33,8 @@ Protocol and maintenance references: - `RunnerTests+Snapshot.swift`: fast/raw snapshot builders and include/filter helpers. - `RunnerTests+SystemModal.swift`: SpringBoard/system modal detection and modal snapshot shaping. - `RunnerTests+ScreenRecorder.swift`: nested `ScreenRecorder` implementation. +- `UnitTests/RunnerTests+Tests.swift`: the `AGENT_DEVICE_RUNNER_UNIT_TESTS` tests for each + source file. The packaged runner source omits this directory. ## Snapshot Strategy diff --git a/packages/contracts/src/scroll-gesture.test.ts b/packages/contracts/src/scroll-gesture.test.ts index 1c5f700560..de068535ad 100644 --- a/packages/contracts/src/scroll-gesture.test.ts +++ b/packages/contracts/src/scroll-gesture.test.ts @@ -60,8 +60,8 @@ test('buildInPageSwipeGesturePlan truncates percentage coordinates on odd viewpo // Cross-language parity table: every case in contracts/fixtures/scroll-gesture.json is asserted // here AND by the Swift port (runnerScrollGesturePlan in RunnerTests+ScrollGesture.swift, gated -// XCTest in the same file). Add vectors to the table, never to one suite — drift on either side -// turns CI red without a simulator. +// XCTest in UnitTests/RunnerTests+ScrollGestureTests.swift). Add vectors to the table, never to +// one suite — drift on either side turns CI red without a simulator. type ScrollGestureFixture = { constants: { defaultIosScrollAmount: number; @@ -210,7 +210,8 @@ test('clampGestureCoordinate returns the lower bound for non-finite coordinates' // Golden parity table: the SAME JSON is asserted against the Swift twin // (ScrollViewportPolicy in apple/runner/AgentDeviceRunner/ -// AgentDeviceRunnerUITests/RunnerScrollViewportPolicy.swift, gated XCTest in the same file), so +// AgentDeviceRunnerUITests/RunnerScrollViewportPolicy.swift, gated XCTest in +// UnitTests/RunnerTests+ScrollViewportPolicyTests.swift), so // a clip that drifts between the iOS runner and the Android/TS owner turns CI red on whichever // side changed, without a simulator. diff --git a/packages/contracts/src/snapshot-tap-point-policy.test.ts b/packages/contracts/src/snapshot-tap-point-policy.test.ts index 9c93995319..526759a786 100644 --- a/packages/contracts/src/snapshot-tap-point-policy.test.ts +++ b/packages/contracts/src/snapshot-tap-point-policy.test.ts @@ -7,9 +7,10 @@ import type { Rect } from '@agent-device/kernel/snapshot'; // ADR 0011 Layer 2 golden parity table: the SAME JSON is asserted against the // Swift twin (TapPointPolicy in apple/runner/AgentDeviceRunner/ -// AgentDeviceRunnerUITests/RunnerTapPointPolicy.swift, gated XCTest in the -// same file), so drift between the runner's ELEMENT_OFFSCREEN guard and the -// runtime's offscreen rule turns CI red on whichever side changed. +// AgentDeviceRunnerUITests/RunnerTapPointPolicy.swift, gated XCTest in +// UnitTests/RunnerTests+TapPointPolicyTests.swift), so drift between the +// runner's ELEMENT_OFFSCREEN guard and the runtime's offscreen rule turns CI +// red on whichever side changed. // // Scope: the table proves the GEOMETRIC rule only — element-frame center // inside the window frame, edges inclusive, empty frame fails open. The diff --git a/scripts/__tests__/xctest-declarations.test.ts b/scripts/__tests__/xctest-declarations.test.ts index a56e7eb4ed..70edbdd68b 100644 --- a/scripts/__tests__/xctest-declarations.test.ts +++ b/scripts/__tests__/xctest-declarations.test.ts @@ -55,8 +55,8 @@ describe('the declaration scan', () => { }); test('reads a file whose name does not start with RunnerTests', () => { - // RunnerTapPointPolicy.swift is the real instance: the synchronized-root-group project - // compiles every .swift in the directory, so file naming carries no membership meaning. + // The synchronized-root-group project compiles every .swift in the directory, so file + // naming carries no membership meaning. expect( parseDeclaredTests(TARGET, [ { diff --git a/scripts/__tests__/xctest-selection.test.ts b/scripts/__tests__/xctest-selection.test.ts index 177b5726e0..20f282fdea 100644 --- a/scripts/__tests__/xctest-selection.test.ts +++ b/scripts/__tests__/xctest-selection.test.ts @@ -113,7 +113,7 @@ describe('the real tree', () => { // own regex, because the Xcode project uses a PBXFileSystemSynchronizedRootGroup — every // .swift file in it is a member. Reusing the check's own file filter would make this // tautological, and a name-based filter is exactly the bug it caught - // (RunnerTapPointPolicy.swift declares a test and does not start with "RunnerTests"). + // (RunnerTapPointPolicy.swift declared a test and does not start with "RunnerTests"). const directory = path.join(repoRoot, RUNNER_TESTS_DIR); const countAddressableMethods = (sourceDirectory: string): number => fs.readdirSync(sourceDirectory, { withFileTypes: true }).reduce((total, entry) => { diff --git a/scripts/xctest-declarations.ts b/scripts/xctest-declarations.ts index 0ab4caf1d3..59ef490648 100644 --- a/scripts/xctest-declarations.ts +++ b/scripts/xctest-declarations.ts @@ -14,9 +14,8 @@ import { activeSource, PLATFORMS, type Platform } from './swift-conditional-comp export const RUNNER_TESTS_DIR = 'apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests'; // Every .swift file below the target directory is a member: the Xcode project uses a -// PBXFileSystemSynchronizedRootGroup, so membership is the directory, not a file list. A -// `RunnerTests*` name filter would miss RunnerTapPointPolicy.swift, which declares a real -// addressable test inside `extension RunnerTests`. +// PBXFileSystemSynchronizedRootGroup, so membership is the directory, not a file list, and a +// file's name says nothing about whether it declares addressable tests. const SWIFT_SOURCE = /\.swift$/; // One ordered pass over the source. A column-0 type declaration moves the enclosing type;