From d97e915077f272e0bda65b386441d0b49a044c35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 23 Sep 2026 17:38:26 +0200 Subject: [PATCH 1/2] refactor(ios-runner): key the runner's read-only retry on typed failures, not message text The response retry now reads a runner-internal `retryableFailure` that the target-app-unavailable responses carry from the place they are produced; it is excluded from encoding so the host sees the same error JSON. The exception retry drops the main-thread-timeout branch (that text is only ever a Swift NSError, never an ObjC exception reason) and the snapshot timeout branch (snapshot dispatches through its own recovery loop and never reaches the exception catcher). Co-Authored-By: Claude Opus 5.5 (1M context) --- .../RunnerTests+CommandExecution.swift | 11 +++-- .../RunnerTests+Lifecycle.swift | 16 ++----- .../RunnerTests+Models.swift | 22 ++++++++++ .../RunnerTests+Snapshot.swift | 2 +- .../RunnerTests+LifecycleTests.swift | 42 +++++++++++++++++++ 5 files changed, 73 insertions(+), 20 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index 1f8477482b..6be914e7cc 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -1576,10 +1576,10 @@ extension RunnerTests { if let bundleId = requestedBundleId { activeApp = activateTarget(bundleId: bundleId, reason: "missing_after_wait") guard activeApp.waitForExistence(timeout: appExistenceTimeout) else { - return .response(Response(ok: false, error: ErrorPayload(message: "app '\(bundleId)' is not available"))) + return .response(Response(ok: false, error: .targetAppUnavailable(bundleId: bundleId))) } } else { - return .response(Response(ok: false, error: ErrorPayload(message: "runner app is not available"))) + return .response(Response(ok: false, error: .targetAppUnavailable(bundleId: nil))) } } @@ -1595,10 +1595,9 @@ extension RunnerTests { requestedBundleId: requestedBundleId ) if !skipInteractionExistenceWait && !activeApp.waitForExistence(timeout: 2) { - if let bundleId = requestedBundleId { - return .response(Response(ok: false, error: ErrorPayload(message: "app '\(bundleId)' is not available"))) - } - return .response(Response(ok: false, error: ErrorPayload(message: "runner app is not available"))) + return .response( + Response(ok: false, error: .targetAppUnavailable(bundleId: requestedBundleId)) + ) } applyInteractionStabilizationIfNeeded() } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift index 57bde60d26..57e4467cd3 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift @@ -431,17 +431,8 @@ extension RunnerTests { func shouldRetryException(_ command: Command, message: String) -> Bool { guard shouldRetryCommand(command) else { return false } - let normalized = message.lowercased() - if normalized.contains("kaxerrorservernotfound") { - return true - } - if normalized.contains("main thread execution timed out") { - return true - } - if normalized.contains("timed out") && command.command == .snapshot { - return true - } - return false + // XCTest raises this AX error as an ObjC exception whose reason is the only handle on it. + return message.lowercased().contains("kaxerrorservernotfound") } // MARK: - Command Classification @@ -460,8 +451,7 @@ extension RunnerTests { func shouldRetryResponse(_ response: Response) -> Bool { guard response.ok == false else { return false } - guard let message = response.error?.message.lowercased() else { return false } - return message.contains("is not available") + return response.error?.retryableFailure != nil } func isInteractionCommand(_ command: CommandType) -> Bool { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift index 4ddd306b16..2a4b84de27 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift @@ -385,8 +385,30 @@ struct SnapshotQualityPayload: Codable { } } +/// A runner failure the read-only retry may recover from by re-resolving the target. +enum RetryableResponseFailure: Equatable { + case targetAppUnavailable +} + struct ErrorPayload: Codable { var code: String? let message: String var hint: String? + /// Runner-internal: read by `shouldRetryResponse` and never encoded, so the host's decoding of + /// the error is unchanged. + var retryableFailure: RetryableResponseFailure? = nil + + private enum CodingKeys: String, CodingKey { + case code + case message + case hint + } + + static func targetAppUnavailable(bundleId: String?) -> ErrorPayload { + let subject = bundleId.map { "app '\($0)'" } ?? "runner app" + return ErrorPayload( + message: "\(subject) is not available", + retryableFailure: .targetAppUnavailable + ) + } } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift index 04e83a38f3..83686731cf 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift @@ -764,7 +764,7 @@ extension RunnerTests { Command.self, from: Data(#"{"command":"snapshot","commandId":"recovery-guard"}"#.utf8) ) - let recovered = Response(ok: false, error: ErrorPayload(message: "target is not available")) + let recovered = Response(ok: false, error: .targetAppUnavailable(bundleId: nil)) setAbandonedMainThreadWork(1) defer { setAbandonedMainThreadWork(0) } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LifecycleTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LifecycleTests.swift index c9c14b436d..cae9e27592 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LifecycleTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LifecycleTests.swift @@ -28,6 +28,48 @@ extension RunnerTests { } } + func testTargetAppUnavailableResponseIsRetried() { + for bundleId in ["com.example.app", nil] as [String?] { + let response = Response(ok: false, error: .targetAppUnavailable(bundleId: bundleId)) + XCTAssertTrue(shouldRetryResponse(response), String(describing: bundleId)) + } + let reworded = ErrorPayload(message: "target vanished", retryableFailure: .targetAppUnavailable) + XCTAssertTrue(shouldRetryResponse(Response(ok: false, error: reworded))) + } + + func testUntypedUnavailableMessageIsNotRetried() { + for message in ["app 'com.example.app' is not available", "runner app is not available"] { + let response = Response(ok: false, error: ErrorPayload(message: message)) + XCTAssertFalse(shouldRetryResponse(response), message) + } + let succeeded = Response(ok: true, error: .targetAppUnavailable(bundleId: nil)) + XCTAssertFalse(shouldRetryResponse(succeeded)) + } + + func testTargetAppUnavailableErrorKeepsItsWireShape() throws { + let encoded = try JSONEncoder().encode(ErrorPayload.targetAppUnavailable(bundleId: "com.example.app")) + let object = try XCTUnwrap(JSONSerialization.jsonObject(with: encoded) as? [String: Any]) + XCTAssertEqual(Array(object.keys), ["message"]) + XCTAssertEqual(object["message"] as? String, "app 'com.example.app' is not available") + XCTAssertEqual( + ErrorPayload.targetAppUnavailable(bundleId: nil).message, + "runner app is not available" + ) + } + + func testExceptionRetryIsLimitedToTheAxServerNotFoundReadOnlyCase() throws { + let readText = try JSONDecoder().decode(Command.self, from: Data(#"{"command":"readText"}"#.utf8)) + let snapshot = try JSONDecoder().decode(Command.self, from: Data(#"{"command":"snapshot"}"#.utf8)) + let tap = try JSONDecoder().decode(Command.self, from: Data(#"{"command":"tap"}"#.utf8)) + let axServerNotFound = "NSException: Error kAXErrorServerNotFound" + XCTAssertTrue(shouldRetryException(readText, message: axServerNotFound)) + XCTAssertFalse(shouldRetryException(tap, message: axServerNotFound)) + XCTAssertFalse( + shouldRetryException(readText, message: "NSException: main thread execution timed out") + ) + XCTAssertFalse(shouldRetryException(snapshot, message: "NSException: query timed out")) + } + func testInlineScreenshotResponseKeepsDisplayFactsBesideTheImage() throws { let pngData = Data([0x89, 0x50, 0x4E, 0x47]) let response = screenshotResponse( From 2d7ec919a3bbee6032d547c7f9bc53ec28559efa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 23 Sep 2026 17:45:01 +0200 Subject: [PATCH 2/2] test(ios-runner): pin the error payload wire shape beside its model Moves the wire-shape pin to a Models test file and adds a field-drift check: every stored ErrorPayload field except the runner-internal retryable failure must reach the encoded JSON. The exception-retry test builds its commands through the shared fixture helper. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../RunnerTests+LifecycleTests.swift | 17 ++-------- .../UnitTests/RunnerTests+ModelsTests.swift | 32 +++++++++++++++++++ 2 files changed, 35 insertions(+), 14 deletions(-) create mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ModelsTests.swift diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LifecycleTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LifecycleTests.swift index cae9e27592..cba532f03b 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LifecycleTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LifecycleTests.swift @@ -46,21 +46,10 @@ extension RunnerTests { XCTAssertFalse(shouldRetryResponse(succeeded)) } - func testTargetAppUnavailableErrorKeepsItsWireShape() throws { - let encoded = try JSONEncoder().encode(ErrorPayload.targetAppUnavailable(bundleId: "com.example.app")) - let object = try XCTUnwrap(JSONSerialization.jsonObject(with: encoded) as? [String: Any]) - XCTAssertEqual(Array(object.keys), ["message"]) - XCTAssertEqual(object["message"] as? String, "app 'com.example.app' is not available") - XCTAssertEqual( - ErrorPayload.targetAppUnavailable(bundleId: nil).message, - "runner app is not available" - ) - } - func testExceptionRetryIsLimitedToTheAxServerNotFoundReadOnlyCase() throws { - let readText = try JSONDecoder().decode(Command.self, from: Data(#"{"command":"readText"}"#.utf8)) - let snapshot = try JSONDecoder().decode(Command.self, from: Data(#"{"command":"snapshot"}"#.utf8)) - let tap = try JSONDecoder().decode(Command.self, from: Data(#"{"command":"tap"}"#.utf8)) + let readText = try runnerCommandFixture(#"{"command":"readText"}"#) + let snapshot = try runnerCommandFixture(#"{"command":"snapshot"}"#) + let tap = try runnerCommandFixture(#"{"command":"tap"}"#) let axServerNotFound = "NSException: Error kAXErrorServerNotFound" XCTAssertTrue(shouldRetryException(readText, message: axServerNotFound)) XCTAssertFalse(shouldRetryException(tap, message: axServerNotFound)) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ModelsTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ModelsTests.swift new file mode 100644 index 0000000000..d0079bf728 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ModelsTests.swift @@ -0,0 +1,32 @@ +import Foundation +import XCTest + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +extension RunnerTests { + func testErrorPayloadEncodesEveryFieldButTheRunnerInternalRetryableFailure() throws { + let payload = ErrorPayload( + code: "CODE", + message: "message", + hint: "hint", + retryableFailure: .targetAppUnavailable + ) + let encoded = try JSONEncoder().encode(payload) + let object = try XCTUnwrap(JSONSerialization.jsonObject(with: encoded) as? [String: Any]) + let storedFields = Set(Mirror(reflecting: payload).children.compactMap(\.label)) + XCTAssertEqual(Set(object.keys), storedFields.subtracting(["retryableFailure"])) + } + + func testTargetAppUnavailableErrorKeepsItsWireShape() throws { + let encoded = try JSONEncoder().encode( + ErrorPayload.targetAppUnavailable(bundleId: "com.example.app") + ) + let object = try XCTUnwrap(JSONSerialization.jsonObject(with: encoded) as? [String: Any]) + XCTAssertEqual(Array(object.keys), ["message"]) + XCTAssertEqual(object["message"] as? String, "app 'com.example.app' is not available") + XCTAssertEqual( + ErrorPayload.targetAppUnavailable(bundleId: nil).message, + "runner app is not available" + ) + } +} +#endif