diff --git a/.github/workflows/ios.yml b/.github/workflows/ios.yml index 81434bc9a0..0fd974f719 100644 --- a/.github/workflows/ios.yml +++ b/.github/workflows/ios.yml @@ -251,7 +251,7 @@ jobs: -only-testing:AgentDeviceRunnerUITests/RunnerTests/testPrivateAXNodesCarryAnnotatedCustomActions \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRequestPinnedBackendReportsItsOwnReason \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testCustomActionCoverageParsesOnlyCompletePairs \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testPartialCustomActionPassIsDisclosedAndCompleteOneIsNot \ + -only-testing:AgentDeviceRunnerUITests/RunnerTests/testStampedPayloadCarriesDisclosuresOnlyInTheVerdict \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testActionNamesAreCappedPerElementAndReported \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testHungCustomActionReadIsContainedAndRecovers \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAbandonedTreeCaptureSkipsQuerySweepAndHonorsWarmupExemption \ diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift index 354ffba76c..f77e0f6035 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift @@ -583,13 +583,9 @@ extension RunnerTests { XCTAssertEqual(coverage[RunnerAXSnapshotCustomActionsCandidatesKey] as? Int, 1) XCTAssertEqual( RunnerAXSnapshotBridge.customActionReadDispatchCount(), dispatchesBefore + 1) - - // And the rendered verdict names the hang, not the scroll remedy. - let blockedWarnings = Self.customActionCoverageWarnings( - Self.privateAXCustomActionCoverage(coverage)!) - XCTAssertEqual(blockedWarnings.count, 1) - XCTAssertTrue(blockedWarnings[0].contains("still hung")) - XCTAssertFalse(blockedWarnings[0].contains("Scroll")) + XCTAssertEqual( + Self.privateAXCustomActionCoverage(coverage), + SnapshotCustomActionCoverage(read: 0, candidates: 1, truncated: 0, blocked: true)) // 4. Recovery: once the wedged call returns, reads resume by themselves. hung.release() @@ -645,34 +641,6 @@ extension RunnerTests { XCTAssertFalse(truncated.boolValue) } - /// A capped pass must say so; a complete one must stay silent. - func testPartialCustomActionPassIsDisclosedAndCompleteOneIsNot() { - let partial = SnapshotQuality( - state: "recovered", backend: "private-ax", reason: nil, reasonCode: "requested-backend", - effectiveDepth: nil, collapsedLeafIndexes: nil, - customActions: SnapshotCustomActionCoverage(read: 12, candidates: 19, truncated: 0, blocked: false)) - let message = Self.legacyQualityMessage(partial) - XCTAssertTrue(message?.contains("12 of 19 merged elements") == true) - XCTAssertTrue(message?.contains("remaining 7") == true) - XCTAssertTrue(message?.contains("Scroll them into view") == true) - - // Every candidate read: nothing to disclose, and a healthy capture stays silent. - let complete = SnapshotQuality( - state: "healthy", backend: "private-ax", reason: nil, reasonCode: nil, - effectiveDepth: nil, collapsedLeafIndexes: nil, - customActions: SnapshotCustomActionCoverage(read: 19, candidates: 19, truncated: 0, blocked: false)) - XCTAssertNil(Self.legacyQualityMessage(complete)) - - // A healthy capture with an incomplete pass still discloses — the guard must - // not key the disclosure off degradation state. - let healthyButCapped = SnapshotQuality( - state: "healthy", backend: "private-ax", reason: nil, reasonCode: nil, - effectiveDepth: nil, collapsedLeafIndexes: nil, - customActions: SnapshotCustomActionCoverage(read: 12, candidates: 19, truncated: 0, blocked: false)) - XCTAssertTrue( - Self.legacyQualityMessage(healthyButCapped)?.contains("12 of 19") == true) - } - /// Action names annotated by the bridge must survive into the emitted node — /// the whole point of the capture is that the merged card names its hidden /// affordances. diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift index 7c6bc29443..16a999ab5d 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift @@ -648,8 +648,7 @@ extension RunnerTests { timing: capture.timing ) return DataPayload( - // Legacy human text for older daemons that read message instead of snapshotQuality. - message: Self.legacyQualityMessage(quality) ?? payload.message, + message: payload.message, nodes: payload.nodes, // Completeness, never provenance: a whole tree that a later backend produced (state // "recovered") stays untruncated, so strict absence reads can trust it. Only a real cap @@ -666,75 +665,6 @@ extension RunnerTests { runnerFatalReason: payload.runnerFatalReason ) } - - /// Response level, one line per incompleteness. An unread merged element is - /// byte-identical to one with no actions, and a clipped list looks complete, - /// so both have to name themselves. - static func customActionCoverageWarnings(_ coverage: SnapshotCustomActionCoverage) -> [String] { - var lines: [String] = [] - if coverage.blocked { - // Scrolling is the remedy for a budget stop, not for this one — saying it - // here would send the reader off doing something that cannot help. - lines.append( - "Custom actions were not read: an earlier accessibility read is still hung, so this " - + "capture skipped the read pass instead of queueing behind it. No element's actions " - + "list is authoritative here. Reads resume once that call returns.") - } else if coverage.read < coverage.candidates { - lines.append( - "Custom actions were read for \(coverage.read) of \(coverage.candidates) merged elements, " - + "on-screen ones first; the remaining \(coverage.candidates - coverage.read) were not read, " - + "so an absent actions list on those is not evidence that they have none. " - + "Scroll them into view and re-run to read them.") - } - if coverage.truncated > 0 { - lines.append( - "\(coverage.truncated) element(s) published more custom actions than are shown; those " - + "lists are clipped to the first 8 names, and long names are shortened.") - } - return lines - } - - static func legacyQualityMessage(_ quality: SnapshotQuality) -> String? { - let customActionWarnings = - quality.customActions.map { Self.customActionCoverageWarnings($0) } ?? [] - guard quality.state != "healthy" || quality.collapsedLeafIndexes != nil - || !customActionWarnings.isEmpty - else { return nil } - var parts: [String] = [] - if quality.state == "recovered" { - let meaning: String - switch quality.reasonCode { - case "budget", "deferred": - meaning = " The primary capture ran out of its time budget (busy app or simulator); the recovered tree is authoritative for this screen." - case "presentation-failed": - meaning = " The runner rejected a regular presentation because its cumulative clip invariant failed; report this as a runner bug and treat screenshot as visual truth." - default: - meaning = " This usually means the app publishes an unhealthy accessibility tree — fixing the app's accessibility is the real cure. Treat screenshot as visual truth when this warning appears." - } - parts.append( - "Detected an overly complex or slow accessibility tree. Fell back to the \(quality.backend) snapshot backend" - + (quality.reason.map { " after: \($0)." } ?? ".") - + meaning - ) - } - if quality.state == "sparse" { - parts.append( - "No snapshot backend could read this screen" - + (quality.reason.map { " (\($0))" } ?? "") - + ". Use screenshot as visual truth and coordinate taps." - ) - } - parts.append(contentsOf: customActionWarnings) - if let depth = quality.effectiveDepth { - // No --depth remedy here: an explicit --depth capture disables the - // frontier extension, so following it would return strictly less than - // this capture did. A plain re-run retries with a fresh extension budget. - parts.append( - "The accessibility server rejected deeper requests; content below depth \(depth) may be missing — re-run snapshot to retry deeper content." - ) - } - return parts.isEmpty ? nil : parts.joined(separator: " ") - } } #if AGENT_DEVICE_RUNNER_UNIT_TESTS @@ -830,28 +760,6 @@ extension RunnerTests { XCTAssertNil(Self.collapsedLeafIndexes([root, prose])) } - func testLegacyQualityMessageStatesFallbackMeaning() { - let recovered = SnapshotQuality( - state: "recovered", - backend: "queries", - reason: "snapshot returned only structural application/window nodes", - reasonCode: "sparse-tree", - effectiveDepth: nil, - collapsedLeafIndexes: nil, - customActions: nil - ) - let message = Self.legacyQualityMessage(recovered) - XCTAssertTrue(message?.contains("queries snapshot backend") == true) - XCTAssertTrue(message?.contains("fixing the app's accessibility") == true) - XCTAssertTrue(message?.contains("screenshot as visual truth") == true) - XCTAssertNil( - Self.legacyQualityMessage( - SnapshotQuality( - state: "healthy", backend: "tree", reason: nil, reasonCode: nil, effectiveDepth: nil, - collapsedLeafIndexes: nil, customActions: nil) - ) - ) - } func testTerminalFailsClosedOnInteractiveAxFailureRegardlessOfSparseBest() { // Interactive AX failure must invalidate + fail closed; a later tier's sparse synthetic-root // "best" must never downgrade this to a returned-sparse payload (regression: best == nil guard). @@ -897,6 +805,42 @@ extension RunnerTests { XCTAssertEqual(payload.nodes?.count, 1) } + func testStampedPayloadCarriesDisclosuresOnlyInTheVerdict() { + let root = planTestNode(index: 0, type: "Application", label: "App") + let merged = planTestNode( + index: 1, + type: "Other", + label: (0...30).map { "Tab \($0)" }.joined(separator: ", "), + parentIndex: 0 + ) + let coverage = SnapshotCustomActionCoverage( + read: 12, candidates: 19, truncated: 0, blocked: false) + let silent = stampedSnapshotPayload( + SnapshotBackendCapture( + payload: DataPayload(nodes: [root, merged], truncated: false), + effectiveDepth: nil, + customActions: coverage + ), + backend: .recursiveTree, + state: "healthy", + reason: nil + ) + XCTAssertNil(silent.message) + XCTAssertEqual(silent.snapshotQuality?.customActions, coverage) + XCTAssertEqual(silent.snapshotQuality?.collapsedLeafIndexes, [1]) + + let underlying = stampedSnapshotPayload( + SnapshotBackendCapture( + payload: DataPayload(message: "underlying", nodes: [root], truncated: false), + effectiveDepth: 4 + ), + backend: .privateAX, + state: "recovered", + reason: (reason: "tree capture timed out", code: "budget") + ) + XCTAssertEqual(underlying.message, "underlying") + } + func testStampedPayloadTruncationTracksCompletenessNotRecoveryProvenance() { let complete = SnapshotBackendCapture( payload: DataPayload( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationInvariantTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationInvariantTests.swift index 65012e982d..c0c739b86d 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationInvariantTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationInvariantTests.swift @@ -338,20 +338,6 @@ extension RunnerTests { XCTAssertEqual(Self.snapshotQualityReasonCode(for: captureFailure), "presentation-failed") XCTAssertNotEqual(Self.snapshotQualityReasonCode(for: captureFailure), "capture-failed") XCTAssertTrue(captureFailure.message.contains("cumulative clip")) - - let warning = Self.legacyQualityMessage( - SnapshotQuality( - state: "recovered", - backend: "queries", - reason: captureFailure.message, - reasonCode: captureFailure.qualityReasonCode, - effectiveDepth: nil, - collapsedLeafIndexes: nil, - customActions: nil - ) - ) - XCTAssertTrue(warning?.contains("runner bug") == true) - XCTAssertFalse(warning?.contains("fixing the app's accessibility") == true) } } #endif diff --git a/packages/platform-apple/src/__tests__/interactor-runner-provider.test.ts b/packages/platform-apple/src/__tests__/interactor-runner-provider.test.ts index d55f0a4c0a..d0106f3392 100644 --- a/packages/platform-apple/src/__tests__/interactor-runner-provider.test.ts +++ b/packages/platform-apple/src/__tests__/interactor-runner-provider.test.ts @@ -266,6 +266,32 @@ test('snapshot publishes runner presentation through the engine and drops its qu assert.equal('qualityPayload' in result, false); }); +test('a message-less runner capture leaves its disclosures to the verdict', async () => { + const coverage = { read: 12, candidates: 19, truncated: 0, blocked: false }; + const interactor = createAppleInteractor( + IOS_SIMULATOR, + {}, + { + hasLiveSession: () => true, + runCommand: async () => ({ + ...runnerResultFor({ command: 'snapshot' }), + snapshotQuality: { + state: 'healthy', + backend: 'tree', + customActions: coverage, + collapsedLeafIndexes: [1], + }, + }), + }, + ); + + const result = presentedSnapshot(await interactor.snapshot()); + + assert.equal('warnings' in result, false); + assert.deepEqual(result.quality?.customActions, coverage); + assert.deepEqual(result.quality?.collapsedLeafIndexes, [1]); +}); + test('macOS app snapshots preserve runner nodes outside the iOS presentation engine', async () => { const nodes = [{ index: 0, type: 'Application', label: 'System Settings' }]; const interactor = createAppleInteractor( diff --git a/src/commands/capture/runtime/snapshot.test.ts b/src/commands/capture/runtime/snapshot.test.ts index fa3056a692..378c3596a1 100644 --- a/src/commands/capture/runtime/snapshot.test.ts +++ b/src/commands/capture/runtime/snapshot.test.ts @@ -364,6 +364,32 @@ test('runtime snapshot renders the structured quality verdict and skips legacy d assert.deepEqual(result.snapshotQuality?.state, 'recovered'); }); +test('runtime snapshot renders healthy-capture disclosures from the verdict alone', async () => { + const tabBarLabel = Array.from({ length: 12 }, (_, i) => `Tab ${i}`).join(', '); + const device = createSnapshotOnlyDevice({ + nodes: [ + { ref: 'e1', index: 0, depth: 0, type: 'Application', label: 'App' }, + { ref: 'e2', index: 1, depth: 1, parentIndex: 0, type: 'Other', label: tabBarLabel }, + { ref: 'e3', index: 2, depth: 1, parentIndex: 0, type: 'Button', label: 'Ok' }, + ], + truncated: false, + backend: 'xctest', + quality: { + state: 'healthy', + backend: 'tree', + customActions: { read: 12, candidates: 19, truncated: 0, blocked: false }, + collapsedLeafIndexes: [1], + }, + }); + + const result = await device.capture.snapshot({ session: 'default' }); + + assert.deepEqual(result.warnings, [ + 'Custom actions were read for 12 of 19 merged elements, on-screen ones first; the remaining 7 were not read, so an absent actions list on those is not evidence that they have none. Scroll them into view and re-run to read them.', + "@e2 [Other] merges many labels into a single accessibility element. The app likely marks a container as accessible, which hides every descendant from assistive tech and automation — the children cannot be addressed individually. Fix the app's accessibility (mark the rows, not the container); until then use screenshot as visual truth and coordinate taps.", + ]); +}); + test('runtime snapshot does not warn for a normal iOS interactive output', async () => { const device = createSnapshotOnlyDevice({ nodes: [