diff --git a/.fallowrc.json b/.fallowrc.json index 84cd610a37..7c3e938e4d 100644 --- a/.fallowrc.json +++ b/.fallowrc.json @@ -115,9 +115,6 @@ "isRetryableIosDeviceTraceRecordFailure", "resolveIosDevicePerfHint", "captureAppleMemorySnapshot", - "readAppleProcessSamples", - "resolveAppleExecutable", - "resolveIosDevicePerfTarget", "sampleAppleFramePerf", "sampleAppleMemoryPerf", "cleanupAppleXctracePerfCapture", @@ -160,7 +157,6 @@ "prepareIosRunner", "resolveRunnerAppBundleId", "detachIosRunnerSessionsForShutdown", - "getRunnerSessionSnapshot", "stopIosRunnerSession", "stopAllIosRunnerSessions", "readStaleRunnerLease", @@ -217,7 +213,7 @@ }, { "comment": "Dedicated CLI command handlers are reached only through the dynamic `import()` table `dedicatedCliCommandHandlerLoaders` in src/cli/commands/router.ts, which --production analysis cannot follow to a consumer. Same shape as the daemon route-handler entry above; that table is what enumerates this list, so add/remove here whenever a loader is added/removed.", - "file": "src/cli/commands/{auth,connection,daemon,device,proxy,replay,screenshot,takeover}.ts", + "file": "src/cli/commands/{auth,connection,daemon,device,proxy,recording,replay,screenshot,takeover}.ts", "exports": [ "authCommand", "connectCommand", @@ -226,6 +222,7 @@ "daemonCommand", "deviceCommand", "proxyCommand", + "recordingCommand", "replayCommand", "screenshotCommand", "diffCommand", @@ -243,7 +240,7 @@ "exports": ["*"] }, { - "comment": "#1642: its three consumers (src/__tests__/cli-device-status.test.ts, src/daemon/__tests__/device-claims.test.ts, src/daemon/__tests__/device-claim-prune.test.ts) all reach it the only way a Vitest module mock can — `vi.mock(path, async (importOriginal) => (await import('...host-process-mock.ts')).pinOwnProcessStartTime(importOriginal))`. Dependency analysis cannot follow that dynamic import to a consumer, the same limitation the daemon route-handler entry above records.", + "comment": "#1642: its three consumers (src/__tests__/cli-device-status.test.ts, src/daemon/device/__tests__/device-claims.test.ts, src/daemon/device/__tests__/device-claim-prune.test.ts) all reach it the only way a Vitest module mock can — `vi.mock(path, async (importOriginal) => (await import('...host-process-mock.ts')).pinOwnProcessStartTime(importOriginal))`. Dependency analysis cannot follow that dynamic import to a consumer, the same limitation the daemon route-handler entry above records.", "file": "src/__tests__/test-utils/host-process-mock.ts", "exports": ["pinOwnProcessStartTime"] }, @@ -257,6 +254,8 @@ "file": "{packages/command-registry/src/registry.ts,src/core/interactors/register-builtins.ts,src/core/platform-descriptor/registry.ts,src/daemon/request-platform-providers.ts}", "exports": [ "CommandOwnerFileClaimsAreComplete", + "CommandUnionStaysLiteral", + "CliCommandUnionStaysLiteral", "BuiltinPluginsCoverAllPlatforms", "PlatformDescriptorsAreTotal", "GatedKeysAreResolverKeys" diff --git a/.github/actions/setup-apple-runner-build/action.yml b/.github/actions/setup-apple-runner-build/action.yml index aad21ef883..c05ecd9326 100644 --- a/.github/actions/setup-apple-runner-build/action.yml +++ b/.github/actions/setup-apple-runner-build/action.yml @@ -44,7 +44,7 @@ runs: id: source-hash run: | set -euo pipefail - echo "value=${{ hashFiles('apple/runner/**', 'apple/snapshot-presentation/**', 'scripts/build-xcuitest-apple.sh', 'scripts/swift-toolchain-tmpdir.ts', 'scripts/patch-xcuitest-runner-icon.ts', 'scripts/write-xcuitest-cache-metadata.mjs', 'packages/platform-apple/src/runner/apple-runner-platform.ts', 'packages/platform-apple/src/runner/runner-cache-metadata.ts', 'packages/platform-apple/src/runner/runner-icon.ts', 'packages/platform-apple/src/runner/runner-xctestrun.ts', 'packages/platform-apple/src/runner/runner-xctestrun-products.ts', '.github/actions/setup-apple-runner-build/action.yml') }}" >> "$GITHUB_OUTPUT" + echo "value=${{ hashFiles('apple/runner/**', 'apple/snapshot-presentation/**', 'scripts/build-xcuitest-apple.sh', 'scripts/swift-toolchain-tmpdir.ts', 'scripts/patch-xcuitest-runner-icon.ts', 'scripts/write-xcuitest-cache-metadata.mjs', 'packages/platform-apple/src/runner/**', '!packages/platform-apple/src/runner/__tests__/**', '.github/actions/setup-apple-runner-build/action.yml') }}" >> "$GITHUB_OUTPUT" shell: bash - name: Resolve Apple runner build variant diff --git a/.github/workflows/1874-diagnose.yml b/.github/workflows/1874-diagnose.yml deleted file mode 100644 index a0e9ee0a01..0000000000 --- a/.github/workflows/1874-diagnose.yml +++ /dev/null @@ -1,132 +0,0 @@ -name: 1874 Diagnose - -# Under the lane rule in #1781: -# -# - Catches: iOS text-entry commit episodes on CI hardware — the dispatched test looped on both -# arches, with and without the neighbour test that shares the simulator. An absorbed episode -# surfaces as a slow pass, so the cadence is kept for passes too. -# - Evidence: the 2026-08-21 dispatch passed 50/50 and still caught an episode (`type-all` -# 2487 ms across 5 polls) that no PR lane would report; it does not reproduce locally. -# - Cost: manual dispatch and the `diagnose/1874-commit-stall` branch only — never a PR or `main`. -# One dispatch is four macOS slots, each a runner build plus `iterations` iterations of -# simulator time. -# - Kill criterion: delete when #2080 closes. It is the last open text-entry question this loop -# is the CI reproduction path for, and nothing else reads these artifacts. - -on: - workflow_dispatch: - inputs: - iterations: - description: 'Loop iterations per job' - required: false - default: '25' - test: - description: 'RunnerTests method to loop' - required: false - default: testBareTypeUsesTappedInputWhenSoftwareKeyboardIsHidden - push: - branches: - - diagnose/1874-commit-stall - -permissions: - contents: read - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - stall-loop: - name: ${{ matrix.arch }} ${{ matrix.mode }} stall loop - runs-on: macos-26 - timeout-minutes: 90 - strategy: - fail-fast: false - matrix: - mode: [isolated, pair] - arch: [x86_64, arm64] - env: - IOS_RUNTIME_VERSION: '26.2' - AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH: ${{ github.workspace }}/.tmp/ios-runner-derived - AGENT_DEVICE_XCUITEST_INCLUDE_UNIT_TESTS: '1' - AGENT_DEVICE_XCUITEST_ARCHS: ${{ matrix.arch }} - steps: - - uses: actions/checkout@v4 - - - name: Setup toolchain - uses: ./.github/actions/setup-node-pnpm - - - name: Restore and build iOS XCTest runner - uses: ./.github/actions/setup-apple-runner-build - with: - derived-path: ${{ env.AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH }} - cache-key-prefix: ios-runner-prebuilt - cache-key-suffix: -ios-${{ env.IOS_RUNTIME_VERSION }}-1874-${{ matrix.arch }} - gate: swift-runner-ios - xcuitest-platform: ios - xcuitest-destination: generic/platform=iOS Simulator - - - name: Boot iOS test simulator - id: ios-simulator - uses: ./.github/actions/boot-ios-test-simulator - with: - runtime-version: ${{ env.IOS_RUNTIME_VERSION }} - preferred-device-name: iPhone 17 Pro - - - name: Stall loop - env: - ITERATIONS_INPUT: ${{ github.event.inputs.iterations }} - TEST_INPUT: ${{ github.event.inputs.test }} - run: | - set -euo pipefail - XCTESTRUN_PATH="$(find "$AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH/Build/Products" -maxdepth 1 -name '*.xctestrun' -print -quit)" - test -n "$XCTESTRUN_PATH" - UDID="${{ steps.ios-simulator.outputs.simulator-udid }}" - ITER="${ITERATIONS_INPUT:-25}" - TEST_NAME="${TEST_INPUT:-testBareTypeUsesTappedInputWhenSoftwareKeyboardIsHidden}" - mkdir -p .tmp - - if [ "${{ matrix.mode }}" = "pair" ]; then - EXTRA_ONLY_TESTING=(-only-testing:AgentDeviceRunnerUITests/RunnerTests/testBareDelayedTypeFailsWhenTappedInputDisappearsMidCommand) - else - EXTRA_ONLY_TESTING=() - fi - - sys_prof="$(system_profiler SPHardwareDataType 2>/dev/null | grep -E 'Chip|Cores|Memory' || true)" - echo "HOST: $sys_prof" | tee stall-summary.txt - - pass=0; skipped=0; failed=0; noresult=0; runfailed=0 - for i in $(seq 1 "$ITER"); do - LOG=".tmp/stall-run-$i.log" - set +e - xcodebuild test-without-building \ - -xctestrun "$XCTESTRUN_PATH" \ - -destination "platform=iOS Simulator,id=$UDID" \ - -only-testing:"AgentDeviceRunnerUITests/RunnerTests/$TEST_NAME" \ - ${EXTRA_ONLY_TESTING[@]+"${EXTRA_ONLY_TESTING[@]}"} \ - > "$LOG" 2>&1 - rc=$? - set -e - verdict="$(node --experimental-strip-types scripts/diagnose-1874-iteration.ts \ - "$LOG" "$TEST_NAME" "$i" "$rc")" - case "$verdict" in - passed) pass=$((pass+1)) ;; - skipped) skipped=$((skipped+1)) ;; - no-result) noresult=$((noresult+1)) ;; - run-failed) runfailed=$((runfailed+1)) ;; - *) failed=$((failed+1)) ;; - esac - done - - echo "RESULT arch=${{ matrix.arch }} mode=${{ matrix.mode }}: $pass passed, $skipped skipped, $failed failed, $noresult no-result, $runfailed run-failed of $ITER" | tee -a stall-summary.txt - - - name: Upload stall evidence - if: always() - uses: actions/upload-artifact@v4 - with: - name: stall-evidence-${{ matrix.arch }}-${{ matrix.mode }} - include-hidden-files: true - path: | - stall-summary.txt - .tmp/stall-evidence-*.log - if-no-files-found: ignore diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index 5b375ede7f..2f89157167 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -7,7 +7,6 @@ on: - 'website/**' - 'README.md' - 'AGENTS.md' - - 'CHANGELOG.md' - 'CONTEXT.md' - 'CONTRIBUTING.md' - 'LICENSE' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a29b930cae..7691398f68 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,7 +7,6 @@ on: - 'website/**' - 'README.md' - 'AGENTS.md' - - 'CHANGELOG.md' - 'CONTEXT.md' - 'CONTRIBUTING.md' - 'LICENSE' diff --git a/.github/workflows/ios.yml b/.github/workflows/ios.yml index ecb5fc3b54..56e58d838e 100644 --- a/.github/workflows/ios.yml +++ b/.github/workflows/ios.yml @@ -7,7 +7,6 @@ on: - 'website/**' - 'README.md' - 'AGENTS.md' - - 'CHANGELOG.md' - 'CONTEXT.md' - 'CONTRIBUTING.md' - 'LICENSE' @@ -98,6 +97,16 @@ jobs: pnpm-lock.yaml examples/test-app/pnpm-lock.yaml + - name: Select iOS runner XCTests + id: xctest-impact + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + if [ "${{ github.event_name }}" = 'pull_request' ]; then + git fetch origin "$BASE_SHA" --depth=1 || echo 'Base fetch failed; XCTest selection will fail open.' + fi + node --experimental-strip-types scripts/apple-ci-impact.ts xctest + - name: Establish host focus canary id: host-focus-canary run: | @@ -126,133 +135,23 @@ jobs: runtime-version: ${{ env.IOS_RUNTIME_VERSION }} preferred-device-name: iPhone 17 Pro - - name: Verify clean-installed Simulator snapshot bridge preparation - if: github.event_name == 'pull_request' - env: - BASE_SHA: ${{ github.event.pull_request.base.sha }} - run: | - git fetch origin "$BASE_SHA" --depth=1 - if git diff --quiet "$BASE_SHA"...HEAD -- \ - apple/snapshot-bridge \ - packages/platform-apple/src/snapshot-source \ - scripts/check-package.ts \ - scripts/size-report-install.mjs \ - scripts/size-report-package.mjs; then - echo "Snapshot bridge packaging is unchanged; skipping preparation proof." - exit 0 - fi - pnpm build - pnpm exec vitest run packages/platform-apple/src/snapshot-source/native-runtime.test.ts - pnpm check:package -- --verify-snapshot-bridge-preparation - - name: Run targeted iOS runner XCTest regressions + id: ios-xctest + if: steps.xctest-impact.outputs.run != 'false' run: | - set -o pipefail + set -euo pipefail XCTESTRUN_PATH="$(find "$AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH/Build/Products" -maxdepth 1 -name '*.xctestrun' -print -quit)" test -n "$XCTESTRUN_PATH" + IOS_PR_TEST_ARGS=() + IOS_PR_TEST_IDS="$(node --experimental-strip-types scripts/check-xctest-selection.ts --ios-pr-tests)" + while IFS= read -r test_id; do + IOS_PR_TEST_ARGS+=("-only-testing:$test_id") + done <<< "$IOS_PR_TEST_IDS" + test "${#IOS_PR_TEST_ARGS[@]}" -gt 0 xcodebuild test-without-building \ -xctestrun "$XCTESTRUN_PATH" \ -destination "platform=iOS Simulator,id=${{ steps.ios-simulator.outputs.simulator-udid }}" \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSinglePointerFlingFallsBackToXCTestCoordinateDragWhenPrivateSynthesisFails \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAlertDispatchResolvesItsOwnModalWithoutCoordinateTapRoutingProbe \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAlertResolutionCannotBypassRequestedDeadline \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testTypeWithoutResolvedInputReturnsTypedFailureBeforeDispatchingText \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testBareTypeUsesTappedInputWhenSoftwareKeyboardIsHidden \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testHardwareKeyboardResponderConfirmsItsOwnKeyboardFocus \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testPenalizedCoordinateTapOnNonTextControlDoesNotAuthorizeBareType \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testBareDelayedTypeFailsWhenTappedInputDisappearsMidCommand \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSynthesizedTextCommitProgressWalksExpectedPrefixOnly \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testEmptyReplacementWithoutResolvableTargetFailsClosed \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testTextEntryTapWitnessIsBoundToTargetIdentity \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testCoordinateTapTextInputProbeSkipsPenalizedXCTestChannel \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testScrollViewportKeyboardClipMatchesGoldenParityTable \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testScrollViewportPolicyUsesParityTableConstants \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testScrollViewportDispatchKeepsTheUnclippedFrameAsItsCoordinateRotationBasis \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSynthesizedGesturePoliciesMatchCommandContracts \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testFreshCoordinateTapContainsUnavailableTextInputProbe \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testTextInputProbeIssueScopeIsThreadBound \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testTextInputProbePreservesEnclosingRunnerWait \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSuppressedAxIssueMakesTextInputProbeUnavailable \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testHealthyCoordinateTapPreservesBareTypingWitness \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testTextInputProbeContainmentExcludesRequiredReadsAndLaterIssues \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testCoordinateTextInputCandidateMustBeEnabledAndContainTheTouchPoint \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testQuerySelectorPrefersHittableMatchOverNonHittableDuplicate \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testActivateTargetSkipsForegroundAndActivatesNonForegroundApplication \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testApplicationStateRawValuesMatchTheActivationDecoder \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testMissingBundleCommandInvalidatesCompleteCachedTargetState \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testCachedTargetInvalidationClearsProcessBoundState \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAlertResolutionCannotBypassRequestedDeadline \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAlertAcceptTreatsOpenAsAffirmative \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAlertAcceptDoesNotActivateAReplacementWithASharedButton \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAlertDismissDoesNotActivateAReplacementWithTheSameTitle \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAlertCannotProveAnIdenticalReplacementAndDoesNotActivateIt \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAlertReplacementCommandDeadlineScalesWithMeasuredLatency \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAlertDeadlineBeforeActivationLeavesTheOriginalUntouched \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAlertHittableProbeCompletingAfterDeadlineLeavesTheOriginalUntouched \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAlertActivationIgnoresAnAppThatNeverSettlesBeforeTheDeadline \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSystemModalProbeSliceSharesAndClampsToPlanDeadline \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testDispatchRecoverySkipsBookkeepingWhileXCTestChannelOccupied \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testBoundedSystemModalProbeTimeoutRecoversThenReleasesOnDrain \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testBoundedSystemModalProbeTimeoutRecoversThenReleasesOnDrainForSnapshotRaw \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSnapshotTraversalIdentityPreservesSameOriginNodesWithDifferentBounds \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSnapshotPresentationPreservesCurrentWireShape \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRegularPresentationKeepsNestedClipGeometryCumulative \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRegularPresentationKeepsFramelessSemanticCarriersNonActionableAndNonClipping \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRawPresentationKeepsReportedOffscreenAndFramelessFacts \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testFixedSeedRegularPresentationMaintainsCumulativeClipInvariant \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testPresentationFailureKeepsItsNamedSnapshotQualityReason \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRegularPresentationPublishesEffectiveRectWhileRawKeepsReportedFrame \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSnapshotPresentationOwnsBackendNeutralEligibility \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSnapshotPresentationOwnsScopeAndRelativeDepth \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testPresentationRefusesAnAcquisitionCapturedForTheOtherProjection \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testCaptureHintIsTheOnlyAcquisitionViewOfARequest \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRegularDepthCutsPresentationNotAcquisition \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testVisibilityFoldKeepsIndependentChildPastClippedParent \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRegularDepthKeepsTurnedKeyboardSubtreeAfterOneNormalizationPass \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRegularFoldClipsScrollOverflowReparentsAndBooksHints \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRegularFoldKeepsWindowCarriersButNeverHittableOutsideClip \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRegularFoldDropsSubPixelContentlessDecorationOnEveryBackend \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testPlainViewportPolicyFoldsWithoutAncestorCursor \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testPlainViewportPolicyDoesNotClipToScrollAncestor \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testEffectiveGeometryIntersectsViewportAndAncestorClip \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testEffectiveGeometryKeepsReportedOriginWhenFullyClipped \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRegularPresentationRoutesThroughVisibilityFold \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRegularPresentationPublishesGeometricActionabilityWithoutOcclusionOrTypeGate \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSnapshotAcquisitionDoesNotReintroduceRunnerOcclusionScan \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSnapshotScopePolicyMatchesGoldenParityTable \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testScrollContainerTypeNamesMatchElementTypeSet \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testPrivateAXDepthLimitedRequiresEveryFrontierResolved \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testDeepExtensionCountsMissedFrontiers \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testPreferredPrivateAXBackendPlansAsPenalized \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testPreferredTreeBackendPinsRegularPlanAndLeavesStructuredEvidence \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRawDiagnosticPlanCarriesOnlyBackendsThatCanServeRaw \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSnapshotBackendDeclarationsMatchCapabilityFixture \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testProjectionMismatchFailureIsStructuredAndNotAnAxFailure \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testPrivateAXRegularPresentationProjectsToViewportAndKeepsScrollHint \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testPrivateAXRawProjectionKeepsEveryAcquiredNode \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testPrivateAXRawProjectionAppliesRequestedTraversalDepth \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testPrivateAXPresentationKeepsOffscreenSubtreeExcludedWhenChildFramesAreClamped \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testPrivateAXGeometrylessSemanticsAreNeverActionableOrScrollContexts \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testPrivateAXAcquisitionDoesNotInterpretScope \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testPrivateAXInteractiveFiltersLoginLikeHiddenDrawer \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testDecodedPreferredBackendReachesOptionsAndApplicablePlan \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSparsePayloadReasonMatrix \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testStampedPayloadTruncationTracksCompletenessNotRecoveryProvenance \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRunnerScreenshotStabilitySettledNeedsEnoughSamples \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRunnerScreenshotStabilitySettledTrueWhenWindowMatches \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRunnerScreenshotStabilitySettledFalseOnMidWindowMismatch \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRunnerScreenshotStabilitySettledFalseOnFailedCapture \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRunnerScreenshotStabilitySettledOnlyLooksAtTheTrailingWindow \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRunnerScreenshotStabilitySettledRejectsDegenerateRequirement \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testCustomActionsRequestPinsPrivateAXBackend \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testPrivateAXNodesCarryAnnotatedCustomActions \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRequestPinnedBackendReportsItsOwnReason \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testCustomActionCoverageParsesOnlyCompletePairs \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testPartialCustomActionPassIsDisclosedAndCompleteOneIsNot \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testActionNamesAreCappedPerElementAndReported \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testHungCustomActionReadIsContainedAndRecovers \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAbandonedTreeCaptureSkipsQuerySweepAndHonorsWarmupExemption \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testPrivateAXPinnedRegularDepthReachesAcquisitionAndPresentation 2>&1 | tee /tmp/agent-device-runner-regressions.log + "${IOS_PR_TEST_ARGS[@]}" 2>&1 | tee /tmp/agent-device-runner-regressions.log node --input-type=module -e ' import { readFileSync } from "node:fs"; const log = readFileSync("/tmp/agent-device-runner-regressions.log", "utf8"); @@ -282,17 +181,6 @@ jobs: run: | echo "Fixture app source: ${{ steps.fixture-app.outputs.source }}" >> "$GITHUB_STEP_SUMMARY" - - name: Run fixture-backed iOS simulator E2E smoke - env: - AGENT_DEVICE_FIXTURE_APP_ID: ${{ steps.fixture-app.outputs.app-id }} - AGENT_DEVICE_FIXTURE_APP_PATH: ${{ steps.fixture-app.outputs.app-path }} - AGENT_DEVICE_IOS_E2E: '1' - AGENT_DEVICE_IOS_E2E_TIER: smoke - AGENT_DEVICE_IOS_UDID: ${{ steps.ios-simulator.outputs.simulator-udid }} - run: | - pnpm gate build - node --experimental-strip-types scripts/node-test-tmpdir.ts --test test/integration/smoke-ios-simulator-coverage.test.ts test/integration/smoke-ios-simulator.test.ts - # #1584: isolated, cheap automatic guard for the #1562 class of regression (iOS silently # ignoring a `gesture pan` duration). Split out of gesture-lab.ad, which stays full-tier # (dispatch-only via replays-manual.yml) because its multi-touch commands are a separate, @@ -307,6 +195,17 @@ jobs: node --experimental-strip-types src/bin.ts test examples/test-app/replays/gesture-pan-duration.ad --udid "${{ steps.ios-simulator.outputs.simulator-udid }}" --timeout 180000 --retries 2 --artifacts-dir test/artifacts/replays-ios-gesture-pan-duration --report-junit test/artifacts/replays-ios-gesture-pan-duration.junit.xml pnpm clean:daemon + - name: Run fixture-backed iOS simulator E2E smoke + env: + AGENT_DEVICE_FIXTURE_APP_ID: ${{ steps.fixture-app.outputs.app-id }} + AGENT_DEVICE_FIXTURE_APP_PATH: ${{ steps.fixture-app.outputs.app-path }} + AGENT_DEVICE_IOS_E2E: '1' + AGENT_DEVICE_IOS_E2E_TIER: smoke + AGENT_DEVICE_IOS_UDID: ${{ steps.ios-simulator.outputs.simulator-udid }} + run: | + pnpm gate build + node --experimental-strip-types scripts/node-test-tmpdir.ts --test test/integration/smoke-ios-simulator-coverage.test.ts test/integration/smoke-ios-simulator.test.ts + - name: Assert simulator automation preserved host focus if: ${{ always() }} run: | @@ -323,6 +222,17 @@ jobs: pnpm clean:daemon node --experimental-strip-types src/bin.ts test test/integration/replays/ios/device/01-physical-lifecycle.ad --udid "$IOS_UDID" --retries 2 --artifacts-dir test/artifacts/replays-ios-device-smoke --report-junit test/artifacts/replays-ios-device-smoke.junit.xml + - name: Assert iOS XCTest selection was honored + if: always() + env: + SELECTED: ${{ steps.xctest-impact.outputs.run }} + OUTCOME: ${{ steps.ios-xctest.outcome }} + run: | + if [ "$SELECTED" = 'true' ] && [ "$OUTCOME" = 'skipped' ]; then + echo '::error::iOS runner XCTest was selected but its step was skipped.' + exit 1 + fi + - name: Upload iOS artifacts if: always() uses: ./.github/actions/upload-agent-device-artifacts diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index eec6b844bc..2d1c29d629 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -7,7 +7,6 @@ on: - 'website/**' - 'README.md' - 'AGENTS.md' - - 'CHANGELOG.md' - 'CONTEXT.md' - 'CONTRIBUTING.md' - 'LICENSE' diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 32320e23d9..8093217e6c 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -7,7 +7,6 @@ on: - 'website/**' - 'README.md' - 'AGENTS.md' - - 'CHANGELOG.md' - 'CONTEXT.md' - 'CONTRIBUTING.md' - 'LICENSE' @@ -152,6 +151,25 @@ jobs: --report-junit test/artifacts/replays-macos.junit.xml + # Clean-install verification may raise macOS local-network permission UI; keep it after replay. + - name: Select Apple bridge proof + if: github.event_name == 'pull_request' + id: bridge-impact + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + git fetch origin "$BASE_SHA" --depth=1 || echo 'Base fetch failed; bridge proof selection will fail open.' + node --experimental-strip-types scripts/apple-ci-impact.ts bridge + + - name: Verify clean-installed Simulator snapshot bridge preparation and the fold-helper -Werror gate + if: github.event_name == 'pull_request' && steps.bridge-impact.outputs.run != 'false' + run: | + pnpm build + pnpm exec vitest run \ + packages/platform-apple/src/snapshot-source/native-runtime.test.ts \ + packages/platform-apple/src/foldable/fold-helper-cache.test.ts + pnpm check:package -- --verify-snapshot-bridge-preparation + - name: Upload macOS artifacts if: always() uses: ./.github/actions/upload-agent-device-artifacts diff --git a/.github/workflows/size.yml b/.github/workflows/size.yml index e2014c1aa6..0163a4d1eb 100644 --- a/.github/workflows/size.yml +++ b/.github/workflows/size.yml @@ -7,7 +7,6 @@ on: - 'website/**' - 'README.md' - 'AGENTS.md' - - 'CHANGELOG.md' - 'CONTEXT.md' - 'CONTRIBUTING.md' - 'LICENSE' diff --git a/.github/workflows/xctest-nightly.yml b/.github/workflows/xctest-nightly.yml index 91ba7a5648..39ae0d1783 100644 --- a/.github/workflows/xctest-nightly.yml +++ b/.github/workflows/xctest-nightly.yml @@ -126,10 +126,8 @@ jobs: # (packages/platform-apple/src/runner/runner-session.ts always passes it as the sole # `-only-testing:`). It compiles unconditionally — the `#if AGENT_DEVICE_RUNNER_UNIT_TESTS` # block ends well above it — so an unfiltered run reaches it in alphabetical order and - # hangs the job until `timeout-minutes`. Its one escape hatch, - # AGENT_DEVICE_RUNNER_NOOP_STARTUP, is an environment variable, and the comment above - # the flag records that env plumbing into a simulator test process is not dependable; - # `-skip-testing:` is the lever that works from here. + # hangs the job until `timeout-minutes`. The method has no in-process escape hatch, so + # `-skip-testing:` is the lever that keeps it out of this run. # # A typo in that identifier silently re-arms the hang, so # `pnpm check:xctest-selection` validates `-skip-testing:` exactly like `-only-testing:`. diff --git a/.oxfmtrc.json b/.oxfmtrc.json index ef771b1b35..00e66b8708 100644 --- a/.oxfmtrc.json +++ b/.oxfmtrc.json @@ -10,6 +10,8 @@ "node_modules/**", "**/*.md", "scripts/maestro-conformance/corpus/**", - "fallow-baselines/**" + "fallow-baselines/**", + // One entry per line, so each pinned runner request changes as one reviewable diff line. + "contracts/fixtures/runner-requests.json" ] } diff --git a/CHANGELOG.md b/CHANGELOG.md deleted file mode 100644 index 4d8783a9c6..0000000000 --- a/CHANGELOG.md +++ /dev/null @@ -1,769 +0,0 @@ -# Changelog - -## Unreleased - -- Fixed (ios): `fold half-open` no longer reports a pose the hinge never held. The command accepted - the requested category as the answer whenever its four-read budget happened to end on a - `half-open` angle, so a hinge still sweeping from 180° toward 0° — which passes through every - half-open angle on the way — came back as a settled Book pose. An angle inside that open interval - proves the category and nothing else, so `half-open` now requires two consecutive readings that - both fall inside it and agree within 0.5°; a pair straddling the 179° boundary is two poses, not - one hinge at rest. A budget ending on an unsettled angle fails with `COMMAND_FAILED` and - `details.reason: "fold-pose-unsettled"`, carrying `requestedPose`, `observedPose`, - `hingeAngleDegrees` and the previous reading, and says the hinge was observed half-open and did - not settle rather than claiming it never got there. `fold-pose-unverified` still covers a budget - that ends on another pose. Supersedes ADR 0025's budget-end exception (#2730). -- Fixed (ios): an interactive snapshot of a list whose row holds selectable text (a `TextView`) no - longer drops every row after that text. A UITextView is a scroll view too, and XCTest publishes - its scroll indicator inside the text; the presentation read that indicator onto the list around - it and clipped the list's viewport to one line, so a post thread whose root post is selectable - text showed the root and none of its replies, while the full snapshot showed all of them. An - indicator whose nearest scrolling ancestor is a text view now belongs to that text view and - derives no list viewport. -- Fixed (ios): a foreground repair names the state the session app was really in. The decoder translated - the runner's raw `XCApplicationState` integer from a handwritten table that had - `runningBackgroundSuspended` (2) and `runningBackground` (3) the wrong way round, so a suspended app - was disclosed as merely backgrounded in `targetActivation.priorState` and in the appended warning — - as shipped in v0.21.7 and v0.21.8. The table now follows what the SDK declares, a runner XCTest pins - every integer against `XCUIApplication.State` itself, and the decoder test reads those pins, so the - two copies cannot drift apart unnoticed. -- Fixed (macos): `press` and `click` on the `frontmost-app`, `desktop`, and `menubar` surfaces post - clicks the way a trackpad does. The helper posted the release in the same tick as the press, so - AppKit dropped it and controls never activated; each press is now held (`--hold-ms`, 60 ms by - default, 40 ms at least). `--count` is that many independent clicks and `--double-tap` posts each - as a double-click pair, the helper's deadline follows the click schedule instead of a fixed 30 s, - and every stop the host applies mid-hold — a cancelled request, a dropped client, the deadline — - reaches the helper as SIGTERM first so it releases the button before it exits; SIGKILL only - follows a helper that has not exited a second later. -- Added (ios): `fold ` and `client.command.fold({ pose })` put a foldable - iPhone simulator (iPhone Duo) into a hinge pose. ADR 0025 recorded that no official host API sets - the pose and left it to the operator; the pose is now set the way the operator did it, by pressing - the pose control in the Xcode Device Hub window through macOS accessibility, and it is confirmed - the way ADR 0025 asked for, by reading the hinge angle back from CoreDevice - (`devicectl device motion hinge-angle`) until it agrees with the request. The response reports - the verified pose, the hinge angle, and the panel the device now lights with its point size, so an - agent can see that its refs and coordinates are stale without another capture. The macOS helper - gained a `device-hub pose` subcommand that finds Device Hub in the process table (LaunchServices - registers the trampolined app with no pid), reopens its window when it shows none, and selects the - simulator through the sidebar row keyed by its UDID so two simulators sharing a name cannot be - confused. Simulator-only; a single-panel simulator refuses with `UNSUPPORTED_OPERATION`, and every - other platform states its own refusal cell. -- Added (diff): `diff screenshot` accepts a JPEG baseline or current image. Both inputs had to be PNG, - so a capture exported by another tool had to be converted first and a HarmonyOS capture — which the - platform serves as JPEG under whatever name the command was given — could never be compared. Each - input is now decoded from its own bytes, so the container is sniffed and a `.png` name holding JPEG - decodes as JPEG. `png-transcode.ts` became `screenshot-image.ts`, the one owner of that sniffing for - both the decode and the provider transcode path, and the PNG worker gained a `decode-image` job that - answers pixels instead of PNG bytes. The `--out` diff image stays PNG, as do the crop, overlay, and - resize passes that rewrite a screenshot in place and could not survive a lossy container. - -- Added (maestro): `setPermissions` and `launchApp.permissions` support `all: allow|deny|unset` - on iOS simulators and Android, with specific entries overriding `all`. Both accept a - Maestro-style permissions map. Entries apply in order until the selected platform refuses one, - and the error then names what landed. Android's only allow level is while-in-use, so - `location: inuse|never` apply `allow`/`deny` there; `location: always` and `photos: limited` - stay Apple-only. -- Fixed (ios): `settings permission` no longer refuses a privacy service that `simctl privacy` - accepts but omits from its own help text — Xcode 26 does that for `camera`, so - `settings permission grant camera` and a Maestro `camera: allow` failed as unsupported while - the raw command worked. Support is now `simctl privacy`'s own verdict per runtime, and a - service it refuses fails with `UNSUPPORTED_OPERATION`. This also drops the cached - `simctl privacy help` probe and the `privacy help` spawn before the first permission change. - iOS `all` still does not cover notifications: current runtimes have no notifications service, - so a targeted notifications change fails loudly and `all` leaves it unchanged. -- Added (limrun): `record start` and `record stop` on Limrun iOS and Android direct sessions. The - runtime declared recording unavailable although the Limrun SDK exposes a server-side recorder. - Start asks the instance to record (`--quality medium` maps to Limrun quality 5, `high` to 8); - stop asks the instance to stop once, then downloads the served MP4 to the output path as a - separate bounded step, so a dropped transfer is retried by the next `record stop` without a - second remote stop. The capture is always the whole simulator or emulator screen; `--fps` and - `--hide-touches` are refused, and a recording cannot be reattached after a daemon restart. The - web recorder now shares the same transport-recording runtime. -- Added (ios): `action-button` presses the iPhone or iPad Action Button once, and `client.command.actionButton()` - does the same. XCUITest exposes `XCUIDevice.press(.action)` with no hold-duration overload, so there is no - `long-press` and no `--duration-ms`, and the slide surface on the same edge is Camera Control, which this - command does not drive. The press is dispatched without the runner's app-activation preflight and with no - post-action observation, so the session app keeps the state the press found; the runner asks the device for - the button with `hasHardwareButton(.action)` and refuses with `UNSUPPORTED_OPERATION` on a model that has - none, and Android, HarmonyOS, Vega, Linux, web, tvOS, macOS, and visionOS each state their own refusal. - Simulators run no Shortcuts or App Intents, so what a press triggers is verifiable only on a physical - iPhone (#2699). -- Changed (all): `home`, `app-switcher` and `action-button` are one system-button family in the - runtime contract. An owner without a button now refuses it with the family's hint (for example - `Android has no key event for this system button.`) rather than a per-button sentence, and a - Limrun or WebDriver session that is no longer active refuses `action-button` with that session's - own reason like every other cell; the command name still leads the `UNSUPPORTED_OPERATION` - message. -- Fixed (android): `clipboard read` and `clipboard write` stop reporting success on a build whose - clipboard service has no shell command. Android 16 (API 36) answers every `adb shell cmd clipboard …` - with the framework default `Binder.handleShellCommand` — `No shell command implementation.` on - stderr and exit status 0 — and the exit status was consulted first, so a read reported `text: ""`, - a write reported "Clipboard updated", and `capabilities` advertised `clipboard` on a clipboard no - adb call ever touched. The response is now classified per stream into one typed verdict shared by - both legs and the capability probe: the service's own sentence on `stderr` outranks a clean exit - (a service that never ran wrote no payload to `stdout`), while `stdout` stays payload until the call - has failed. Such a device now refuses with `UNSUPPORTED_OPERATION` and a hint naming the substitute, - and the exported `readAndroidClipboardWithAdb` / `writeAndroidClipboardWithAdb` helpers reject - instead of resolving an empty string or nothing at all (#2674). -- Fixed (limrun): `screenshot` on Limrun iOS direct sessions writes a PNG. Limrun serves its capture - as JPEG and the interactor wrote those bytes straight to the `.png` path, so every capture failed - downstream with "Screenshot file is not a valid PNG". The bytes are now sniffed and a JPEG is - transcoded to PNG before it is written; a PNG passes through unchanged. -- Changed (ios): a regular `snapshot --depth N` on the XCTest runner is a presentation cut over a - full acquisition, not a bound on the walk. Acquisition publishes the frames the platform reported, - one normalization pass turns geometry into the app's orientation space and recomputes `hittable`, - and the visibility fold and depth cut run on that array. Presented trees are unchanged on the - screens measured; a boundary container under `--depth N` now carries its scroll hints from its real - children. The runner's regular-depth capability is `presentation-cut` (was `presented-frontier`), - and every `CGRect` becomes a `SnapshotRect` through one initializer (#2661). -- Fixed (daemon): `close` now stops an active app-log stream (and audio probe / perf capture / - recording) on an implicitly cwd-scoped session. Teardown addressed those resources by - `session.name` (`default`) instead of the store address (`cwd::default`), so the record - read as missing, the `log stream` child leaked, and the next `logs start` on that device failed - with "has not reached a confirmed terminal state" (#2647). -- Fixed (ios): Simulator AX bridge snapshots report `enabled`. The bridge requested no state - attribute, so a disabled control — a React Native `Pressable` with `disabled`, for example — read - as a plain button while the XCTest runner answered the same screen with `enabled: false`. The - bridge now reads the element's accessibility traits (sent as a decimal string, since the word has - bits past 2^53) and derives `enabled` from `UIAccessibilityTraitNotEnabled`. Source version - `agent-device-simulator-ax-v1.6.0` rebuilds the cached bridge on first use. -- Changed (ios): a node the source declares disabled is presented `hittable: false` even when the - capture has no hittability evidence, so on the Simulator bridge a disabled control stops counting - as an interactive node and as a Maestro atomic-dispatch candidate. The navigation title affordance - (a disabled title field presented as an enabled Button for the whole row) no longer carries the - field's `hittable: false`; on the XCTest runner path that Button now counts as interactive and - becomes a Maestro atomic-dispatch candidate where it was excluded before. -- Fixed (ios): a landscape iPhone snapshot reports the system keyboard's rects in the app's own - orientation space. iOS hosts `UIRemoteKeyboardWindow` in the device's native portrait space, so its - whole subtree arrived quarter-turned — a key measured 45 pt wide and 72 pt tall at `x 154` in an - 874 x 402 app, drawing a strip down the left edge where the screenshot shows a 724 x 204 band - docked at `y 198`. Rules that read those numbers refused app content the keyboard was nowhere near - and let a tap land on a key. The Apple runner now reads the app's interface orientation at capture - time and publishes every rect under a turned surface host in the app's space, using the exact - inverse of the rotation its synthesized touches already rotate through, so a reported rect and a - performed tap cannot disagree about which pixel is which. The Simulator AX bridge reader has no - interface orientation in its attribute set, so it refuses a capture holding a surface host reporting - the app box quarter-turned — `Simulator AX snapshot unavailable - (window-coordinate-space-unresolved); used XCTest for this capture, which reports captured geometry - in the app's own orientation space.` — and the route sends that one capture to the runner instead of - retiring the app generation, so the next capture of a healthy app still uses the bridge. One table - proves both languages apply one rule: `contracts/fixtures/window-coordinate-space.json` (#2612). -- Changed (ios): the rotation table and the coordinate-space rule moved from the XCTest runner bundle - into the `AgentDeviceSnapshotPresentation` package, where `swift test` replays the golden table - without a simulator. No behaviour change. -- Added (limrun): `longpress` on Limrun iOS direct sessions. The interactor refused it as - unsupported although the SDK exposes the HID primitives; it now holds one touch as a - `performActions` batch of `touchDown`, `wait`, `touchUp`, defaulting to the 800 ms the Android - and Linux interactors use when no duration is given. -- Fixed (limrun): `press --double-tap` on Limrun iOS direct sessions sends both taps in one - `performActions` batch with an 80 ms on-device pause. The interactor issued two independent `tap` - requests, so a network round trip sat between the taps and iOS recognized them as two slow - single taps. -- Added (ios): `type` and `fill` work in the Apple Pay sheet on iOS Simulator instead of failing - with `TEXT_INPUT_NOT_FOCUSED`. `com.apple.PassbookUIService` is served in place like the web - sign-in host (#2438). -- Changed (ios): Simulator captures also probe for `com.apple.PassbookUIService`. It can keep - running after the Apple Pay sheet closes, so a later capture in that app can report - `system-surface-host-lingering` while the process stays alive. -- Changed (ios): the in-place system surface disclosure now names the sheet kind. The web sign-in - sentence changed from "A system web sign-in sheet is presented over the app, so this snapshot - shows that sheet (hosted out of the app process)." to "This snapshot shows a system web sign-in - sheet presented over the app (hosted out of the app process), not app content". The payment host - says "the system Apple Pay sheet" instead. -- Changed (android): a private adb server no longer absorbs a port its caller named. An adb request - that names a server other than the one its route holds — `-P 5037` in argv, or `serverPort` in the - call's options — is now refused with `managed-device-transport-mismatch` before anything is - dispatched, where previously a `-P` in argv was rewritten onto the route's own port. A caller that - asked for 5037 could read a zero exit as an answer about 5037 after the request had run on 15038. - It reaches `createLocalAndroidAdbProvider(device, { serverPort })`, - `runAndroidHostAdb(invocation, { serverPort })`, and the Limrun runtime dependency's adb calls. A - request that names no server, or names the one the route already holds, runs exactly as before, and - ambient adb is unchanged: with no private server named, the caller's argv is what runs, `-P` - included (#2632). -- Fixed (android): a chunked `record stop` (recordings over 170 s) no longer warns that screenrecord - stopped before record stop at the 180 s limit. Rotation always ends every earlier chunk before - stop, so the warning now fires only when the last chunk's recorder had already exited. -- Fixed (android): a snapshot helper that could not prove it released device automation no longer - refuses the next command on the strength of an `adb` call. The old code read the outcome of - `am force-stop` as the fact it was supposed to measure, and on a loaded host that round trip can - outlive its budget while the device is healthy — the shape of #2553 — so the helper process was - already gone and the command still failed with `Android automation helper is still holding device - automation ownership`. Ownership is read off the device now: the probe asks the device shell for - `pidof com.callstack.agentdevice.snapshothelper` and tells it to echo a marker when nothing matched. - A process id is `occupied`, the bare marker is `released`, and everything else is `unknown` — - `error: closed`, `cannot connect to daemon`, `device offline`, or an adb client killed by a signal - before it wrote anything. A release is therefore claimed only by an answer the transport cannot - produce about itself, and no exit status is trusted: `adb shell` answers 0 for a device command that - failed, and an adb that dies by signal leaves the executor inventing an exit code it never saw. A - refusal requires two reads that both name the process, which keeps a helper still inside Android's - exit path from costing a command. Scripts that match the failure reason see - `android_snapshot_helper_runtime_occupied`, which replaces - `android_snapshot_helper_retirement_unconfirmed`. -- Changed (android): a snapshot helper session that reaches ready settles a release the previous - teardown could not prove. `am instrument` force-stops whatever is already instrumenting the helper - package, so a session that reported itself ready is the only helper process the device has left, - and the unproven release went away with the process that owed it; the next command no longer - force-stops the session it has just started because `pidof` happens to be unreadable. A helper start - that fails is also retried after a backoff scaled to how long it spent failing (10 s to 60 s) - instead of on every command, which had roughly doubled command time on hosts where the helper never - starts. And the wait for a started helper to announce itself no longer uses a fixed 10 s: it takes - half of the helper-command budget the capture was built with, 15 s today, which is what had been - pushing devices slower than a capture off the persistent path. On a host where the helper took 12 s - to announce itself, the command used to answer with the one-shot transport and now answers from the - session. The CLI's `--timeout` reaches that wait as its deadline aborting it, not as the number. - -- Fixed: an iOS snapshot whose XCTest query-sweep tier cannot read the screen no longer ends the - runner process. On a live React Native feed (Bluesky Home, images re-rendering) the AX server - rejects each of the sweep's 19 element-type queries with `kAXErrorIllegalArgument`, and XCTest - records every rejection as a test failure worded `Failed to resolve query: ...`. The runner muted - the sibling `Failed to get matching snapshot: ... kAXError...` wording and not this one, and XCTest - ends the test case as soon as the main-thread block that recorded an unmuted failure returns, so - the runner died right after (or in the middle of) every hostile snapshot and the next command - paid a full `xcodebuild` boot. Both wordings are now muted when they carry an AX server code; the - timeout and `Application X is not running` variants keep recording. The sweep also no longer runs - behind an abandoned read: every bounded main-thread dispatch that outlives its slice now counts - as occupying the main thread (one counter, where the tree XPC and the system-modal probe used a - second one of their own), so once the tree tier's viewport read grinds past its 1 s slice the plan - goes straight to private AX and answers instead of queueing the sweep, the post-snapshot mark, and - its own bookkeeping behind seconds of main-thread work. A capture that fails outright while that - work is still grinding queues its cached-target drop behind it instead of waiting a second for it, - and a fresh process's first capture is no longer penalized by the tree slice timeout, which had - bypassed the warmup exemption every other penalty honors. The per-bundle penalty and - accepted-depth memory that make later captures of the same screen cheap now survive, because the - runner does. -- Changed (android): the snapshot helper release manifest no longer carries `installArgs`, and the - helper installs with a fixed `adb install -r` like the IME helper. The array only ever spelled - `install -r` plus the `-t` that #2603 retired with the `testOnly` flag, so the manifest → flag → - option → flag round trip and its allowlist carried nothing. Older manifests that still contain - the field parse unchanged; the field is ignored. The adb provider `install` capability now takes - only `replace` (#2364). -- Changed (android, harmonyos): every `adb shell`, `adb exec-out`, and `hdc shell` command now goes - through one device-shell funnel that renders each argument for the quoting its transport applies - (#2026, #2611). The device shell re-parses the arguments it is handed, so an unquoted dynamic value - was a command injection: Android quoted a few sites by hand and HarmonyOS quoted none. The two - transports do not quote alike, which is why a caller names the transport (`'adb'` or `'hdc'`) rather - than choosing quoting. `adb` escapes nothing, so a word reaches the device inside whatever quotes it - is given. `hdc` wraps every element it forwards in double quotes, where a single quote is inert while - `$`, a backquote, and `"` stay live: on a nova 14, `fill` text containing `$(id)` came back as the - device's own `id` output and a `"` in the text ended its argument, so words on that transport are - escaped instead of wrapped. Four effects are visible on a device. HarmonyOS `type` and `fill` text now - reaches `uitest uiInput inputText` byte-identical, confirmed with a payload carrying quotes, a command - substitution, a pipe, and a redirection. On Android an empty argument renders as `''` rather than - vanishing from the command, so a command that read its own shift and its operand as two words no - longer misaligns; HDC drops an empty element on the way to the device and cannot carry one, so it is - refused with `details.reason` `hdc-empty-word-unsupported`, as is a script fragment, which could only - arrive there as literal text. A script body handed to `sh -c` on the adb transport arrives as one - argument. Custom adb executors and providers (SDK, MCP, and relay transports such as Limrun's) that - pass a raw `['shell', …]` or `['exec-out', …]` argv are now refused with `INVALID_ARGS` and - `details.reason` `unguarded-device-shell-argv`, because a command the transport is about to let the - device parse has to be one the funnel built. Build it with `runAndroidShell` / `runAndroidExecOut` - (a `DeviceInfo`) or `runAdbShell` / `runAdbExecOut` (an `AndroidAdbExecutor`), all exported from - `agent-device/android-adb`, and pass each dynamic value as its own word. SDK: `AndroidAdbExecutor` - and `AndroidAdbProvider.exec` now receive `readonly string[]`, so a custom executor annotated - `(args: string[])` must widen its parameter to take the command. -- Fixed: BrowserStack sessions honour `--provider-project`, `--provider-build`, and - `--provider-session-name`. The capability builder emitted the legacy JSON Wire keys `device`, - `os_version`, and `app` at the top level next to the W3C `bstack:options` block; the hub treats a - request carrying any legacy key as a legacy session, reads the labels from the legacy top-level - `project`/`build`/`name` (never set), and ignores `bstack:options`, so every session landed in - "Untitled Project" / "Untitled Build" with an empty name — even after #2495 carried the flags to - the provider (#2494). The builder now emits `appium:deviceName`, `appium:platformVersion`, and - `appium:app` and no legacy key, verified against live App Automate sessions. -- Added: `--provider-appium-version ` (alias `--appium-version`) pins the Appium server - BrowserStack runs for the session, as `bstack:options.appiumVersion`. Unset, BrowserStack falls - back to Appium 1.x, which predates the `mobile:` commands the interactor issues (`deepLink`, - `pressButton`, `activateApp`). -- Changed (iOS runner): what recovers a stuck runner is decided by the recorded error, not by its - wording. A readiness preflight marks the error it gives up with, and that marker is now the whole - test for restarting the session and replaying the command — except for a request that was canceled, - which that same catch also marks: a command nobody is going to send again has no restart to spend, - and the session it would tear down may be one that still works. Two message checks decided it before, - and a preflight reaches the caller in whatever shape its connect loop ended with — "Runner did not - accept connection", "Runner endpoint probe failed", a killed `simctl` fallback, a post that ran out - of its budget — so only some of those restarted and the rest failed the command. The other half is - what no longer happens: when a slow - boot spends the whole prepare budget, the health check reports "prepare ios-runner timed out", and - that no longer wipes a restored `xcodebuild` artifact on the way to a rebuild — the runner session - is invalidated and prepare retries with the artifact intact. Only a failure that indicts the - artifact rebuilds it, so a runner that refuses the connection or never answers on any route still - wipes it and rebuilds, which is what that rule is for. -- Added (record): `record stop` now answers the question a video cannot — whether its recorder - actually stopped — beside the export. The result carries `recorder` and `nativePathDisposition` on - CLI `--json`, the Node client, and MCP (ADR 0024). Today a stop says `confirmed`, or `lost` with - `owner-session-lost` for an Apple recording whose session was invalidated, and names its artifact - path `retirable` or `retired`; the wider vocabulary those two fields declare (`unconfirmed`, the - identity-mismatch reasons, `pending`) arrives with the later ADR 0024 steps that gain the probes - those states describe. Both are disclosures about the recorder and the path it writes to, not - failures: the export is served either way, and a stop with nothing to report — an older session's - replay, or a backend whose recorder writes the served file itself — omits them. No stop changes - outcome in this release; the fields land first so the coordinator can report what it already knows - before it starts acting on it. -- Fixed (record): an Android stop credits the device with retiring its recording only when the - device proved the chunks gone. The probe read any failed `test -e` as "not there", so an adb that - timed out or a device dropped mid-call turned chunks that were still on the device into - `nativePathDisposition: "retired"`, and a reattach that could not question the device at all - declared the recording's artifact lost. A probe that never ran now answers uncertain: the path - stays `retirable` and the recording stays finishable until the device answers. -- Changed (record): an iOS Simulator recording no longer has the recorder write the file the caller - asked for. `simctl` records to a `.native.` sibling; `record stop` copies that file to a - `.collected.` sibling, checks that the copy has an MP4 container, writes the caller's - `--out` path once from the copy, checks that export is a playable video, and retires the recorder's - own file (ADR 0024 2.3). An export the finalizer refuses is removed, so `--out` only ever holds a - video that passed that check. The - caller's path is never a file a recorder is still writing, and a stop that fails before the export - exists leaves it absent with the copy in place — so the next stop resumes from the copy instead of - signalling a recorder that already stopped. Both siblings are deleted on success, and a stop that - got that far answers `nativePathDisposition: retired` rather than the `retirable` it had to answer - while the recorder's file and the served file were one file. A recording whose stop fails and is - then abandoned (session close, daemon restart, forced cleanup) keeps its video in these siblings - beside `--out`, where the recorder used to leave it at `--out` itself; they are the only copy of that - video, so nothing deletes them. -- Changed (record): an Android recording is no longer pulled onto the path the caller asked for. - `record stop` pulls the device's chunks to `.collected.` siblings, retrying each pull - until it plays, then writes the caller's paths from copies of them and drops the collected set once the - finalization is journaled (ADR 0024 2.3). Chunk paths the finalizer refused are removed. A stop that dies before the export exists leaves it absent with the pulled set in - place, and the next stop resumes from that set instead of signalling a `screenrecord` process that - already stopped and pulling a file that may have moved since. The 180s platform-limit disclosure now - travels with the recorder's own observation rather than with the export, so a stop that has to be - driven again still says it; every disclosure is still in the one `warning`. A retried stop serves - every chunk and the `capturedDurationMs` its first attempt finalized, and measures the idle tail - against the instant that attempt signalled the recorder rather than the time of the retry. A stop - that fails and is then abandoned keeps the pulled set on the host beside `--out`. Chunk paths and - `--client-output-path` naming are unchanged. -- Fixed: a process lock that could not be given back no longer waits for the daemon to restart. A - release that cannot verify ownership — a refused `unlink`, an unreadable record — left its record - standing and naming the live daemon, and the next acquire read that as a live owner and timed out - after 30 s on every runner build or launch until the process restarted. The claim inside is spent - the moment the release is asked for, so a reclaim here now reads it as dead and takes the path - back, and the failed release is recorded in the request log as `process_lock_release_unverified`. - A record is only read that way when this loading of the lock code issued it: a second bundled copy - of the module in the same process shares the pid and cannot have a live claim cleared under it. -- Changed (lock errors): code that takes a process lock now answers one question in one helper, which - of its two failures the caller hears. The work inside the lock outranks a lock that could not be - handed back, so a build or a publish that failed keeps its own error instead of being replaced by - `Timed out waiting for …`, and work that succeeded still reports the lock it could not give back. - The Apple runner's artifact, cache, lease and disposal paths, the managed-allocation store, the - device-claim store, the iOS snapshot bridge cache, the Swift recording cache and the agent-browser setup - moved onto it, replacing hand-written try/catch pairs that each chose differently. -- Fixed: the redirect of `~/Library/Developer/XCTestDevices` gives itself back in one order — restore - the host's own device set, then release the lock — and a restore that was refused is what the caller - is told, whichever give-back door a teardown used. The lock's own complaint used to replace it in a - `finally` and the best-effort door then dropped it, leaving the symlink pointed at the agent-device - simulator set with nothing said about why. A simulator whose set already is `XCTestDevices` is no - longer failed by a lock it could not verify, either. A leftover from an interrupted build — the host's - set renamed aside and `XCTestDevices` symlinked into a simulator's own set — is now put back before the - redirect decides whether it is needed, so the first build after an interruption still gets its own - device set instead of the host's. A redirect that could not be installed reports the restore that failed - with it, and names a backup path only when that backup is really on disk. -- Changed (sessions): the implicit session is now keyed by workspace **and platform**, so one checkout - - can drive iOS and Android without inventing a `--session` name for every command (#2580). An - implicit session was addressed by `cwd::default`, one slot per checkout, and it stayed - bound to the first device it touched. A repo that tests both platforms — a visual-regression run - across `react-native-paper` components, for example — could not open the second one at all: after - `open --platform ios`, `open --platform android` from the same directory failed with `INVALID_ARGS` - ("already bound to apple device \"iPhone 17\""), and so did `boot --platform android`, which binds - nothing and only needed a device to start. The hint was correct (`--session android` works), but - following it means threading a hand-written name through every command of both legs, and a script - that drops one flag silently talks to the other platform's device. `--platform` now selects the - implicit session, so the platform a script already declares is the only handle it needs: - `cwd::ios` and `cwd::android` coexist, each with its own device, app, and - artifact directory. Three rules keep the behavior that was already earned. - 1. A platform-naming request joins the workspace's session only when it genuinely agrees with it, - so a session opened without `--platform` — which keeps the platform-less `default` leaf — stays - reachable from commands that do name a platform and keeps writing to its own artifact directory - instead of growing a twin beside it. A request that agrees on platform but names a different - device or target still gets the existing `INVALID_ARGS` refusal; it does not silently fork a - second same-platform session that one `--platform` would then have to disambiguate. - 2. A request naming no platform joins the workspace's only implicit session — what every - single-platform script does today, so those are unaffected — and refuses with `AMBIGUOUS_MATCH` - when the workspace holds several, naming every address and the `--platform`/`--session` flag - that selects it, instead of choosing a platform by open order. Same for a broad selector such as - `--platform apple` that matches both this workspace's iPhone and its Mac. - 3. Inventory commands — `session list`, `devices`, `doctor`, `capabilities`, `apps`, which the - registry classifies as `sessionKind: 'inventory'` — never claim a session and keep resolving an - address through the ambiguity. Refusing those would refuse the command an agent runs to resolve - it. `events` and `close` are deliberately not in that set: each reads or tears down one session, - and a silently-empty `events` or a `close` that closed nothing would be worse than the refusal. - Two adjacent repairs were required. A settling interaction published the live session under - `SessionState.name`, which for an implicit session is only `default` and not the address it answers - to, so `agent-device session list` already reported a phantom second row beside the real session; - under platform keying that phantom would also have read as a second session the caller has to - disambiguate, so the publisher now stores under the key the store itself owns, and workspace-session - counting collapses one session reachable under two addresses regardless of writer. And - `--session-lock`'s platform reaches `flags` only after routing has chosen the key, so routing now reads `meta.lockPlatform` too — otherwise a - lock-policy caller, the exact audience that avoids naming sessions, would land on the wrong leaf. - Explicit `--session ` is untouched and still addresses one session verbatim. - -- Fixed: a `record stop` that could not produce its export no longer destroys what its own retry reads - back (#2591). The shared coordinator compensated every failed finish by force-cleaning the resource, - which on Android deleted the device-side MP4 and the native manifest the hint just told you to re-pull: - the second `record stop` had nothing left to collect and the recording was gone. Each durable kind now - declares what its retry needs. `screen-recording` and `perf-capture` preserve, so a failed stop keeps - the device artifact, keeps the manifest `open`, returns its own error, and the next `record stop` in - that session exports it — proven on an emulator whose `adb pull` failed its first attempts and then - exported the 90 KB the first stop left on the device. `app-log` and `audio-probe` say - `dispose-on-failed-finish` themselves, because their retries re-read a log file and a status file that - cleanup never touches. A finish also states why it was asked for: `capture` may preserve, while - `disposal` — session teardown, the one caller that owns forced cleanup — disposes whatever a failed - finish left. Two Apple paths had to become genuinely retryable for that to be true: the runner path - memoized its refused stop and handed the same rejection to every later `record stop` without asking - the runner again, and the simulator path rejected a recording purely because `simctl recordVideo` had - exited non-zero — discarding a video the recorder had already written and leaving an exit code no - retry could change. A stopped recorder is now an observation rather than a failure: the exit is - disclosed on the completion and the file is collected, so those recordings export instead of erroring. - Expect one trade: while a preserved recording is open, `record start` on that device is refused until - its session runs `record stop` or closes. A simulator recording no retry can rescue now names the - exit that made it unreadable — `simctl recordVideo exited with code 1`, `was killed by SIGKILL`, plus - the recorder's stderr — drops its retriable flag, and points at closing the session, while a recorder - still finalizing its file keeps the retry hint. `perf stop` no longer memoizes a refused finish, so its - second attempt re-pulls the trace the first one preserved. - -- Changed: `record stop` no longer carries a start-trim step no recorder could arm. The trim cut the - interval between recorder start and target-app readiness, but the runner's `recordStart` answer has - never carried either timing and the simulator path supplies none, so every built-in recording took - the untrimmed branch while the packaged Swift trimmer, its contract fields, and the telemetry - timestamp shifting stayed shipped with it. Gesture telemetry now carries the timestamps it recorded; - recorded videos are byte-for-byte the videos those paths already produced. Overlay burn-in, output - stability checks, and the recorder-start gesture-clock anchor are unchanged (#2584). - -- Fixed: an iOS capture that no backend could read now says which backend was asked and what was on - screen, instead of failing on the internal `regular iOS snapshot presentation requires a valid - viewport` invariant alone (#2560). The runner declares such a payload sparse — backend, reason code, - and its own reason — and the daemon read that verdict, then discarded it while reconstructing a - viewport from a synthetic root that has none. A `snapshot --actions` capture of a web sign-in sheet - therefore reported a viewport problem and named neither the private-AX backend that cannot read an - out-of-process surface nor the surface presented over the app. The verdict now travels with the - refusal as `error.details.snapshotQuality`, and the hint composes the shared sparse-capture advice - with the bundle id of the surface host when one is presented. On `replay` and `test` the verdict now - survives into the `REPLAY_DIVERGENCE` details, which carried only four cause keys before. Captures - that used to succeed are untouched: a sparse payload whose tree presentation can still serve presents - exactly as before, and a payload the runner did not declare sparse keeps the plain invariant byte for - byte. - -- Fixed: skipped `optional: true` Maestro steps are no longer invisible on the human surface - (#2560). The warning a skip leaves now travels with the run whether it later passes or fails: - `replay` prints a `Warning:` line after its summary and repeats the run's warnings after a - failed run's error, `test` prints a `Warnings:` section after the suite summary naming each - test, and a failed test result gained the `warnings` array the passing result already had, so - `--json` and JUnit carry the skipped steps of a failing test too. -- Fixed: an Apple runner presentation refusal now carries the registry identity of the system - surface the tree was acquired from as `error.details.systemSurface` (#2560). Previously a - selector-backed command failing at the capture boundary under `optional: true` gave nothing - naming that boundary — the web sign-in sheet out of `SafariViewService` was invisible in the - error, and the miss looked like a selector problem inside the app. The sparse-declared case - already names the surface host in its hint via #2572; this covers the provenance everywhere - the runner reports one. -- Changed: a capture that a backend cut at one of its limits now says so in the snapshot's - warnings, on every platform, instead of only setting `truncated: true` in JSON. The text path - had no disclosure at all, so an agent read a screen missing its footer, tab bar, or the items - after a long list as complete — the backends walk the tree in document order, so what falls - off is what comes last, on screen or not. One shared warning renders from the shared flag; the - limit and dimension stay backend-side. The depth-cap warning no longer suggests `--scope` as a - way to read deeper: on iOS, scope narrows the presented view and acquisition stays scope-blind. -- Changed: the iOS Simulator AX bridge caps a capture at 5000 nodes, up from 1500, the Android - helper's bound. Measured on a synthetic 600-row screen, acquisition time did not move with the - cap (the native read fetches the whole tree; the cap only stops conversion) while the 1500 cut - dropped the screen's on-screen footer. -- Fixed: Android snapshots carry the accessibility `selected` state an app sets on a control, so - `is selected`, a `selected=true` selector, and a Maestro `selected:` qualifier work on Android - (#2462). The helper never serialized the attribute, and the host reads only the helper's XML, so - no later layer could recover it: `get attrs` had no `selected` field, no snapshot - node was marked selected, and the same assertion that passed on iOS failed on Android with - "Maestro visible condition did not match" for an element that is visible — while `selected: false` - matched every Android node. The helper now emits both answers, like `enabled` and `password`, so - an unselected control answers `false` and a helper older than the attribute answers nothing; the - host parser, the Android hierarchy node, and the published snapshot node carry it through to - `get attrs` and the `[selected]` marker in snapshot text. Snapshot lines now render that marker - whenever selection is rendered, not only when text surfaces are summarized: `--settle` and `diff` - already compared selection, and a line that compares a fact it cannot display turns a tab tap - into a changed pair whose two lines look identical. -- Fixed: Replay test artifacts with colliding filenames retain distinct copies without overwriting - other diagnostics, replay sources, timing traces, or attempt manifests. -- Fixed: Custom test reporters reject invalid exit codes, including values such as `256` that - could wrap to success and hide a failing suite. `getExitCode` accepts integers from `0` to `255` - or `undefined`; JSON output reports an invalid code as one `INVALID_ARGS` error. -- Fixed: Android `record start` no longer refuses to begin after a reused emulator reassigned the - previous recorder's pid. A completed recording's native marker is retired only once its recorder is - proven gone, but only an absent pid counted as proof — a pid that now names an unrelated process, - or a recorder that exited and waits to be reaped, did not. `record start` then failed every later - attempt with `Android screenrecord completed evidence cannot be safely retired`, and `record stop` - could not return an already-finalized recording. Proven termination now retires the marker and - returns the stored completion; a live or unreadable recorder still blocks, and the unrelated - process is never signalled. A reused pid that runs a replacement `screenrecord` on the same - remote path proves the old recorder gone but not that the path is free, so that marker and - artifact are retained until the replacement ends, and neither is signalled (#2476). -- Fixed: Android `record start` no longer refuses forever on an emulator re-adopted under a new - serial. The device-side marker records the device identity that wrote it and a later start retired it - only when that identity matched, so a marker left by an earlier session on the same AVD blocked every - recording on the re-adopted device with `Android screenrecord native recovery evidence already exists` - — reported as an internal error whose hint asked for a bug report — while `record stop` answered that - no recording was active and no session could reach the marker, because recovery always binds the - device identity its own record names. A marker now retires when its recording is terminal or when it - names a device identity this transport can no longer address, and only once every recorder it names - has provably released its artifact; an interrupted launch commits no recorder identity, so its - artifact is checked against the recorders running on the device first. A recorder that is still - writing is left alone: `record start` refuses with `DEVICE_IN_USE` and `details.writer` naming whether - the marker's own recorder or another one holds the path, and only the unmanaged one ends on its own at - Android's 180 second limit. A device that cannot answer that question — an unreadable process table, - or a candidate process whose identity cannot be read — is refused rather than assumed free, even when - other recorders writing that path were identified, so an artifact is never removed under a recorder - the probe failed to see. What still refuses — unreadable or undecodable evidence, a marker the other - transport mode wrote, and an open recording this device identity still owns — is now a typed error - naming the marker path and the command that clears it, `record stop --session ` or removing the - marker once no session owns it, instead of `UNKNOWN` (#2550). -- Fixed: a polling `wait` no longer surrenders its whole budget the first time the iOS runner - answers `RUNNER_BUSY`. That code means an earlier command exceeded the runner's execution - watchdog and its abandoned main-thread work is still draining, which clears on its own, so a - `wait text ... 20000` could fail in under a second with a failure the runner itself asked the - caller to retry. A poll refused with a failure its producer marked retriable is now ridden out - like an unreadable capture: the wait keeps polling to its deadline, records the poll as - `retriable` in its timeout evidence, and surfaces the refusal only if no readable capture ever - completed. That surfaced refusal keeps the producer's code, message and retry details and now - also carries the wait's own `captures`, `readableCaptures`, `waitedMs` and `polls`, so a budget - spent entirely on refusals is distinguishable from one immediate refusal. A wedged runner - (`RUNNER_WEDGED`) is not retriable and still ends the wait at once. -- Fixed: a runner failure recovered from the lifecycle journal after its transport response was - lost is now classified exactly like the same failure on a live response. `RUNNER_BUSY` reached - callers as a bare `RUNNER_BUSY` wire code without the `retriable` flag on that path, while the - live path published it as `COMMAND_FAILED` with `details.runnerErrorCode` and `retriable: true`; - both paths now read the runner's code through one classifier. -- Fixed: iOS Simulator snapshots of Safari and of apps with a `WKWebView` stopped showing the - page in 0.21.0 — chrome plus empty `[webview]` nodes, no links, text, or form fields, so no ref - could reach the page (#2484). The host AX bridge that 0.21.0 made the Simulator's snapshot source - reads one process, and WebKit content lives in another; the bridge delivered the boundary as - an `AXRemoteElement` leaf and the tree was published as if that were the screen. The source now - refuses a tree whose on-screen web view ends at that leaf (`remote-content-boundary`) and the - route serves XCTest, which resolves remote elements, for the rest of that app generation — the - same path 0.20.x used. The snapshot discloses the switch through its warning, and a relaunch - re-enables the bridge. -- Fixed: `test` expands relative globs from the caller's literal working directory, so directory - names containing glob characters no longer cause missing suites or select a different directory. - Missing non-glob inputs also retain their not-found error in these directories. -- Fixed: JUnit reports remain readable when replay results contain characters forbidden by XML 1.0, - replacing them with U+FFFD while preserving legal Unicode and whitespace. Original suite values - remain available in JSON and other reporters. -- Fixed: `replay export` preserves deep links without `//`, including `tel:` and `mailto:`, as - Maestro `openLink` commands in both standalone and app-plus-link `open` actions. -- Changed: a command whose synopsis is generated names each option with the label its declaration - carries, so `snapshot` now shows `--depth, -d ` and `--scope, -s ` where it used to - show the short aliases, and `--record` is documented under `Command flags:` instead of inside the - `snapshot` and `is` synopsis lines. `snapshot`, `proxy`, `daemon`, `device`, `doctor`, `prepare` - and `tv-remote` no longer restate their option list in a hand-written usage string, so adding an - option to those commands updates `--help` on its own (#2444). -- Fixed: iOS `--depth` on `snapshot`, `is`, `wait`, `get`, and `find` no longer fails with - `regular iOS snapshot presentation requires a valid viewport` when the runner plan is pinned or - deferred to the private AX backend (custom actions, a private AX verdict on the session, or the - XCTest channel penalty). The runner refused a regular depth-capped request on every backend but - the recursive tree, fell through to its synthetic sparse root, and the daemon rejected that root - as a missing viewport. Presentation applies the presented-depth cut to whatever hierarchy a - backend acquired, so every backend serves the request; the private AX declaration is now - `regular-depth=presentation-cut` and an acquisition that stopped short of the cut keeps - disclosing that through `truncated`/`effectiveDepth` as it does unscoped. -- Fixed: repeated unfiltered Android snapshots stay compact when identical element bounds arrive - with a different property order. Changes to the bounds still re-emit the tree. -- Fixed: iOS `network dump` no longer omits requests that reused a keep-alive connection. - CFNetwork logs a request URL only on the line that opens a connection, so a second request to - the same host produced no `url:` line and was dropped from the dump entirely — an "this endpoint - was called" check read as a definite fail. Such a request is now reported against the origin its - connection was opened for, with `pathUnavailable` set, its status, and its timing. A reused - request whose connection was opened before the scanned window cannot be named at all; those are - counted in the dump's `unnamedRequests`, so an empty result still reports that traffic was - observed. The identities behind that count reconcile the app-log and recovery windows internally - — so overlapping traffic is not double-counted and disjoint traffic is not under-reported — but - the response carries only the count, which stays bounded however large the scan window was. The notes say absence of an endpoint does not prove it was not called. -- Fixed: a URL logged as a delimited `url: ,` field no longer keeps the separator the log - format put after it, so an entry's `url` compares equal to the endpoint under test. A bare URL - elsewhere is left alone, since nothing there establishes that trailing punctuation is not part of - the path. -- Added: `replay export` supports flows that switch apps and return, preserving each - `open ` target as an explicit Maestro `launchApp.appId`. -- Added: `replay export` converts recorded `home` actions to Maestro `pressKey: Home`, allowing - app-to-home-to-app journeys to be exported. -- Added: polling `wait` timeouts (`wait `, `wait text`, `wait @ref`, and `wait absent` - after a readable capture) carry a per-poll timeline in `error.details` (`captures`, `polls[]` - with `startedMs`, `durationMs`, and a typed `outcome`: readable, unreadable, deadline, - runner-restart) next to the unchanged `reason`, so a failure says where its budget went without - opening the request log. Long waits keep the first five and last twenty-five polls. The replay - landmark-mismatch refusal carries the same poll evidence next to its mismatch details; `wait - --stable` timeouts and a never-readable strict absence keep their existing diagnostics. -- Fixed: Android `orientation` now returns once the display reports the requested rotation - (polling `dumpsys display`, up to 15s) instead of right after writing the settings. On a loaded - emulator the rotation takes seconds, during which accessibility reads hang, so the next command - paid for the transition; a `wait` issued right after `orientation` could spend its whole budget - there. A display that never reaches the requested rotation now fails the command with the - observed rotation instead of reporting success; a display that reports no rotation is left to - the setting as before. -- Fixed: the iOS Simulator AX snapshot route bounds how long a capture waits for app discovery - and stops starting a discovery per capture. Discovery (`simctl launchctl list` through xcrun) - takes seconds on a loaded host; a capture now waits at most 1.5s for the one in-flight - discovery, takes the XCTest fallback, and the discovery keeps running under its own 15s - deadline for the captures that follow. Previously each capture ran its own probe with a 3s - timeout on its critical path, so a `wait` issued right after `open` could spend its budget on - probe timeouts and report `wait_capture_stalled` with the app already on screen. -- Fixed: iOS snapshots no longer report `truncated: true` merely because a later backend produced - them. The runner stamped every recovered capture as truncated — including a complete private-AX - tree taken while the XCTest channel was penalized as slow — so a strict `is absent` / `wait absent` - refused it with "capture was truncated" on loaded CI hosts. `truncated` now tracks completeness - only: payload truncation, a depth-limited capture, or a sparse terminal payload. -- Fixed: Android `alert accept` / `alert dismiss` return only once the dialog has left the - accessibility tree (a different alert taking its place counts as dismissed), matching the iOS - runner's re-check. Previously they returned right after the button press, so the next read could - still see only the dialog window. A dialog that stays visible past the action budget now fails with - `alert did not dismiss the visible alert`. -- Added strict `wait absent [timeoutMs]` polling for zero selector matches. Incomplete, - sparse, truncated, scoped, depth-limited, and Android unreadable captures cannot prove absence; - deadline diagnostics retain typed capture evidence and stable first-match details (#2236). -- Added: the device-claim store can hold an allocator-held claim (schema v3) for a device an - allocator-managed pool owns. It has no owning process, so `device status` lists it in the normal - view (never as stale), `device release --stale` refuses it with `allocator-held-owner`, the - daemon-startup sweep and session close leave it alone, and a command that binds the device - ordinarily is refused `DEVICE_IN_USE` / `DEVICE_CLAIM_ALLOCATOR_HELD`. Process-owned claim files - are unchanged at schema v2. Two notes for mixed installations: a daemon older than this release - reads a v3 file as an unreadable claim record and fails closed rather than clearing it, and - `devices` reports no `claimedBy` for such a device until the managed-inventory filter lands. -- Added the `harmonyos-instance` lease contract and CLI/runtime plumbing as a prerequisite for - HarmonyOS proxy support; provider/daemon allocation remains gated until its end-to-end lifecycle - is implemented and validated (#2266). - -- Fixed: `settings airplane on|off` now takes an Android device offline. It is applied through - the connectivity service (`cmd connectivity airplane-mode`), which drives the radios, instead of - writing `airplane_mode_on` and broadcasting `ACTION_AIRPLANE_MODE_CHANGED` — a broadcast Android - refuses for non-system callers, so the old path failed *after* writing the setting and left the - device reporting airplane mode with the network still up (#2223). The response now reports the - `airplaneMode` the connectivity service holds after the change, and an Android build that does not - expose that command is refused with `UNSUPPORTED_OPERATION` before anything is written. -- Fixed: the MCP registry entry (`server.json`) now declares the fixed `mcp` subcommand via - `packageArguments`, so a registry-format launcher — the MCP Registry or the website's - `/.well-known/mcp.json` discovery manifest — starts the stdio MCP server. Previously it ran - `agent-device` with no subcommand, i.e. the bare CLI (#2275). -- Breaking (0.21): iOS Appium/WebDriver snapshots now expose engine-owned acquisition facts and - typed fidelity warnings. The SDK snapshot `truncated` field is optional when Appium cannot report - hierarchy completeness; regular snapshots fail closed without valid viewport evidence, while - `snapshot --raw` remains available for diagnostics (#2195). -- Security (daemon, remote/proxy HTTP only): when `AGENT_DEVICE_HTTP_AUTH_HOOK` is configured and a - request's hook result does not attest a `tenantId`, the request is now refused (401) outright — the - daemon no longer runs it as whichever tenant the client declared (RPC body `meta.tenantId` or - `flags.tenant`, or the `x-agent-device-tenant` header on the upload/artifact-download/diagnostics - routes) and no longer admits it unscoped when the client declares nothing either. This closes both - a shared-token impersonation path and an unscoped-access path to tenant-owned sessions/artifacts in - multi-tenant deployments. Deployments with no hook configured (the local loopback CLI) are - unaffected. A hook must attest `tenantId` on every request it wants the daemon to admit. -- Breaking (0.21): removed aggregate performance compatibility (`perf`, `perf sample`, `perf metrics`, the `metrics` alias, optionless `client.observability.perf()`, and SDK `area: 'metrics'`). Use `perf frames`, `perf memory sample`, `perf cpu profile start|stop|report`, or `perf trace start|stop`; removed CLI and raw daemon forms fail with this migration guidance. -- Breaking (0.21): removed legacy batch JSON steps with `positionals`/`flags`. Use `{"command":"...","input":{...}}`; rejected steps now include a concrete structured example. -- Breaking (0.21): removed the deprecated Node client `command.rotate` wrapper and its `RotateCommandOptions` / `RotateCommandResult` exports. Use `command.orientation`; the already-removed CLI `rotate` form keeps its targeted migration error. -- Security (MCP/AI-SDK tool surface): the operator-owned endpoint and path inputs — `daemonBaseUrl`, - the Metro `proxyBaseUrl`, `stateDir`, `cwd`, `iosSimulatorDeviceSet`, `iosXctestrunFile`, - `iosXctestDerivedDataPath`, `iosXctestEnvDir` — follow the credential inputs off the - model-writable tool surface: no longer advertised, refused as explicit input with guidance, and - resolved from env/config only (a model-writable `daemonBaseUrl`/`proxyBaseUrl` would redirect the - env-resolved token to an arbitrary server). Dropping these plus the credential fields shrinks - `tools/list` by roughly half. CLI flags and the SDK client options are unchanged. -- MCP tool descriptions now declare their enforced client timeout envelope (90s default, 180s - install, 300s+ lease allocation, unbounded only for the streaming `test` runner), sourced from - the descriptor registry's timeout policy so the declared number cannot drift from the enforced - one. -- Security (MCP/AI-SDK tool surface): the shared command-tool executor now enforces the advertised - tool schema as an admission boundary — every raw `tools/call` argument must appear in the tool's - advertised (`additionalProperties: false`) schema, or it is refused before config/env resolution. - Hiding a key from `tools/list` alone was insufficient: the router forwards raw arguments verbatim - and the MCP config resolver read `config`/`remoteConfig` as CLI flags, so a model-supplied config - file could load `daemonBaseUrl`/`daemonAuthToken` and redirect the operator's token to an - arbitrary endpoint. Deny-by-default closes that, the operator keys, and any unknown key at once; - operator env/config defaults still resolve (they never arrive as tool input). Retired keys are - still admitted so their migration guidance answers. -- Security (MCP/AI-SDK tool surface): `daemonAuthToken` and the Metro `bearerToken` are no longer - advertised as tool input properties, and an explicit value is refused with guidance instead of - being forwarded. Credentials are operator-owned: set `AGENT_DEVICE_DAEMON_AUTH_TOKEN` (or - `daemonAuthToken` in `~/.agent-device/config.json`) and `AGENT_DEVICE_METRO_BEARER_TOKEN` on the - process serving the tools. The model both reads untrusted app UI text and picks tool arguments, - so a model-writable credential parameter was a prompt-injection exfiltration path. CLI flags - (`--daemon-auth-token`, `--bearer-token`) and env/config resolution are unchanged. -- Release hygiene: after `npm publish`, `release:mark-dev` moves `main` to the next patch with a - `-dev` prerelease marker so the version on `main` never equals a published version (registry - scanners diff the tool surface per version string, and a moving surface under a released number - reads as a republish). `release:prepare` refuses to publish while the `-dev` marker is in place. -- Windows `--platform web` works again. `agent-device web setup` no longer fails with - `npm not found in PATH`, and every web command — including `web doctor` — no longer fails with - `spawn EINVAL`. The managed `agent-browser` backend is now launched as `node ` on every - platform instead of through its `node_modules/.bin` console shim, which is a `.cmd` file on - Windows that `child_process.spawn` refuses without a shell (CVE-2024-27980 hardening); - `shell: true` would only trade that for argument-quoting hazards and a `DEP0190` warning on every - command. Setup spawns `npm` from PATH unchanged on macOS and Linux, and runs npm's own - `npm-cli.js` under the current Node only on Windows, where a bare `npm` is not spawnable. A - managed install now counts as present only when the backend package itself is, and - `web setup --json` / `web doctor --json` gain `entryScript` and `packageDir`; the published - `binaryPath` is unchanged and still names npm's console shim, now informational rather than the - spawned command (#2022). -- Parameterized `fill --record-as` protection is now recording-session-scoped instead of - fill-step-scoped (ADR 0017 amendment): a later, unrelated recorded action (`wait`, `is`, `get`) can no - longer re-serialize an app-rendered echo of an already-parameterized value into its own result or - `target-v1` identity evidence. An echoing `wait` landmark no longer qualifies as an ADR 0016 - destination guard, so `session save-script` refuses it and directs the author to a stable landmark - instead of silently publishing the secret. The protection uses one small, explicit, ephemeral, - never-serialized per-session map populated only from values the author already opted to parameterize; - ordinary non-parameterized recordings are unaffected (#1398). -- Android covered-state publication now has one owner: same-window surfaces remain visible for - diagnosis, while the daemon marks exactly ordered covered controls non-actionable with - `interactionBlocked: "covered"`. Helper-only `drawing-order` stays private rather than entering - the snapshot contract, sparse floating overlays remain usable, and API 23 fails conservative with - the existing `androidSnapshot.occlusionScanUnavailable` disclosure (#1832). -- iOS regular `snapshot --depth` now measures depth after structural accessibility wrappers - collapse. The recursive-tree backend follows a bounded presented-depth frontier, so controls - that fit the requested regular depth are no longer lost behind raw wrappers; raw `--depth` - remains a traversal-depth limit. Flat query recovery is limited to one presented level, and - private AX does not claim deeper regular-depth completeness until it has a hierarchy-aware - frontier (#1797). -- iOS regular snapshot nodes now publish presentation-owned effective geometry through the existing - `rect` field: backend-reported frames remain available to acquisition, while regular output uses - the viewport and declared scroll-clip intersection. Raw snapshots and direct element reads retain - reported geometry (#1797). -- Android recording-session commands now warn when blocking-dialog readiness inspection fails open. The requested command still runs, but the successful response discloses that readiness could not be inspected and preserves the inspection error's actionable hint, including `pnpm build:android` when the snapshot helper is unavailable (#1895). -- Breaking (device selection): when a command needs one concrete device, carries no `--device`/`--udid`/`--serial`, and more than one candidate is equally preferred, agent-device now refuses with `AMBIGUOUS_MATCH` and the candidate list instead of quietly picking one. Established preferences are unchanged — virtual over physical, booted over offline, and the Apple kind/target ranking — so a single booted emulator beside offline ones still resolves, an existing session binding still resolves (its identity is already fixed), and explicit selectors still resolve. What no longer happens is choosing between two equally booted devices by discovery order or alphabetically: that produced a **successful response describing a device the caller never selected**, and reads are no safer than writes there. `devices` and other genuinely multi-device commands never enter singular resolution and are unaffected. The error carries the bounded candidate list in the declared `devices` details domain, so CLI and MCP print it, with a hint naming the right selector for the platform (`--serial` for Android/HarmonyOS, `--udid` for Apple, or `--device ""`). -- `--udid` with `--platform android` (and `--serial` with an Apple platform) now fails as the flag mistake it is — `INVALID_ARGS` naming the right flag — instead of reaching device resolution and answering `No Apple device with UDID emulator-5580` for an explicitly Android request. `--udid` addresses Apple devices, `--serial` addresses Android and HarmonyOS; matching pairs and requests that name no platform are unchanged. -- Breaking (`--session-lock strip`): a device selector that names a different device than the bound session is no longer silently discarded. `strip` exists to drop redundant platform/scope selectors; when it also dropped `--udid`/`--serial`/`--device`, the command kept running against the *bound* device instead of the one the caller named — a wrong-device action that looks like a success. Such a request now fails with `INVALID_ARGS` under both `reject` and `strip`, and the error carries the two identities structurally (`requestedDevice`, `boundDevice`) plus a hint offering the two real recoveries: close the bound session if the requested device is intended, or remove the selector if the bound device is. The hint no longer suggests `--session-lock strip` for an identity conflict, since following that advice is what produced the wrong-device run. Scope-only stripping (`--platform`, `--target`, `--ios-simulator-device-set`, `--android-device-allowlist`) is unchanged. -- iOS regular snapshots now apply one backend-neutral eligibility rule after every capture backend: a node survives when its accessibility type is interactive or it carries a non-empty label, identifier, or value. This removes the tree backend's extra "hittable non-Other" membership path and drops unlabeled decorative nodes consistently; labeled images, identifier-only nodes, and value-only nodes still survive. Raw snapshot membership is unchanged. -- iOS regular snapshots now run one shared clip fold inside presentation for every capture backend (#1797). Backends serialize reported facts -- every traversed node, at raw traversal depth -- and presentation alone decides what the viewport and scroll clips hide, books the scroll hints, and collapses depth; no backend carries its own copy of that interpretation anymore (the copies are what produced the scroll-overflow leak class, #1784). Three intentional edge deltas ride along, all in the direction of one backend-neutral rule: sub-pixel content-free decorations are now dropped by every backend (previously private-AX only); labeled offscreen Application/Window carriers now survive on every backend (previously tree only), still never hittable; and a query-sweep recovery snapshot without `-i` no longer lists offscreen elements. Nothing outside its clip, and nothing without geometry, is ever `hittable` in a regular snapshot, whatever the backend reported. -- iOS `snapshot --raw` is now the acquired accessibility tree on every backend that can serve it (#1797). A raw request that recovered onto the private-AX backend — the route an app whose XCTest tree capture fails takes — returned the *regular* projection's viewport-pruned nodes labeled raw: everything scrolled out of the viewport, and every sub-pixel decoration, was missing from the one view whose purpose is showing what the pruned view hid. Raw now keeps every node the backend serialized, at traversal depth, and `--depth` still narrows it (for raw, presented depth *is* traversal depth). Two structural rules replace the hand-synchronized ones: the raw capture plan is derived from each backend's declared ability to serve raw, so the interactive query sweep — which has no hierarchy to return — cannot be planned for a raw request; and presentation refuses an acquisition captured for the other projection instead of relabeling it, dropping that tier with a structured failure. Breaking in the same direction: `snapshot --raw -i` now returns the acquired tree instead of an interactive-filtered one — `-i` narrows the regular projection, and the pair used to produce a third membership rule that differed per backend. Regular and `-i` output is unchanged. Backends now read one derived capture hint rather than the request itself, so what a capture is allowed to skip is stated once, next to the proof that skipping it keeps the projection complete. -- Android `snapshot --raw` is now the acquired accessibility tree (#1832 C3): the regular-projection classifiers for nodes Android marks invisible and stale application windows no longer run at parse time, so `--raw` keeps everything the helper serialized (normalization only). Covered same-window surfaces are publication annotations rather than membership pruning. Also: Android blocking-dialog recovery now reads the same daemon presentation an agent's `snapshot` sees instead of a hand-rolled subset, and acts on its occlusion result — a stale "App isn't responding" surface left under the foreground one no longer triggers recovery, and a covered "Close app" is never tapped ahead of the visible one; the Android freshness route signature no longer keys on `role`, a field the Android backend never carries; and the Android helper's declared fidelity residues (no `checked`/`checkable`/`long-clickable`, 5000-node cap before scoping, API-level cache-reset divergence) are recorded in `CONTEXT.md`. -- `agent-device mcp` now carries its own usage guidance, so MCP-only clients (Codex CLI, Cursor, custom agents) no longer depend on a separately installed skill (#1833). The handshake `instructions` — returned by both `server/discover` and, newly, the legacy `initialize` — is a compact (< 2 KB, the Claude Code truncation limit) workflow card: start with `open {app, foreground: true}` instead of probing, act with `settle: true` and continue from the diff, verify with `wait`/`is`/`get`/`find`, copy `@refs` byte-for-byte, recover from sparse/AX-unavailable, follow error hints, `close`. A new MCP-only `help` tool serves the full guides on demand: no `topic` returns the CLI's decision card; `topic` returns `agent-device help ` verbatim (workflow, gestures, scripting, tv, macos, web, remote, debugging, …, or any tool name for its complete flag reference), prefixed with the one-line CLI→tool-property mapping. `help` is router-owned rather than a command descriptor, so it appears in `tools/list` only — not in the CLI, Node client, or `batch` — and its description tells the model it is not a startup step. Legacy `initialize` gains the optional `instructions` field; no other legacy field changes. -- Android `snapshot --scope` (and every selector command's `--scope`, e.g. `press "Save" --scope Panel`) now resolves scope exactly once, inside the Android projection, under the shared scope specification: the scope root is the first node **in document order** whose label, value, or identifier contains the scope text (case-insensitive) **and whose subtree still has content in the projection you asked for**, the result is that subtree re-rooted at depth 0, and no match returns an empty snapshot (#1832). That second clause is what makes `snapshot -i --scope panel` return the button inside a structural container `-i` drops, and stops a decorative heading that happens to match from emptying the snapshot. Before, Android ran two passes with contradictory rules — a breadth-first platform match that fell back to the full tree on a miss, then the daemon's document-order pass — so a shallower later container could win over an earlier match, and an interaction capture whose scope reached only the daemon layer was silently unscoped. `--depth` under `--scope` counts from the scope root, filtering the depths the response prints (a node shown at depth 0 is never hidden by `--depth 0`), and ancestor context above the scope root (a clickable row, a list) still shapes `-i` membership inside it. The rule is pinned by `contracts/fixtures/snapshot-scope-policy.json`, the same golden table the iOS runner consumes (#1797). -- New `hover ` command for `--platform web` (#1783). It moves the pointer over the target without pressing, so hover-gated UI — a message row's `...` toolbar, a menu that opens on pointer enter — becomes reachable through agent-device the way it already was through the underlying `agent-browser` backend (`mouse move`). It is a member of the targeted-touch family: same `@ref`/selector/coordinate targeting, occlusion and off-screen guards, and `--settle` (the settled diff carries the revealed controls with fresh refs, e.g. `+ @e4 [button] "Delete"`), but no `--verify`, since hover reveals rather than activates. `hover @ref` publishes as a portable selector line in recorded scripts, and the Node client exposes `interactions.hover`. Hover is a pointer state that touch platforms do not have, so `capabilities` advertises it on web only and iOS/Android/Linux reject it during admission with `UNSUPPORTED_OPERATION` and a hint naming `--platform web`; `longpress` remains the mobile hold-gesture verb. -- Internal: session recording is now derived from the script-publication lifecycle instead of being stored beside it. `SessionState.recordSession` is removed; `isRecordingPublication` answers the question from the aggregate (ordinary authoring records only while ARMED, a repair transaction records for its whole lifetime). The stored flag was a second source of truth that handler surfaces set directly, which is how #1533's aborted-recording drift arose — no surface can now arm recording without moving the lifecycle that authorizes it, and the script writer's publication gate is answered entirely by the aggregate. Behavior-preserving: the derivation reproduces what the flag held at every transition. -- Fixed: a script recording aborted by a second `open` is no longer published by a later bare `close` (#1533). `open --save-script` followed by a second successful `open` terminates the recording and warns "Script publication was aborted…", and `close --save-script` correctly refuses it with "Retry with plain close; it will tear down the session without writing." But when that second `open` itself carried `--save-script`, the flag re-armed recording behind the terminal status, and a bare `close` then wrote the full session log to disk — publishing a recording the caller had been told was aborted, and breaking the promise the refusal makes. An aborted authoring lifecycle is now terminal by construction: `--save-script` arms nothing on any surface that handles it — the re-open builder, the close finalizer, and the recorded-action ingress — and the script writer refuses the lifecycle from every path that reaches it (bare `close`, teardown, idle-reap, active publication). This also stops an aborted session from paying recording-time costs it can never publish: a re-opened aborted recording no longer keeps the direct iOS selector fast paths for `click` and `get` disabled. Armed recordings, published recordings, and every repair transaction are unaffected. -- `agent-device mcp` now serves the stateless MCP `2026-07-28` revision alongside the handshake-based revisions it already spoke, as the spec's "dual-era server". Modern clients probe `server/discover`, which advertises the supported revisions, the tools capability, and server identity; their requests declare a protocol version in `_meta`, and their results carry `resultType: "complete"` plus `_meta["io.modelcontextprotocol/serverInfo"]`. `tools/list` and `server/discover` now return the `ttlMs`/`cacheScope` cache hints, so a client can cache the 55-tool, ~223KB tool list for an hour instead of re-fetching it on every start; the list was already emitted in a deterministic (sorted) order, which is the other half of what makes it cacheable. Each revision is answered on its own wire contract: a request declaring `2025-11-25` or `2025-06-18` through modern framing still gets the legacy result shape, and `initialize` never agrees to `2026-07-28`, which has no handshake to establish. A declared revision this server does not implement is rejected with `UnsupportedProtocolVersionError` (`-32022`) naming the ones it does, rather than being served under a version the client did not ask for, and modern framing that omits its required `protocolVersion`/`clientCapabilities` metadata — or supplies a `clientInfo` that is not a valid `Implementation` — is rejected as invalid params. `initialize` and `ping` were removed in `2026-07-28`, so a modern-framed call to either is answered `-32601` rather than served inside a `resultType: "complete"` envelope. Responses to legacy clients are unchanged byte-for-byte — `initialize` and `ping` are still served, and no cache, `resultType`, or `_meta` field is added to their results. Nothing here affects the CLI, Node, or daemon surfaces: the stdio transport, the tool set, and every tool's input/output schema are untouched. -- Fixed: the MCP `initialize` handshake now answers with the protocol revision the client requested when it is one this server implements, instead of always answering `2025-11-25`. A client pinned to `2025-06-18` was told to speak a revision it had not asked for, which the lifecycle contract answers by disconnecting. -- `agent-device help workflow` is now a compact ~8KB card instead of a ~41KB dump; the same depth still exists, split into `help scripting` (save-script, secret-safe fills, batch JSON, replay divergence/repair, recording) and `help gestures` (multi-touch shapes and platform quirks), plus a few paragraphs folded into the topics that already owned the subject (`help debugging`, `help physical-device`, `help validate`). Every `help ` first line is now `agent-device — ` so an agent can read the installed version from its mandatory first help read instead of a separate `agent-device --version` call. -- Cloud iOS (BrowserStack, AWS Device Farm): `snapshot` and `diff` no longer fail with `SESSION_NOT_FOUND` on a live provider session (#1658). The app-session guard they ran belongs to the local XCUITest runner, which must attach to a target app; a cloud capture reads the provider's own driver session and needs no app identity, so the guard now applies to local Apple targets only. Relatedly, a cloud iOS `open com.example.app` now records that bundle id on the session — the provider path skips local app resolution (no simctl/devicectl reaches a hosted device), and used to drop an explicitly spelled bundle id along with it, leaving the session with no app identity at all. Opening a second bundle id replaces the first, matching the local path, where an explicitly spelled target always wins over the session's current app; deep links, display names, and bare `open` still keep the app already tracked. -- Cloud `fill` (BrowserStack, AWS Device Farm) now witnesses that the field it tapped actually holds text-entry focus before sending its keys, instead of dispatching tap and keys in back-to-back requests (#1658). A WebView input — an OAuth/SSO page in a Safari view controller, for example — takes first responder asynchronously, so the keys used to land with nothing focused while `fill` still answered "Filled N chars"; tapping and filling as two separate commands worked only because the round trip between them gave the field time to focus. The witness is the focused element's own geometry: `fill` polls the active element and proceeds only once it contains the point it tapped, which is the one signal that identifies *which* field took focus. Keyboard visibility cannot — it reads the same before and after a second fill into an already-open form, so it could not tell a focused password field from the email field the previous fill left focused. The response discloses `textEntryReadiness`: `focused-element`, or `keyboard-shown` when the driver has no active-element route but the keyboard rose from hidden after the tap. Both describe a fill that witnessed focus before typing; there is deliberately no value for typing without evidence, because nothing renders this field and such a value would reach a caller as an ordinary success. Breaking: when focus cannot be witnessed, cloud `fill` now FAILS with `COMMAND_FAILED` / `text_entry_focus_not_observed` and sends no keys, instead of typing into whatever holds first responder and answering "Filled N chars" — a fill with no witness must not read as a filled field. That covers a tap that focused nothing, a keyboard already up on a driver that cannot name the focused field, and a driver that reports neither (`text_entry_focus_unobservable`, which points at `press` + `type` as the deliberate way to enter text unwitnessed). Only a positively classified unimplemented route counts as unsupported, so a dead session, an auth rejection, or a grid outage surfaces instead of degrading into a blind text entry. -- Changed mutating selector ambiguity semantics (press/click/fill/longpress): duplicate accessibility wrappers collapse only when every match forms one ancestor-descendant chain resolving to the same actionable node. Matches in distinct subtrees now fail fast with `AMBIGUOUS_MATCH` and a bounded, immediately reusable candidate-ref frame; visible/depth/area geometry no longer silently picks a mutation target. The direct iOS XCTest path now counts raw exact matches before hittability and delegates ambiguity to the same runtime rule. AppControlBench provenance: element-14 ran on 0.20.5; this change is intended for 0.20.7+, and comparative benchmark reports should note that it can replace a wrong-success recovery loop with one candidate-pick turn while occasionally adding that turn for genuinely distinct duplicates. -- `scroll` and `back` now accept `--settle` (with `--settle-quiet` and `--timeout`), collapsing scroll-then-observe and back-then-observe into one call (#1638). The response carries the same settled payload the touch commands return — verdict, changed-lines diff with fresh refs on added lines, the unchanged-interactive tail, and `refsGeneration` when the settled tree was stored — and is best-effort: it never fails the action. One difference is deliberate: `scroll`/`back` resolve no element, so the diff baseline is the session's stored pre-action tree ("the last tree you observed") rather than a freshly resolved pre-action capture. Both commands now also preserve the daemon on timeout, like the other settle-capable commands. -- Security: repository `./agent-device.json` now accepts only project-safe automation defaults. It rejects daemon endpoint/auth/transport/server settings, tenant/run/lease selectors, provider/cloud and Metro connection fields, headers, executable reporter modules, local write destinations, and other operator-controlled values before local module loading or any daemon health/RPC request. Put remote endpoint and token together in protected CI environment variables, user config, an explicit `--config` file, or the existing `connect`/`--remote-config` workflow. Daemon auth tokens no longer travel in serialized command flags. -- `viewport` is now rejected during capability admission on Apple targets instead of reaching the device and failing inside dispatch. No Apple backend can resize a screen — simulator and device geometry is fixed by the selected device type — so `viewport` on iOS/iPadOS/tvOS/macOS now fails with `UNSUPPORTED_OPERATION`, `viewport is not supported on this device`, and a hint pointing at `--platform web` and at picking a different simulator. `capabilities` no longer advertises `viewport` on Apple targets. Web viewport resizing (`agent-device viewport 1280 900 --platform web`) is unchanged, and Android was already denied. -- `--save-script` is now accepted only by the commands that declare it — `open`, `close`, and `replay`. A hand-built daemon request (or a `batch` step) that set `saveScript` on any other command, such as `record` or `trace`, used to arm script publication and could write a `.ad` artifact; it is now rejected with `INVALID_ARGS` before the request reaches admission, the device, or any handler. CLI, Node, and MCP usage of `--save-script` on its documented commands is unchanged. -- `diff screenshot` no longer runs the retired best-effort OCR and non-text analyzers. Their optional `ocr` and `nonTextDeltas` fields remain in the result type for source compatibility but are no longer emitted; use the baseline/current images and diff artifact with vision for qualitative interpretation. -- Breaking: removed the deprecated `--session-locked` and `--session-lock-conflicts` flags. Use `--session-lock reject|strip` instead; passing either old flag now fails with `Unknown flag: ... Use --session-lock reject|strip instead.` -- Breaking: removed the `replay export --format` flag. `replay export` always writes Maestro YAML. -- Breaking: removed the unused `LeaseAllocatePayload`, `LeaseHeartbeatPayload`, and `LeaseReleasePayload` type exports from `agent-device/contracts`. Lease request metadata is fully described by `DaemonRequestMeta`. -- Maestro compat: `assertVisible` and `assertNotVisible` now accept `childOf` for ancestor scoping, matching `tapOn` (#1294). -- Breaking: removed deprecated gesture duration and rotate velocity inputs (#1218). - - `swipe x1 y1 x2 y2` no longer accepts a trailing `durationMs` positional; use `gesture pan x1 y1 (x2-x1) (y2-y1) durationMs` for deliberate timed drags. - - Maestro `swipe` operations with a duration continue to normalize to `gesture pan` with the `endpoint-hold` execution profile, preserving the Maestro-compatible fast-swipe-then-hold behavior on iOS. - - `gesture fling direction x y` no longer accepts a trailing `durationMs` positional; use `gesture pan` for timed movement. - - `gesture swipe preset` no longer accepts a trailing `durationMs` positional; use `gesture pan` for timed movement. - - `gesture rotate degrees [x] [y]` no longer accepts a trailing `velocity` positional; rotation pacing is derived from `degrees`. - - MCP/Node schemas no longer advertise `velocity` or `durationMs` on `swipe`/`fling`/`gesture swipe`; `durationMs` remains on `gesture pan` and `gesture transform`. - - A `.ad` script that still carries a removed positional now fails when the script is parsed, before the replay executes any device action, naming the line and its rewrite (for example `swipe accepts 4 arguments: x1 y1 x2 y2 (line 6). The trailing durationMs positional was removed: use "gesture pan 197 650 0 -350 300" ...`). Previously the script ran up to that step and then failed as a replay divergence. - - Published the [gesture migration guide](https://agent-device.dev/docs/migrating-gestures) covering CLI, Node.js, MCP, and saved `.ad` recordings, plus the deprecation policy the next such removal follows (#1216). - - `replay export` now writes an explicit `duration: 100` for `swipe` — the canonical fling duration — instead of omitting it and letting Maestro apply its own 400ms default. Maestro flows replayed by `agent-device` are unaffected; a timed Maestro `swipe` still normalizes to `gesture pan` with the `endpoint-hold` profile. -- Breaking: the deprecated `rotate` CLI command alias has been removed. Use `orientation` instead; invoking `rotate` now fails with `rotate was renamed to orientation; for the two-finger gesture use: gesture rotate`. -- Breaking (ADR 0014, session ref-frame lifetime): a mutation through an `@ref` now expires the session's ref frame, so a later ref mutation without a fresh observation fails closed with a typed `details.reason` (`ref_frame_expired`, `ref_generation_mismatch`, `plain_ref_requires_complete_frame`, or `ref_not_issued`) instead of acting on a possibly-navigated screen. A ref-oriented sequence that performs several mutations must re-`snapshot` between them, consume an honestly issued `--settle` ref in pinned `@eN~s` form, or use selectors. Enforcement applies on every platform, not just iOS. Legacy hand-written `.ad` scripts that reuse several bare refs from one snapshot must capture between mutations or use selectors. -- Ref reads resolve against the authorized ref frame's source tree, so an internal read-only capture (including Android freshness) can no longer retarget an admitted `@ref` by positional coincidence. Read-only ref consumers keep the structured staleness warning while the frame retains the ref's evidence. - -## 0.15.0 - -- Breaking: `apps` discovery and public app-list helpers now default to user-installed apps. Use `--all` or `filter: 'all'` to include system/OEM apps. -- Breaking: removed the `agent-device/android-apps` public subpath. Use the Android app helpers from `agent-device/android-adb`. -- Breaking: removed the `agent-device/daemon` public subpath. Use `agent-device/contracts` for daemon request/response types. -- Breaking: removed public local ADB bypass/selection helpers such as `spawnAndroidAdbBySerial` and `resolveAndroidAdbProvider`; use `createLocalAndroidAdbProvider(device)` or pass providers directly to the helpers from `agent-device/android-adb`. -- Added Android ADB provider helpers for exec, stream, clipboard, keyboard, app lifecycle, logcat, and port reverse workflows. diff --git a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeXml.java b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeXml.java index 0753f242d8..775d622f7b 100644 --- a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeXml.java +++ b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeXml.java @@ -43,6 +43,9 @@ static void appendNode( // (#2063 empty-fill verification). if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { appendAttribute(xml, "hint-showing", Boolean.toString(node.isShowingHintText())); + // The hint itself, whether or not the field is showing it: `text` holds the hint only while + // the field is empty, so a filled field's placeholder is readable nowhere else. + appendNonEmptyAttribute(xml, "hint", node.getHintText()); } appendAttribute(xml, "editable", Boolean.toString(node.isEditable())); // Accessibility selection offsets, not a measurement of the value's length. Read-only @@ -53,6 +56,8 @@ static void appendNode( appendAttribute(xml, "class", node.getClassName()); appendNonEmptyAttribute(xml, "package", node.getPackageName()); appendNonEmptyAttribute(xml, "content-desc", node.getContentDescription()); + appendNonEmptyAttribute(xml, "role-description", roleDescription(node)); + appendTrueAttribute(xml, "heading", isHeading(node)); appendAttribute(xml, "visible-to-user", Boolean.toString(node.isVisibleToUser())); appendDrawingOrderAttribute(xml, node); appendTrueAttribute(xml, "clickable", node.isClickable()); @@ -64,6 +69,12 @@ static void appendNode( // unselected control reports selected=false and a helper older than this attribute reports // nothing at all. appendAttribute(xml, "selected", Boolean.toString(node.isSelected())); + // Present only on a checkable control, with both answers: an unchecked switch reports + // checked=false, while a node that cannot be checked reports nothing, like a helper older than + // this attribute. + if (node.isCheckable()) { + appendAttribute(xml, "checked", Boolean.toString(node.isChecked())); + } boolean scrollable = node.isScrollable(); if (scrollable) { appendAttribute(xml, "scrollable", "true"); @@ -146,9 +157,21 @@ private static void appendTrueAttribute(StringBuilder xml, String name, boolean } } - // Declared residue (agent-device #1832): checked / checkable / long-clickable are not serialized, - // so toggle state is invisible to agents. Adding them is a helper protocol change (new attributes - // + host parser + fields on the wire node), tracked there. + // The platform node has no role description getter: androidx writes the value an app set + // (AccessibilityNodeInfoCompat.setRoleDescription) into the node extras under this key, and + // TalkBack reads it from there. + private static CharSequence roleDescription(AccessibilityNodeInfo node) { + return node.getExtras().getCharSequence("AccessibilityNodeInfo.roleDescription"); + } + + // isHeading() arrived in API 28. Older releases keep the compat flag in an extras bit this + // helper does not read, so a heading on API 23-27 reports nothing. + private static boolean isHeading(AccessibilityNodeInfo node) { + return Build.VERSION.SDK_INT >= Build.VERSION_CODES.P && node.isHeading(); + } + + // Declared residue (agent-device #1832): long-clickable is not serialized. Adding it is a helper + // protocol change (new attribute + host parser + field on the wire node). private static void appendDrawingOrderAttribute(StringBuilder xml, AccessibilityNodeInfo node) { if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) { appendAttribute(xml, "drawing-order", Integer.toString(node.getDrawingOrder())); diff --git a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java index 9abf3bdb85..6f7742a186 100644 --- a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java +++ b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java @@ -2,6 +2,7 @@ import android.app.Instrumentation; import android.app.UiAutomation; +import android.content.res.Resources; import android.os.Bundle; import android.util.Base64; import java.io.BufferedReader; @@ -120,6 +121,11 @@ private static void putCaptureMetadata( result.putString("nodeCount", Integer.toString(capture.nodeCount)); result.putString("truncated", Boolean.toString(capture.truncated)); result.putString("elapsedMs", Long.toString(elapsedMs)); + // Physical pixels per dp of the display the bounds above were measured on, from the same + // configuration the framework lays out with (a `wm density` override included), so a host that + // works in dp has the factor beside the pixels instead of a second adb round trip. + result.putString( + "pixelDensity", Float.toString(Resources.getSystem().getDisplayMetrics().density)); } private void runOneShotViewport(Bundle result) { diff --git a/apple/fold-helper/Fold.m b/apple/fold-helper/Fold.m new file mode 100644 index 0000000000..f028c6438e --- /dev/null +++ b/apple/fold-helper/Fold.m @@ -0,0 +1,97 @@ +#import +#import +#import + +// Private simulator HID entry points; the host verifies the resulting hinge independently. +extern CFDataRef IOCFSerialize(CFTypeRef object, CFOptionFlags options); +extern CFTypeRef IOHIDEventCreateVendorDefinedEvent(CFAllocatorRef allocator, uint64_t timestamp, + uint32_t usagePage, uint32_t usage, uint32_t version, const uint8_t *data, + CFIndex length, uint32_t options); +extern CFTypeRef IOHIDEventSystemClientCreate(CFAllocatorRef allocator); +extern void IOHIDEventSystemClientDispatchEvent(CFTypeRef client, CFTypeRef event); + +static BOOL sendAngle(CFTypeRef client, double angle) { + NSDictionary *payload = @{ + @"provider": @"com.apple.Virtualization.VirtualMachines", + @"source": @"hinge-slider-control", @"type": @"range", @"value": @(angle), + }; + CFDataRef data = IOCFSerialize((__bridge CFTypeRef)payload, 1); + if (!data) return NO; + CFTypeRef event = IOHIDEventCreateVendorDefinedEvent(kCFAllocatorDefault, + mach_absolute_time(), 0xff61, 0x5b, 0, CFDataGetBytePtr(data), CFDataGetLength(data), 0); + if (event) IOHIDEventSystemClientDispatchEvent(client, event); + CFRelease(data); + if (!event) return NO; + CFRelease(event); + return YES; +} + +static BOOL validNumber(id value) { + return [value isKindOfClass:NSNumber.class] && + CFGetTypeID((__bridge CFTypeRef)value) != CFBooleanGetTypeID() && isfinite([value doubleValue]); +} + +static NSArray *readFrames(NSString *argument) { + NSNumber *preset = (@{@"closed": @0, @"half-open": @130, @"open": @180})[argument]; + if (preset) return @[@{@"atMs": @0, @"angle": preset}]; + id frames = [NSJSONSerialization JSONObjectWithData:[argument dataUsingEncoding:NSUTF8StringEncoding] + options:0 error:NULL]; + if (![frames isKindOfClass:NSArray.class] || [frames count] < 2 || [frames count] > 64) return nil; + double previous = -1; + for (id frame in frames) { + if (![frame isKindOfClass:NSDictionary.class] || [frame count] != 2 || + !validNumber(frame[@"atMs"]) || !validNumber(frame[@"angle"])) return nil; + double time = [frame[@"atMs"] doubleValue], angle = [frame[@"angle"] doubleValue]; + if (time < 0 || time > 60000 || floor(time) != time || time <= previous || + (previous == -1 && time != 0) || angle < 0 || angle > 180) return nil; + previous = time; + } + return frames; +} + +static double angleAtTime(NSArray *frames, double elapsed) { + NSUInteger segment = 0; + while (segment + 1 < frames.count && [frames[segment + 1][@"atMs"] doubleValue] <= elapsed) segment++; + double angle = [frames[segment][@"angle"] doubleValue]; + if (segment + 1 < frames.count) { + double from = [frames[segment][@"atMs"] doubleValue]; + double to = [frames[segment + 1][@"atMs"] doubleValue]; + angle += ([frames[segment + 1][@"angle"] doubleValue] - angle) * (elapsed - from) / (to - from); + } + return angle; +} + +static BOOL runFrames(CFTypeRef client, NSArray *frames) { + mach_timebase_info_data_t clock; + mach_timebase_info(&clock); + uint64_t start = mach_absolute_time(); + double duration = [frames.lastObject[@"atMs"] doubleValue]; + double elapsed = 0; + while (YES) { + @autoreleasepool { + if (!sendAngle(client, angleAtTime(frames, elapsed))) return NO; + } + if (elapsed >= duration) return YES; + double now = (mach_absolute_time() - start) * clock.numer / (double)clock.denom / 1e6; + // Absolute deadlines skip missed frames instead of extending the user's timeline. + double next = fmin(duration, (floor(now * 60 / 1000) + 1) * 1000 / 60); + mach_wait_until(start + (uint64_t)(next * 1e6 * clock.denom / clock.numer)); + elapsed = fmin(duration, (mach_absolute_time() - start) * clock.numer / (double)clock.denom / 1e6); + } +} + +int main(int argc, const char *argv[]) { + @autoreleasepool { + NSArray *frames = argc == 2 ? readFrames(@(argv[1])) : nil; + if (!frames) { + fprintf(stderr, "Expected a pose or 2-64 {atMs,angle} keyframes spanning 0 to at most 60000ms\n"); + return 2; + } + CFTypeRef client = IOHIDEventSystemClientCreate(kCFAllocatorDefault); + if (!client) return 1; + BOOL dispatched = runFrames(client, frames); + CFRelease(client); + if (!dispatched) fprintf(stderr, "Unable to create simulator HID event\n"); + return dispatched ? 0 : 1; + } +} diff --git a/apple/macos-helper/Package.swift b/apple/macos-helper/Package.swift index a287ecdb48..844c6cb2bb 100644 --- a/apple/macos-helper/Package.swift +++ b/apple/macos-helper/Package.swift @@ -14,20 +14,17 @@ let package = Package( .target( name: "AgentDeviceMacOSInput" ), - .target( - name: "AgentDeviceMacOSDeviceHub" - ), .executableTarget( name: "AgentDeviceMacOSHelper", - dependencies: ["AgentDeviceMacOSInput", "AgentDeviceMacOSDeviceHub"] + dependencies: ["AgentDeviceMacOSInput"] ), .testTarget( name: "AgentDeviceMacOSInputTests", dependencies: ["AgentDeviceMacOSInput"] ), .testTarget( - name: "AgentDeviceMacOSDeviceHubTests", - dependencies: ["AgentDeviceMacOSDeviceHub"] + name: "AgentDeviceMacOSHelperTests", + dependencies: ["AgentDeviceMacOSHelper"] ), ] ) diff --git a/apple/macos-helper/Sources/AgentDeviceMacOSDeviceHub/DeviceHubPose.swift b/apple/macos-helper/Sources/AgentDeviceMacOSDeviceHub/DeviceHubPose.swift deleted file mode 100644 index 0cde6aa94f..0000000000 --- a/apple/macos-helper/Sources/AgentDeviceMacOSDeviceHub/DeviceHubPose.swift +++ /dev/null @@ -1,47 +0,0 @@ -import Foundation - -/// The decisions the Device Hub pose press makes without a window server: which control a pose -/// names, which window belongs to a device, and which sidebar row identifies it. They live apart -/// from the helper executable so the tests can exercise them without an accessibility session. - -/// Device Hub's three pose presets for a foldable simulator, named as the helper's `--pose` -/// argument spells them. Each maps to the accessibility description of its action-bar button. -public enum DeviceHubPose: String, CaseIterable, Sendable { - case closed - case book - case open - - public init?(argument: String) { - self.init(rawValue: argument.lowercased()) - } - - /// The `AXDescription` Device Hub gives the button for this pose. - public var controlDescription: String { - switch self { - case .closed: return "Closed" - case .book: return "Book" - case .open: return "Open" - } - } -} - -/// Device Hub is launched through a trampoline, so LaunchServices registers it with no process -/// identifier; the process table is matched on the executable path instead. -public let deviceHubExecutableSuffix = "/DeviceHub.app/Contents/MacOS/DeviceHub" - -/// The separator Device Hub puts between the device name and the OS in a window title. -public let deviceHubWindowTitleSeparator = " – " - -/// Device Hub titles a device window ` – `, so a window shows the -/// device when its title is the name or the name followed by that exact separator. A bare -/// space would also match a simulator whose name merely extends this one ("iPhone Duo Lab"). -public func deviceHubWindowShows(deviceName: String, title: String?) -> Bool { - guard let title else { return false } - return title == deviceName || title.hasPrefix(deviceName + deviceHubWindowTitleSeparator) -} - -/// The accessibility identifier of the sidebar row for one simulator. The UDID is the one device -/// identity Device Hub exposes that two simulators sharing a name cannot confuse. -public func deviceHubDeviceRowIdentifier(udid: String) -> String { - return "TableRow.Device.\(udid)" -} diff --git a/apple/macos-helper/Sources/AgentDeviceMacOSHelper/DeviceHubPose.swift b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/DeviceHubPose.swift deleted file mode 100644 index c33f4a69a4..0000000000 --- a/apple/macos-helper/Sources/AgentDeviceMacOSHelper/DeviceHubPose.swift +++ /dev/null @@ -1,320 +0,0 @@ -import AgentDeviceMacOSDeviceHub -import AppKit -import ApplicationServices -import Darwin -import Foundation - -/// The Xcode Device Hub application. Its device windows carry an action bar whose pose controls -/// are the only public seam onto the private channel that folds a simulator. -private let deviceHubBundleId = "com.apple.dt.Devices" - -/// How long a reopen gets to restore a window, and a sidebar selection to switch the window's -/// device, before the press is refused. -private let deviceHubSettleDeadline: TimeInterval = 8 -private let deviceHubPoll: TimeInterval = 0.25 - -struct DeviceHubPoseResponse: Encodable { - let pose: String - let control: String - let windowTitle: String - /** Whether Device Hub had to be asked to reopen a window before the device could be selected. */ - let reopened: Bool - /** Whether the window was switched to the device through its sidebar row. */ - let selected: Bool -} - -func handleDeviceHub(arguments: [String]) throws -> any Encodable { - guard arguments.first == "pose" else { - throw HelperError.invalidArgs("device-hub requires pose") - } - let rest = Array(arguments.dropFirst()) - guard let udid = helperOptionValue(arguments: rest, name: "--udid")? - .trimmingCharacters(in: .whitespacesAndNewlines), - !udid.isEmpty - else { - throw HelperError.invalidArgs("device-hub pose requires --udid ") - } - guard let deviceName = helperOptionValue(arguments: rest, name: "--device-name")? - .trimmingCharacters(in: .whitespacesAndNewlines), - !deviceName.isEmpty - else { - throw HelperError.invalidArgs("device-hub pose requires --device-name ") - } - guard let pose = helperOptionValue(arguments: rest, name: "--pose").flatMap(DeviceHubPose.init(argument:)) - else { - throw HelperError.invalidArgs("device-hub pose requires --pose ") - } - let control = pose.controlDescription - guard AXIsProcessTrusted() else { - throw HelperError.commandFailed( - "fold needs Accessibility permission to press the Device Hub pose control", - details: ["reason": "accessibility-permission", "permission": "accessibility"] - ) - } - guard let pid = deviceHubProcessIdentifier() else { - throw HelperError.commandFailed( - "Xcode Device Hub is not running", - details: ["reason": "device-hub-not-running", "bundleId": deviceHubBundleId] - ) - } - - let appElement = AXUIElementCreateApplication(pid) - let (window, reopened) = try deviceHubWindow(in: appElement, pid: pid, deviceName: deviceName) - let selected = try selectDevice(udid: udid, deviceName: deviceName, in: window, appElement: appElement) - let windowTitle = stringAttribute(window, attribute: kAXTitleAttribute as String) ?? "" - guard let button = awaitPoseButton(in: window, description: control) else { - throw HelperError.commandFailed( - "Device Hub shows no \(control) pose control for \(deviceName)", - details: [ - "reason": "device-hub-pose-control-missing", - "windowTitle": windowTitle, - "control": control, - ] - ) - } - let status = AXUIElementPerformAction(button, kAXPressAction as CFString) - guard status == .success else { - throw HelperError.commandFailed( - "Device Hub refused the \(control) pose press", - details: ["reason": "device-hub-press-failed", "status": "\(status.rawValue)"] - ) - } - return SuccessEnvelope( - data: DeviceHubPoseResponse( - pose: pose.rawValue, - control: control, - windowTitle: windowTitle, - reopened: reopened, - selected: selected - ) - ) -} - -/// Device Hub is launched through a trampoline, so LaunchServices registers it with no process -/// identifier (`NSRunningApplication.processIdentifier` is -1) and an accessibility element built -/// from that identifier is invalid. The process table is the only place its real pid appears. -private func deviceHubProcessIdentifier() -> pid_t? { - let executableSuffix = deviceHubExecutableSuffix - let byteCount = proc_listpids(UInt32(PROC_ALL_PIDS), 0, nil, 0) - guard byteCount > 0 else { return nil } - var pids = [pid_t](repeating: 0, count: Int(byteCount) / MemoryLayout.size + 64) - let filled = proc_listpids( - UInt32(PROC_ALL_PIDS), 0, &pids, Int32(pids.count * MemoryLayout.size) - ) - guard filled > 0 else { return nil } - var path = [CChar](repeating: 0, count: 4096) - for pid in pids.prefix(Int(filled) / MemoryLayout.size) where pid > 0 { - guard proc_pidpath(pid, &path, UInt32(path.count)) > 0 else { continue } - if String(cString: path).hasSuffix(executableSuffix) { - return pid - } - } - return nil -} - -/// A device window to drive: the one already showing this device when there is one, otherwise -/// any device window, since its sidebar can switch it to the device. A Device Hub with no window -/// at all — a simulator booted headlessly leaves it that way — is asked to reopen one. -private func deviceHubWindow( - in appElement: AXUIElement, - pid: pid_t, - deviceName: String -) throws -> (window: AXUIElement, reopened: Bool) { - if let window = preferredWindow(in: appElement, deviceName: deviceName) { - return (window, false) - } - try sendReopenEvent(to: pid) - let deadline = Date().addingTimeInterval(deviceHubSettleDeadline) - while Date() < deadline { - Thread.sleep(forTimeInterval: deviceHubPoll) - if let window = preferredWindow(in: appElement, deviceName: deviceName) { - return (window, true) - } - } - throw HelperError.commandFailed( - "Device Hub shows no device window to drive", - details: ["reason": "device-hub-window-missing", "deviceName": deviceName] - ) -} - -private func preferredWindow(in appElement: AXUIElement, deviceName: String) -> AXUIElement? { - let candidates = windows(of: appElement) - return candidates.first { windowShows(deviceName: deviceName, $0) } ?? candidates.first -} - -private func windowShows(deviceName: String, _ window: AXUIElement) -> Bool { - return deviceHubWindowShows( - deviceName: deviceName, - title: stringAttribute(window, attribute: kAXTitleAttribute as String) - ) -} - -/// `kAEReopenApplication` is what the Dock sends when an app with no open windows is clicked, and -/// it is the one event Device Hub answers by restoring the device window. Sent to the process -/// directly, because LaunchServices cannot address a trampolined app by bundle identifier. -private func sendReopenEvent(to pid: pid_t) throws { - let target = NSAppleEventDescriptor(processIdentifier: pid) - let event = NSAppleEventDescriptor( - eventClass: AEEventClass(kCoreEventClass), - eventID: AEEventID(kAEReopenApplication), - targetDescriptor: target, - returnID: AEReturnID(kAutoGenerateReturnID), - transactionID: AETransactionID(kAnyTransactionID) - ) - do { - _ = try event.sendEvent(options: [.noReply], timeout: 5) - } catch { - let code = (error as NSError).code - throw HelperError.commandFailed( - "fold could not ask Device Hub to reopen its device window", - details: [ - "reason": code == -1743 ? "automation-permission" : "device-hub-reopen-failed", - "error": String(describing: error), - ] - ) - } -} - -/// Switches the window to the device through its sidebar row, whose accessibility identifier is -/// `TableRow.Device.` — the one place Device Hub exposes a device identity that two -/// simulators sharing a name cannot confuse. A window already titled with the device still gets -/// the selection when the row is visible, because the title alone cannot tell such twins apart. -private func selectDevice( - udid: String, - deviceName: String, - in window: AXUIElement, - appElement: AXUIElement -) throws -> Bool { - var shownSidebar = false - var row = deviceRow(udid: udid, in: window) - if row == nil, showSidebar(appElement: appElement) { - shownSidebar = true - let deadline = Date().addingTimeInterval(deviceHubSettleDeadline) - while row == nil, Date() < deadline { - Thread.sleep(forTimeInterval: deviceHubPoll) - row = deviceRow(udid: udid, in: window) - } - } - defer { - if shownSidebar { _ = pressMenuItem(appElement: appElement, menu: "View", item: "Hide Sidebar") } - } - guard let row else { - if windowShows(deviceName: deviceName, window) { - return false - } - throw HelperError.commandFailed( - "Device Hub lists no device \(udid) in its sidebar", - details: ["reason": "device-hub-device-missing", "udid": udid, "deviceName": deviceName] - ) - } - let status = AXUIElementSetAttributeValue(row, kAXSelectedAttribute as CFString, kCFBooleanTrue) - guard status == .success else { - throw HelperError.commandFailed( - "Device Hub refused to select \(deviceName) in its sidebar", - details: ["reason": "device-hub-select-failed", "status": "\(status.rawValue)"] - ) - } - let deadline = Date().addingTimeInterval(deviceHubSettleDeadline) - while !windowShows(deviceName: deviceName, window), Date() < deadline { - Thread.sleep(forTimeInterval: deviceHubPoll) - } - guard windowShows(deviceName: deviceName, window) else { - throw HelperError.commandFailed( - "Device Hub did not switch its window to \(deviceName)", - details: [ - "reason": "device-hub-select-unconfirmed", - "windowTitle": stringAttribute(window, attribute: kAXTitleAttribute as String) ?? "", - ] - ) - } - return true -} - -private func deviceRow(udid: String, in window: AXUIElement) -> AXUIElement? { - guard let label = findElement(root: window, depth: 0, where: { - stringAttribute($0, attribute: "AXIdentifier") == deviceHubDeviceRowIdentifier(udid: udid) - }) else { - return nil - } - var current: AXUIElement? = label - while let element = current { - if stringAttribute(element, attribute: kAXRoleAttribute as String) == "AXRow" { - return element - } - current = elementAttribute(element, attribute: kAXParentAttribute as String) - } - return nil -} - -private func showSidebar(appElement: AXUIElement) -> Bool { - return pressMenuItem(appElement: appElement, menu: "View", item: "Show Sidebar") -} - -private func pressMenuItem(appElement: AXUIElement, menu: String, item: String) -> Bool { - guard let menuBar = elementAttribute(appElement, attribute: kAXMenuBarAttribute as String) else { - return false - } - for menuBarItem in children(of: menuBar) - where stringAttribute(menuBarItem, attribute: kAXTitleAttribute as String) == menu { - for submenu in children(of: menuBarItem) { - for menuItem in children(of: submenu) - where stringAttribute(menuItem, attribute: kAXTitleAttribute as String) == item { - return AXUIElementPerformAction(menuItem, kAXPressAction as CFString) == .success - } - } - } - return false -} - -/// The action bar is rebuilt for the device the window shows, so right after a sidebar selection -/// the window already carries the new title while the pose controls are still being laid out. -/// The controls are therefore awaited, not looked up once. -private func awaitPoseButton(in window: AXUIElement, description: String) -> AXUIElement? { - let deadline = Date().addingTimeInterval(deviceHubSettleDeadline) - while true { - if let button = poseButton(in: window, description: description) { - return button - } - guard Date() < deadline else { return nil } - Thread.sleep(forTimeInterval: deviceHubPoll) - } -} - -/// The pose controls sit in the window's action bar as `AXButton`s described by their preset -/// name; the simulated screen inside the same window is an iOS content group whose own buttons -/// carry app labels, never these three. -private func poseButton(in window: AXUIElement, description: String) -> AXUIElement? { - return findElement(root: window, depth: 0) { - stringAttribute($0, attribute: kAXRoleAttribute as String) == "AXButton" - && stringAttribute($0, attribute: kAXDescriptionAttribute as String) == description - } -} - -private func findElement( - root: AXUIElement, - depth: Int, - where matches: (AXUIElement) -> Bool -) -> AXUIElement? { - if depth > 14 { - return nil - } - for child in children(of: root) { - if matches(child) { - return child - } - if stringAttribute(child, attribute: kAXSubroleAttribute as String) == "iOSContentGroup" { - continue - } - if let nested = findElement(root: child, depth: depth + 1, where: matches) { - return nested - } - } - return nil -} - -func helperOptionValue(arguments: [String], name: String) -> String? { - guard let index = arguments.firstIndex(of: name), arguments.indices.contains(index + 1) else { - return nil - } - return arguments[index + 1] -} diff --git a/apple/macos-helper/Sources/AgentDeviceMacOSHelper/main.swift b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/main.swift index 21b10d0bff..1dc3445312 100644 --- a/apple/macos-helper/Sources/AgentDeviceMacOSHelper/main.swift +++ b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/main.swift @@ -75,7 +75,6 @@ struct PressResponse: Encodable { struct ScreenshotResponse: Encodable { let path: String let surface: String? - let fullscreen: Bool } struct AgentDeviceMacOSHelper { @@ -127,8 +126,6 @@ struct AgentDeviceMacOSHelper { return try handleScreenshot(arguments: Array(arguments.dropFirst())) case "audio-probe": return try handleAudioProbe(arguments: Array(arguments.dropFirst())) - case "device-hub": - return try handleDeviceHub(arguments: Array(arguments.dropFirst())) default: throw HelperError.invalidArgs("unknown command: \(command)") } @@ -438,9 +435,8 @@ struct AgentDeviceMacOSHelper { } let surface = optionValue(arguments: arguments, name: "--surface") - let fullscreen = arguments.contains("--fullscreen") - try captureSurfaceScreenshot(surface: surface, outPath: outPath, fullscreen: fullscreen) - return SuccessEnvelope(data: ScreenshotResponse(path: outPath, surface: surface, fullscreen: fullscreen)) + try captureSurfaceScreenshot(surface: surface, outPath: outPath) + return SuccessEnvelope(data: ScreenshotResponse(path: outPath, surface: surface)) } static func handleAudioProbe(arguments: [String]) throws -> any Encodable { @@ -549,8 +545,7 @@ private func pressAtPosition(_ request: MouseClickRequest) throws { } } -private func captureSurfaceScreenshot(surface: String?, outPath: String, fullscreen: Bool) throws { - _ = fullscreen +private func captureSurfaceScreenshot(surface: String?, outPath: String) throws { guard #available(macOS 15.2, *) else { throw HelperError.commandFailed( "screenshot on macOS desktop and menubar surfaces requires macOS 15.2 or newer" diff --git a/apple/macos-helper/Tests/AgentDeviceMacOSDeviceHubTests/DeviceHubPoseTests.swift b/apple/macos-helper/Tests/AgentDeviceMacOSDeviceHubTests/DeviceHubPoseTests.swift deleted file mode 100644 index c6ef1b49e5..0000000000 --- a/apple/macos-helper/Tests/AgentDeviceMacOSDeviceHubTests/DeviceHubPoseTests.swift +++ /dev/null @@ -1,30 +0,0 @@ -import XCTest - -@testable import AgentDeviceMacOSDeviceHub - -final class DeviceHubPoseTests: XCTestCase { - func testEveryPoseNamesTheActionBarControlDeviceHubDescribes() { - XCTAssertEqual(DeviceHubPose(argument: "closed")?.controlDescription, "Closed") - XCTAssertEqual(DeviceHubPose(argument: "book")?.controlDescription, "Book") - XCTAssertEqual(DeviceHubPose(argument: "OPEN")?.controlDescription, "Open") - XCTAssertNil(DeviceHubPose(argument: "half-open"), "the helper takes Device Hub's names, not the CLI's") - XCTAssertEqual(Set(DeviceHubPose.allCases.map(\.controlDescription)), ["Closed", "Book", "Open"]) - } - - // Device Hub titles the window ` – iOS 27.1`; a simulator whose name merely starts - // with another's must not claim that window. - func testWindowTitleMatchesTheDeviceNameAsAPrefixWord() { - XCTAssertTrue(deviceHubWindowShows(deviceName: "iPhone Duo", title: "iPhone Duo – iOS 27.1")) - XCTAssertTrue(deviceHubWindowShows(deviceName: "iPhone Duo", title: "iPhone Duo")) - XCTAssertFalse(deviceHubWindowShows(deviceName: "iPhone Duo", title: "iPhone Duo Lab – iOS 27.1")) - XCTAssertFalse(deviceHubWindowShows(deviceName: "iPhone Duo", title: "bench-golden-v1 – iOS 27.0")) - XCTAssertFalse(deviceHubWindowShows(deviceName: "iPhone Duo", title: nil)) - } - - func testSidebarRowIsKeyedByUdid() { - XCTAssertEqual( - deviceHubDeviceRowIdentifier(udid: "4F879835-4AB3-4046-B033-5AB769209DD4"), - "TableRow.Device.4F879835-4AB3-4046-B033-5AB769209DD4" - ) - } -} diff --git a/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/ScreenshotResponseTests.swift b/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/ScreenshotResponseTests.swift new file mode 100644 index 0000000000..a5471c009f --- /dev/null +++ b/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/ScreenshotResponseTests.swift @@ -0,0 +1,15 @@ +import XCTest + +@testable import AgentDeviceMacOSHelper + +/// `captureSurfaceScreenshot` always reads the main display, so the response describes only +/// what was actually captured: no field for an argument the helper does not read. +final class ScreenshotResponseTests: XCTestCase { + func testResponseNeverCarriesAFullscreenField() throws { + let response = ScreenshotResponse(path: "/tmp/out.png", surface: "desktop") + let data = try JSONEncoder().encode(response) + let decoded = try JSONSerialization.jsonObject(with: data) as? [String: Any] + + XCTAssertEqual(Set(decoded?.keys.map { $0 } ?? []), ["path", "surface"]) + } +} diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunner.xcodeproj/project.pbxproj b/apple/runner/AgentDeviceRunner/AgentDeviceRunner.xcodeproj/project.pbxproj index 506ee028df..f6c7706c48 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunner.xcodeproj/project.pbxproj +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunner.xcodeproj/project.pbxproj @@ -21,9 +21,22 @@ 20EA2EDD2F2CFC7C001CF0EF /* AgentDeviceRunnerUITests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = AgentDeviceRunnerUITests.xctest; sourceTree = BUILT_PRODUCTS_DIR; }; /* End PBXFileReference section */ +/* Begin PBXFileSystemSynchronizedBuildFileExceptionSet section */ + 20EA2EF02F2CFC7C001CF0EF /* Exceptions for "AgentDeviceRunner" folder in "AgentDeviceRunner" target */ = { + isa = PBXFileSystemSynchronizedBuildFileExceptionSet; + membershipExceptions = ( + Info.plist, + ); + target = 20EA2EC52F2CFC7B001CF0EF /* AgentDeviceRunner */; + }; +/* End PBXFileSystemSynchronizedBuildFileExceptionSet section */ + /* Begin PBXFileSystemSynchronizedRootGroup section */ 20EA2EC82F2CFC7B001CF0EF /* AgentDeviceRunner */ = { isa = PBXFileSystemSynchronizedRootGroup; + exceptions = ( + 20EA2EF02F2CFC7C001CF0EF /* Exceptions for "AgentDeviceRunner" folder in "AgentDeviceRunner" target */, + ); path = AgentDeviceRunner; sourceTree = ""; }; @@ -352,6 +365,8 @@ DEVELOPMENT_TEAM = 2S799L9W4M; ENABLE_PREVIEWS = YES; GENERATE_INFOPLIST_FILE = YES; + INFOPLIST_FILE = AgentDeviceRunner/Info.plist; + "INFOPLIST_FILE[sdk=macosx*]" = ""; INFOPLIST_KEY_UIApplicationSceneManifest_Generation = NO; INFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents = YES; INFOPLIST_KEY_UILaunchScreen_Generation = YES; @@ -386,6 +401,8 @@ DEVELOPMENT_TEAM = 2S799L9W4M; ENABLE_PREVIEWS = YES; GENERATE_INFOPLIST_FILE = YES; + INFOPLIST_FILE = AgentDeviceRunner/Info.plist; + "INFOPLIST_FILE[sdk=macosx*]" = ""; INFOPLIST_KEY_UIApplicationSceneManifest_Generation = NO; INFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents = YES; INFOPLIST_KEY_UILaunchScreen_Generation = YES; diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m index 7ed0af0bb4..b9cd0b9005 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m @@ -56,14 +56,24 @@ int main(int argc, const char *argv[]) { #else #import +#if TARGET_OS_IOS +#import +#endif @interface AgentDeviceRunnerViewController : UIViewController @property(nonatomic, strong) UILabel *alertActionStatus; +@property(nonatomic, strong) UILabel *alertActivationBusyAnswer; @property(nonatomic, assign) NSUInteger firstAlertActions; @property(nonatomic, assign) NSUInteger replacementAlertActions; @property(nonatomic, assign) BOOL alertFixtureStarted; @property(nonatomic, strong) NSTimer *alertActivationBusyBackstop; +@property(nonatomic, strong) NSTimer *alertBannerRepost; +@end + +#if TARGET_OS_IOS +@interface AgentDeviceRunnerViewController () @end +#endif @implementation AgentDeviceRunnerViewController @@ -72,15 +82,21 @@ @implementation AgentDeviceRunnerViewController // may receive an event: the app keeps reporting work in flight, which is the state that cost an alert // command its whole deadline in #2546. It stops the moment an alert button is answered, since that // answer is the event the runner is trying to land, and the backstop stops it even when no answer -// arrives so a regressed run finishes rather than waiting out XCTest's own timeout. A layer +// arrives so a regressed run finishes rather than waiting out XCTest's own timeout. The test passes +// the backstop after `--agent-device-alert-activation-busy`, sized to outlast its whole resolution +// and activation budget, so a slow host cannot end the busy state before the answer lands. A layer // animation on its own is not enough; only a UIView animation counts as in-flight work here. -static NSTimeInterval const kAgentDeviceAlertActivationBusyWindow = 20.0; +static NSTimeInterval AgentDeviceAlertActivationBusyWindow(void) { + NSArray *arguments = NSProcessInfo.processInfo.arguments; + NSUInteger flag = [arguments indexOfObject:@"--agent-device-alert-activation-busy"]; + return flag + 1 < arguments.count ? arguments[flag + 1].doubleValue : 0; +} - (void)startAlertActivationBusy { if (self.alertActivationBusyBackstop != nil) { return; } - self.alertActivationBusyBackstop = [NSTimer scheduledTimerWithTimeInterval:kAgentDeviceAlertActivationBusyWindow + self.alertActivationBusyBackstop = [NSTimer scheduledTimerWithTimeInterval:AgentDeviceAlertActivationBusyWindow() target:self selector:@selector(stopAlertActivationBusy) userInfo:nil @@ -101,6 +117,57 @@ - (void)stopAlertActivationBusy { self.alertActivationBusyBackstop = nil; } +// A banner from this app, shown over its own alert and re-posted before the previous one expires so +// one is on screen for as long as the first alert is unanswered. XCTest treats such a banner as an +// interruption of every event aimed at the app (#2546's late tap, from the banner side). +- (void)startAlertBannerThen:(dispatch_block_t)presentAlert { + UNUserNotificationCenter *center = UNUserNotificationCenter.currentNotificationCenter; + center.delegate = self; + [center requestAuthorizationWithOptions:UNAuthorizationOptionAlert + completionHandler:^(BOOL granted, NSError *error) { + (void)error; + if (!granted) { + return; + } + dispatch_async(dispatch_get_main_queue(), ^{ + presentAlert(); + [self postAlertBanner]; + self.alertBannerRepost = [NSTimer scheduledTimerWithTimeInterval:2.0 + target:self + selector:@selector(postAlertBanner) + userInfo:nil + repeats:YES]; + }); + }]; +} + +- (void)postAlertBanner { + UNMutableNotificationContent *content = [[UNMutableNotificationContent alloc] init]; + content.title = @"Agent Device banner"; + content.body = @"Shown over the alert fixture"; + UNNotificationRequest *request = [UNNotificationRequest requestWithIdentifier:NSUUID.UUID.UUIDString + content:content + trigger:nil]; + [UNUserNotificationCenter.currentNotificationCenter addNotificationRequest:request withCompletionHandler:nil]; +} + +- (void)stopAlertBanner { + if (self.alertBannerRepost == nil) { + return; + } + [self.alertBannerRepost invalidate]; + self.alertBannerRepost = nil; + [UNUserNotificationCenter.currentNotificationCenter removeAllDeliveredNotifications]; +} + +- (void)userNotificationCenter:(UNUserNotificationCenter *)center + willPresentNotification:(UNNotification *)notification + withCompletionHandler:(void (^)(UNNotificationPresentationOptions options))completionHandler { + (void)center; + (void)notification; + completionHandler(UNNotificationPresentationOptionBanner); +} + - (void)updateAlertActionStatus { self.alertActionStatus.text = [NSString stringWithFormat:@"First actions: %lu; replacement actions: %lu", @@ -123,7 +190,12 @@ - (void)presentAlertFixtureReplacement:(BOOL)replacement { ? UIAlertActionStyleCancel : UIAlertActionStyleDefault; [alert addAction:[UIAlertAction actionWithTitle:buttonTitle style:style handler:^(UIAlertAction *action) { (void)action; + if (!replacement) { + self.alertActivationBusyAnswer.text = self.alertActivationBusyBackstop != nil + ? @"Answered while busy" : @"Answered after the app went idle"; + } [self stopAlertActivationBusy]; + [self stopAlertBanner]; if (replacement) { self.replacementAlertActions += 1; } else { @@ -145,9 +217,16 @@ - (void)viewDidAppear:(BOOL)animated { if (!self.alertFixtureStarted && [NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-alert-replacement-regression"]) { self.alertFixtureStarted = YES; - [self presentAlertFixtureReplacement:NO]; - if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-alert-activation-busy"]) { - [self startAlertActivationBusy]; + dispatch_block_t presentAlert = ^{ + [self presentAlertFixtureReplacement:NO]; + if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-alert-activation-busy"]) { + [self startAlertActivationBusy]; + } + }; + if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-alert-banner"]) { + [self startAlertBannerThen:presentAlert]; + } else { + presentAlert(); } } } @@ -185,6 +264,19 @@ - (void)viewDidLoad { [self updateAlertActionStatus]; } + if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-alert-activation-busy"]) { + UILabel *busyAnswer = [[UILabel alloc] init]; + busyAnswer.text = @"Unanswered"; + busyAnswer.accessibilityIdentifier = @"agent-device-alert-busy-answer"; + busyAnswer.translatesAutoresizingMaskIntoConstraints = NO; + [self.view addSubview:busyAnswer]; + [NSLayoutConstraint activateConstraints:@[ + [busyAnswer.centerXAnchor constraintEqualToAnchor:self.view.centerXAnchor], + [busyAnswer.topAnchor constraintEqualToAnchor:label.bottomAnchor constant:24], + ]]; + self.alertActivationBusyAnswer = busyAnswer; + } + if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-regression"]) { UITextField *textField = [[UITextField alloc] init]; textField.accessibilityIdentifier = @"agent-device-hardware-keyboard-input"; @@ -203,6 +295,24 @@ - (void)viewDidLoad { ]]; } + if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-crowded-screen"]) { + for (NSUInteger row = 0; row < 500; row++) { + UILabel *rowLabel = [[UILabel alloc] initWithFrame:CGRectMake(0, 60 + (row % 30) * 24, 400, 16)]; + rowLabel.text = [NSString stringWithFormat:@"Crowded row %lu", (unsigned long)row]; + rowLabel.accessibilityIdentifier = [NSString stringWithFormat:@"agent-device-crowded-row-%lu", (unsigned long)row]; + [self.view addSubview:rowLabel]; + } + } + + BOOL markedButton = [NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-dismiss-popup"]; + if (markedButton || [NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-dismiss-popup-window"]) { + UIButton *dismissRegion = [UIButton buttonWithType:UIButtonTypeSystem]; + dismissRegion.accessibilityIdentifier = markedButton ? @" Dismiss Popup " : @"agent-device-close-popover"; + [dismissRegion setTitle:@"Close popover" forState:UIControlStateNormal]; + dismissRegion.frame = CGRectMake(40, 40, 200, 44); + [self.view addSubview:dismissRegion]; + } + if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-selector-read-regression"]) { NSString *const duplicateIdentifier = @"agent-device-selector-read-duplicate"; @@ -227,7 +337,7 @@ - (void)viewDidLoad { @end -#if defined(TARGET_OS_VISION) && TARGET_OS_VISION +// UIApplicationSceneManifest in Info.plist names this class; a rename must update the manifest. @interface AgentDeviceRunnerSceneDelegate : UIResponder @property(nonatomic, strong) UIWindow *window; @end @@ -246,50 +356,20 @@ - (void)scene:(UIScene *)scene self.window = [[UIWindow alloc] initWithWindowScene:(UIWindowScene *)scene]; self.window.rootViewController = [[AgentDeviceRunnerViewController alloc] init]; +#if TARGET_OS_IOS + if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-dismiss-popup-window"]) { + self.window.accessibilityIdentifier = @"Dismiss popup"; + } +#endif [self.window makeKeyAndVisible]; } @end -#endif @interface AgentDeviceRunnerAppDelegate : UIResponder -@property(nonatomic, strong) UIWindow *window; @end @implementation AgentDeviceRunnerAppDelegate - -- (BOOL)application:(UIApplication *)application - didFinishLaunchingWithOptions:(NSDictionary *)launchOptions { - (void)application; - (void)launchOptions; - -#if defined(TARGET_OS_VISION) && TARGET_OS_VISION - return YES; -#else - self.window = [[UIWindow alloc] initWithFrame:UIScreen.mainScreen.bounds]; - self.window.rootViewController = [[AgentDeviceRunnerViewController alloc] init]; - [self.window makeKeyAndVisible]; - - return YES; -#endif -} - -#if defined(TARGET_OS_VISION) && TARGET_OS_VISION -- (UISceneConfiguration *)application:(UIApplication *)application - configurationForConnectingSceneSession:(UISceneSession *)connectingSceneSession - options:(UISceneConnectionOptions *)options { - (void)application; - (void)connectingSceneSession; - (void)options; - - UISceneConfiguration *configuration = - [[UISceneConfiguration alloc] initWithName:@"Default Configuration" - sessionRole:UIWindowSceneSessionRoleApplication]; - configuration.delegateClass = AgentDeviceRunnerSceneDelegate.class; - return configuration; -} -#endif - @end int main(int argc, char *argv[]) { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/Info.plist b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/Info.plist new file mode 100644 index 0000000000..af35d72e3e --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/Info.plist @@ -0,0 +1,23 @@ + + + + + UIApplicationSceneManifest + + UIApplicationSupportsMultipleScenes + + UISceneConfigurations + + UIWindowSceneSessionRoleApplication + + + UISceneConfigurationName + Default Configuration + UISceneDelegateClassName + AgentDeviceRunnerSceneDelegate + + + + + + diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/AgentDeviceRunnerUITests-Bridging-Header.h b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/AgentDeviceRunnerUITests-Bridging-Header.h index a571613e70..90898722a2 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/AgentDeviceRunnerUITests-Bridging-Header.h +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/AgentDeviceRunnerUITests-Bridging-Header.h @@ -2,3 +2,4 @@ #import "RunnerAXSnapshotBridge.h" #import "RunnerSynthesizedGesture.h" #import "RunnerSynthesizedTextEntry.h" +#import "RunnerXCTestEventBridge.h" diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAppScreenCapture.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAppScreenCapture.swift new file mode 100644 index 0000000000..e44934777f --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAppScreenCapture.swift @@ -0,0 +1,183 @@ +import XCTest +#if canImport(UIKit) +import UIKit +#endif + +/// One capture of the display that actually hosts the target app, with the facts needed to read its +/// pixels without guessing a panel: which display was captured, how many pixels the encoded image +/// holds, and how many of them represent one point. +struct CapturedAppScreen { + let image: RunnerImage + let displayID: UInt + let pixelWidth: Int + let pixelHeight: Int + let pixelsPerPoint: Double +} + +/// Why the runner could not name the display hosting the target app. Carried as a reason code +/// rather than as message text so a required capture can fail closed with a code the host keys on, +/// and an optional observation can report "unknown" without parsing a sentence (#2728). +enum RunnerAppScreenCaptureFailure: String, Error { + case unresolvedWindow = "APP_SCREEN_WINDOW_UNRESOLVED" + case unresolvedScreen = "APP_SCREEN_UNRESOLVED" + case unrenderableImage = "APP_SCREEN_CAPTURE_UNRENDERABLE" + + init(_ failure: RunnerApplicationScreenFailure) { + switch failure { + case .unresolvedWindow: + self = .unresolvedWindow + default: + self = .unresolvedScreen + } + } + + var message: String { + switch self { + case .unresolvedWindow: + return "Neither the target app nor the system surface resolved a window, so no display could be named." + case .unresolvedScreen: + return "The resolved app window did not report the display it is shown on." + case .unrenderableImage: + return "The resolved display handed back an image that could not be drawn upright." + } + } + + var hint: String { + switch self { + case .unresolvedWindow: + return "The capture follows the display that owns a window, so this is a runtime that is answering no window for any process; runner.log names the window frame and the query answer it refused." + case .unresolvedScreen: + return "Retry the capture. If it persists, the XCTest runtime is not reporting the window's display; runner.log names the window frame it resolved." + case .unrenderableImage: + return "Retry the capture. The display was resolved, so this is the capture itself failing, not a panel question." + } + } +} + +extension RunnerTests { + /// The target rule, kept apart from the two queries so the rule itself is testable: an unresolved + /// window asks the system surface, and nothing else does. + func selectObservedScreenCapture( + resolving: () -> Result, + fallingBack: () -> Result + ) -> Result { + let outcome = resolving() + if case .failure(.unresolvedWindow) = outcome { + return fallingBack() + } + return outcome + } +} + +#if canImport(UIKit) && os(iOS) +extension RunnerTests { + /// Captures the display hosting `app` instead of `XCUIScreen.main`. + /// + /// On a foldable, `XCUIScreen.main` names one fixed panel while the app can be on the other, and + /// the panel that is dark still answers with a valid PNG of black — a capture that looks like + /// evidence and proves nothing. The window's own screen is the only source that follows the app, + /// and on a panel the system presents turned it hands back an image whose buffer is sideways, so + /// the pixels are redrawn upright at that image's own logical size and scale before anyone + /// encodes or compares them (#2728). + /// + /// Never activates `app`: an observation must not change which app is foregrounded just to get a + /// screen, so the caller passes the app context it already intends to observe. + func captureResolvedAppScreen( + app: XCUIApplication + ) -> Result { + var outcome: Result = .failure( + .unresolvedScreen + ) + let capture = { + outcome = Self.resolveCapturedAppScreen(app: app) + } + if Thread.isMainThread { + capture() + } else { + DispatchQueue.main.sync(execute: capture) + } + return outcome + } + + /// Captures the display a caller should observe, asking `app` first and the system surface second. + /// + /// A capture with no session app, or with one that is not running, still has a window on screen: + /// the home screen is SpringBoard's window, and its panel is the lit one. Asking the system surface + /// keeps that capture honest — it names a display a window actually occupies instead of reaching + /// for `XCUIScreen.main`, which on a foldable is one fixed panel whether or not it is dark (#2728). + /// + /// Only an unresolved *window* moves on to the system surface. A window that resolved and then + /// refused to name its display is the failure the host has to see, and no second question about a + /// different process should bury it. + func captureObservedScreen(app: XCUIApplication) -> Result< + CapturedAppScreen, RunnerAppScreenCaptureFailure + > { + selectObservedScreenCapture( + resolving: { captureResolvedAppScreen(app: app) }, + fallingBack: { captureResolvedAppScreen(app: springboard) } + ) + } + + private static func resolveCapturedAppScreen( + app: XCUIApplication + ) -> Result { + var screen: AnyObject? + var displayID: UInt = 0 + var resolution: RunnerApplicationScreenFailure = .unresolvedScreen + guard RunnerResolveApplicationScreen(app, &screen, &displayID, &resolution), + let captureScreen = screen as? XCUIScreen + else { + return .failure(RunnerAppScreenCaptureFailure(resolution)) + } + let image = captureScreen.screenshot().image + guard let upright = runnerUprightCaptureImage(image) else { + return .failure(.unrenderableImage) + } + // A scale that is not a finite positive number cannot be reported to a host that divides by it, + // and cannot be drawn at, so the capture is refused rather than encoded with a value JSON cannot + // carry (#2728). + guard upright.scale.isFinite, upright.scale > 0 else { + return .failure(.unrenderableImage) + } + guard let cgImage = runnerCGImage(from: upright) else { + return .failure(.unrenderableImage) + } + // A zero-pixel image is a capture that did not happen, not a tiny one. Refusing it here — at the + // type that owns the fact — keeps a required consumer (a recording sizing its writer from this + // frame) from mistaking it for a usable frame and falling back to an untyped error (#2728). + guard cgImage.width > 0, cgImage.height > 0 else { + return .failure(.unrenderableImage) + } + return .success( + CapturedAppScreen( + image: upright, + displayID: displayID, + pixelWidth: cgImage.width, + pixelHeight: cgImage.height, + pixelsPerPoint: Double(upright.scale) + ) + ) + } + + /// Redraws a turned capture so its encoded rows run the way the panel is presented, and leaves an + /// already-upright capture untouched so a single-panel capture keeps the exact bytes it produced + /// before panels existed. The draw box is the image's own oriented logical size and the renderer's + /// scale is the image's own scale: neither comes from a nominal panel size, and the one-pixel + /// difference between what XCTest hands back and what CoreDevice calls the panel is preserved + /// rather than stretched away (#2728). + private static func runnerUprightCaptureImage(_ image: UIImage) -> UIImage? { + if image.imageOrientation == .up { + return image + } + guard image.size.width > 0, image.size.height > 0, image.scale.isFinite, image.scale > 0 else { + return nil + } + let format = UIGraphicsImageRendererFormat() + format.scale = image.scale + let renderer = UIGraphicsImageRenderer(size: image.size, format: format) + return renderer.image { _ in + image.draw(in: CGRect(origin: .zero, size: image.size)) + } + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerScrollViewportPolicy.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerScrollViewportPolicy.swift index 18c8638222..a2dad024e4 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerScrollViewportPolicy.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerScrollViewportPolicy.swift @@ -12,9 +12,10 @@ import AgentDeviceSnapshotPresentation // // The pure rule below is geometry on purpose — no XCUIApplication — so its exact decision is proven // against the golden table shared with its TS twin, `clipScrollViewportAboveKeyboard` in -// packages/contracts/src/scroll-gesture.ts, asserted in that file's test beside this one. The table -// carries only frames representable in both languages: `CGRect` standardizes a negative extent into a -// positive height at a moved origin, so a negative `height` is tested on the TS side alone. +// packages/contracts/src/scroll-gesture.ts, asserted in that file's test and in +// UnitTests/RunnerTests+ScrollViewportPolicyTests.swift. The table carries only frames +// representable in both languages: `CGRect` standardizes a negative extent into a positive height +// at a moved origin, so a negative `height` is tested on the TS side alone. // // The `extension RunnerTests` below is the one impure caller: it reads the runner's own live keyboard // frame, because a frame threaded from the daemon would predate the keyboard. @@ -248,165 +249,3 @@ extension RunnerTests { } #endif } - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS -private struct ScrollViewportPolicyFixture: Decodable { - struct Frame: Decodable { - let x: Double - let y: Double - let width: Double - let height: Double - - var cgRect: CGRect { - CGRect(x: x, y: y, width: width, height: height) - } - } - - struct Constants: Decodable { - let minVisibleFraction: Double - let accessoryAllowance: Double - } - - struct Expected: Decodable { - let kind: String - let viewport: Frame? - let keyboardMinY: Double? - let visibleHeight: Double? - } - - struct TestCase: Decodable { - let name: String - let viewport: Frame - let keyboard: Frame - let expected: Expected - } - - let constants: Constants - let cases: [TestCase] -} - -extension RunnerTests { - /// Golden parity table (#2500): every case in contracts/fixtures/scroll-keyboard-policy.json must - /// agree with the vitest twin. Add cases there, never fork the rule. - func testScrollViewportKeyboardClipMatchesGoldenParityTable() throws { - let fixture = try loadScrollViewportPolicyFixture() - XCTAssertFalse(fixture.cases.isEmpty, "parity table must not be empty") - for testCase in fixture.cases { - let clip = ScrollViewportPolicy.clip( - viewport: testCase.viewport.cgRect, - keyboard: testCase.keyboard.cgRect - ) - switch testCase.expected.kind { - case "unobstructed": - XCTAssertEqual(clip, .unobstructed, testCase.name) - case "avoided": - let expectedFrame = try XCTUnwrap(testCase.expected.viewport, testCase.name).cgRect - let expectedMinY = try XCTUnwrap(testCase.expected.keyboardMinY, testCase.name) - XCTAssertEqual( - clip, - .avoided(frame: expectedFrame, keyboardMinY: expectedMinY), - testCase.name - ) - case "occluded": - let expectedMinY = try XCTUnwrap(testCase.expected.keyboardMinY, testCase.name) - let expectedVisibleHeight = try XCTUnwrap(testCase.expected.visibleHeight, testCase.name) - XCTAssertEqual( - clip, - .occluded(keyboardMinY: expectedMinY, visibleHeight: expectedVisibleHeight), - testCase.name - ) - default: - XCTFail("unknown expected kind `\(testCase.expected.kind)` in \(testCase.name)") - } - } - } - - /// The thresholds are the table's, not this file's. The refusal reason and the runner code are - /// each one side's own vocabulary: the reason is what the host publishes, the code is what this - /// runner answers with, and neither is a shared clip constant. - func testScrollViewportPolicyUsesParityTableConstants() throws { - let constants = try loadScrollViewportPolicyFixture().constants - XCTAssertEqual(constants.minVisibleFraction, ScrollViewportPolicy.minVisibleFraction) - XCTAssertEqual(constants.accessoryAllowance, ScrollViewportPolicy.accessoryAllowance) - } - - /// A clipped landscape band shortens the frame, and `CoordinateSpaceRotation.native(point:)` derives a - /// `landscapeRight` native x from the frame's HEIGHT. Rotating inside the band therefore moves the - /// dispatched path sideways by exactly what the keyboard took, off the lane the plan was built for, - /// so the plan band and the coordinate basis stay separate values through dispatch (#2500). - func testScrollViewportDispatchKeepsTheUnclippedFrameAsItsCoordinateRotationBasis() throws { - let viewport = CGRect(x: 0, y: 0, width: 1210, height: 834) - let keyboard = CGRect(x: 0, y: 588, width: 1210, height: 246) - let clip = ScrollViewportPolicy.clip(viewport: viewport, keyboard: keyboard) - guard case .avoided(let band, let keyboardMinY) = clip else { - return XCTFail("expected a landscape keyboard to be avoided, got \(clip)") - } - XCTAssertEqual(band.height, 576) - - guard case .gesture(let gesture) = ScrollViewportPolicy.frames( - referenceFrame: viewport, - clip: clip - ).gestureDispatch(direction: .up, amount: nil, pixels: nil) else { - return XCTFail("expected a gesture inside the clipped band") - } - XCTAssertEqual(gesture.planFrame, band) - XCTAssertEqual(gesture.keyboardMinY, keyboardMinY) - XCTAssertEqual(gesture.coordinateFrame, viewport, "the rotation basis must survive the clip") - XCTAssertLessThanOrEqual( - max(gesture.plan.y1, gesture.plan.y2), - keyboard.minY - ScrollViewportPolicy.accessoryAllowance, - "a landscape swipe must stay clear of the keys" - ) - - let reported = gesture.attachingEvidence( - to: Response( - ok: true, - data: DataPayload(referenceWidth: viewport.width, referenceHeight: viewport.height), - error: nil - ) - ) - XCTAssertEqual( - reported.data?.referenceHeight, - band.height, - "the payload names the band the plan ran inside, not the synthesis frame" - ) - XCTAssertEqual(reported.data?.referenceWidth, viewport.width) - XCTAssertEqual(reported.data?.keyboardMinY, keyboardMinY) - XCTAssertEqual(reported.data?.keyboardAvoided, true) - - let orientedStartY = gesture.planFrame.minY + gesture.plan.y1 - let dispatchedFromViewport = CoordinateSpaceRotation.native( - point: CGPoint(x: gesture.planFrame.minX + gesture.plan.x1, y: orientedStartY), - in: gesture.coordinateFrame, - interfaceOrientation: RunnerInterfaceOrientation.landscapeRight - ) - let dispatchedFromBand = CoordinateSpaceRotation.native( - point: CGPoint(x: gesture.planFrame.minX + gesture.plan.x1, y: orientedStartY), - in: gesture.planFrame, - interfaceOrientation: RunnerInterfaceOrientation.landscapeRight - ) - XCTAssertEqual( - dispatchedFromViewport.x - dispatchedFromBand.x, - viewport.height - band.height, - accuracy: 0.001, - "rotating inside the clipped band would shift native x by what the keyboard took" - ) - } - - private func loadScrollViewportPolicyFixture() throws -> ScrollViewportPolicyFixture { - let fixtureURL = URL(fileURLWithPath: #filePath) - .deletingLastPathComponent() // AgentDeviceRunnerUITests - .deletingLastPathComponent() // AgentDeviceRunner - .deletingLastPathComponent() // runner - .deletingLastPathComponent() // apple - .deletingLastPathComponent() // repo root - .appendingPathComponent("contracts") - .appendingPathComponent("fixtures") - .appendingPathComponent("scroll-keyboard-policy.json") - return try JSONDecoder().decode( - ScrollViewportPolicyFixture.self, - from: Data(contentsOf: fixtureURL) - ) - } -} -#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.h b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.h index b6c23da7b1..27ee022545 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.h +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.h @@ -2,40 +2,39 @@ NS_ASSUME_NONNULL_BEGIN -FOUNDATION_EXPORT NSInteger RunnerContinuousDragFrameCount(double durationMs); FOUNDATION_EXPORT NSInteger RunnerControlledScrollFrameCount(double durationMs); FOUNDATION_EXPORT double RunnerControlledScrollProgress(double t); @interface RunnerSynthesizedGesture : NSObject +// `resolvedWindow` is the app window the caller already resolved for the gesture's reference +// frame. The record's display ID is read from that same window so geometry and routing can never +// name different windows; pass nil to fall back to the application's first window. + (NSString * _Nullable)synthesizeSwipeWithApplication:(id)application + resolvedWindow:(id _Nullable)resolvedWindow x:(double)x y:(double)y x2:(double)x2 y2:(double)y2 durationMs:(double)durationMs; -+ (NSString * _Nullable)synthesizeContinuousDragWithApplication:(id)application - x:(double)x - y:(double)y - x2:(double)x2 - y2:(double)y2 - durationMs:(double)durationMs; - + (NSString * _Nullable)synthesizeControlledScrollWithApplication:(id)application + resolvedWindow:(id _Nullable)resolvedWindow x:(double)x y:(double)y x2:(double)x2 - y2:(double)y2 + y2:(double)y2 durationMs:(double)durationMs; + (NSString * _Nullable)synthesizeTapWithApplication:(id)application + resolvedWindow:(id _Nullable)resolvedWindow x:(double)x y:(double)y; // Each pointer is an ordered array of { x, y, offsetMs } samples. The first sample // starts contact; subsequent samples move it; all pointers lift at their final offset. + (NSString * _Nullable)synthesizeGestureWithApplication:(id)application + resolvedWindow:(id _Nullable)resolvedWindow pointerSamples:(NSArray *> *> *)pointerSamples; // UIInterfaceOrientation of the app (1 portrait, 2 upsideDown, 3 landscapeRight, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m index dbbf4ed5b0..bb6a2039aa 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m @@ -6,13 +6,13 @@ static NSString *const RunnerGestureSynthesisSurface = @"event"; -typedef id (*RunnerMsgSendInitRecord)(id, SEL, NSString *, NSInteger); +typedef id (*RunnerMsgSendInitRecord)(id, SEL, NSString *, NSUInteger, NSInteger); typedef id (*RunnerMsgSendInitPath)(id, SEL, CGPoint, NSTimeInterval); typedef void (*RunnerMsgSendPathMove)(id, SEL, CGPoint, NSTimeInterval); typedef void (*RunnerMsgSendPathOffset)(id, SEL, NSTimeInterval); -// Gesture-specific extension of the shared bridge: the 2-arg -// `initWithName:interfaceOrientation:` record initializer and the touch-path +// Gesture-specific extension of the shared bridge: the display-aware +// `initWithName:displayID:interfaceOrientation:` record initializer and the touch-path // factory/mutator selectors, none of which text-entry synthesis needs. typedef struct { RunnerXCTestEventBridge core; @@ -37,6 +37,7 @@ typedef id (*RunnerDragPointerPathFactory)( ); static NSString * _Nullable RunnerCreateEventRecord( id application, + id _Nullable resolvedWindow, NSString *recordName, RunnerGestureEventBridge *bridge, id *record @@ -51,12 +52,6 @@ static id RunnerSwipePointerPath( CGPoint end, double durationMs ); -static id RunnerContinuousDragPointerPath( - const RunnerGestureEventBridge *bridge, - CGPoint start, - CGPoint end, - double durationMs -); static id RunnerControlledScrollPointerPath( const RunnerGestureEventBridge *bridge, CGPoint start, @@ -71,16 +66,16 @@ static id RunnerSampledDragPointerPath( RunnerDragProgressFunction progress, NSInteger frameCount ); -static double RunnerSmoothstepProgress(double t); static NSString * _Nullable RunnerTrySynthesizeDrag( id application, + id _Nullable resolvedWindow, CGPoint start, CGPoint end, double durationMs, NSString *recordName, RunnerDragPointerPathFactory pathFactory ); -static NSString * _Nullable RunnerTrySynthesizeTap(id application, CGPoint point); +static NSString * _Nullable RunnerTrySynthesizeTap(id application, id _Nullable resolvedWindow, CGPoint point); // XCTest's proven swipe profile reaches the endpoint in 100 ms, then holds for the planned // fling duration. Fast movement is what lets UIKit distinguish a fling from a timed pan. static const NSTimeInterval RunnerSwipeMovementDurationSeconds = 0.1; @@ -94,14 +89,16 @@ static id RunnerTapPointerPath( @implementation RunnerSynthesizedGesture + (NSString * _Nullable)synthesizeSwipeWithApplication:(id)application - x:(double)x - y:(double)y - x2:(double)x2 - y2:(double)y2 + resolvedWindow:(id _Nullable)resolvedWindow + x:(double)x + y:(double)y + x2:(double)x2 + y2:(double)y2 durationMs:(double)durationMs { @try { return RunnerTrySynthesizeDrag( application, + resolvedWindow, CGPointMake(x, y), CGPointMake(x2, y2), durationMs, @@ -113,35 +110,17 @@ + (NSString * _Nullable)synthesizeSwipeWithApplication:(id)application } } -+ (NSString * _Nullable)synthesizeContinuousDragWithApplication:(id)application - x:(double)x - y:(double)y - x2:(double)x2 - y2:(double)y2 - durationMs:(double)durationMs { - @try { - return RunnerTrySynthesizeDrag( - application, - CGPointMake(x, y), - CGPointMake(x2, y2), - durationMs, - @"agent-device-continuous-drag", - RunnerContinuousDragPointerPath - ); - } @catch (NSException *exception) { - return RunnerFormatXCTestException(exception, @"private XCTest event synthesis failed"); - } -} - + (NSString * _Nullable)synthesizeControlledScrollWithApplication:(id)application + resolvedWindow:(id _Nullable)resolvedWindow x:(double)x y:(double)y x2:(double)x2 - y2:(double)y2 - durationMs:(double)durationMs { + y2:(double)y2 + durationMs:(double)durationMs { @try { return RunnerTrySynthesizeDrag( application, + resolvedWindow, CGPointMake(x, y), CGPointMake(x2, y2), durationMs, @@ -154,22 +133,25 @@ + (NSString * _Nullable)synthesizeControlledScrollWithApplication:(id)applicatio } + (NSString * _Nullable)synthesizeTapWithApplication:(id)application + resolvedWindow:(id _Nullable)resolvedWindow x:(double)x y:(double)y { @try { - return RunnerTrySynthesizeTap(application, CGPointMake(x, y)); + return RunnerTrySynthesizeTap(application, resolvedWindow, CGPointMake(x, y)); } @catch (NSException *exception) { return RunnerFormatXCTestException(exception, @"private XCTest event synthesis failed"); } } + (NSString * _Nullable)synthesizeGestureWithApplication:(id)application + resolvedWindow:(id _Nullable)resolvedWindow pointerSamples:(NSArray *> *> *)pointerSamples { @try { RunnerGestureEventBridge bridge; id record = nil; NSString *error = RunnerCreateEventRecord( application, + resolvedWindow, @"agent-device-gesture-plan", &bridge, &record @@ -224,6 +206,7 @@ + (NSInteger)interfaceOrientationForApplication:(id)application { static NSString * _Nullable RunnerTrySynthesizeDrag( id application, + id _Nullable resolvedWindow, CGPoint start, CGPoint end, double durationMs, @@ -232,7 +215,7 @@ + (NSInteger)interfaceOrientationForApplication:(id)application { ) { RunnerGestureEventBridge bridge; id record = nil; - NSString *error = RunnerCreateEventRecord(application, recordName, &bridge, &record); + NSString *error = RunnerCreateEventRecord(application, resolvedWindow, recordName, &bridge, &record); if (error != nil) return error; id path = pathFactory(&bridge, start, end, durationMs); @@ -244,11 +227,12 @@ + (NSInteger)interfaceOrientationForApplication:(id)application { return RunnerSynthesizeEventRecord(&bridge, record); } -static NSString * _Nullable RunnerTrySynthesizeTap(id application, CGPoint point) { +static NSString * _Nullable RunnerTrySynthesizeTap(id application, id _Nullable resolvedWindow, CGPoint point) { RunnerGestureEventBridge bridge; id record = nil; NSString *error = RunnerCreateEventRecord( application, + resolvedWindow, @"agent-device-tap", &bridge, &record @@ -271,14 +255,14 @@ + (NSInteger)interfaceOrientationForApplication:(id)application { NSString *missing = RunnerResolveXCTestEventBridge(application, RunnerGestureSynthesisSurface, &core); if (missing != nil) return missing; - SEL initRecordSelector = NSSelectorFromString(@"initWithName:interfaceOrientation:"); + SEL initRecordSelector = NSSelectorFromString(@"initWithName:displayID:interfaceOrientation:"); SEL interfaceOrientationSelector = NSSelectorFromString(@"interfaceOrientation"); SEL initPathSelector = NSSelectorFromString(@"initForTouchAtPoint:offset:"); SEL moveSelector = NSSelectorFromString(@"moveToPoint:atOffset:"); SEL liftSelector = NSSelectorFromString(@"liftUpAtOffset:"); missing = RunnerRequireSelector( - core.recordClass, initRecordSelector, @"initWithName:interfaceOrientation:", RunnerGestureSynthesisSurface + core.recordClass, initRecordSelector, @"initWithName:displayID:interfaceOrientation:", RunnerGestureSynthesisSurface ); if (missing != nil) return missing; missing = RunnerRequireSelector( @@ -311,6 +295,7 @@ + (NSInteger)interfaceOrientationForApplication:(id)application { static NSString * _Nullable RunnerCreateEventRecord( id application, + id _Nullable resolvedWindow, NSString *recordName, RunnerGestureEventBridge *bridge, id *record @@ -318,6 +303,12 @@ + (NSInteger)interfaceOrientationForApplication:(id)application { NSString *missing = RunnerResolveGestureEventBridge(application, bridge); if (missing != nil) return missing; + NSUInteger displayID = 0; + if (resolvedWindow == nil) { + return @"private XCTest event synthesis unavailable: no resolved application window"; + } + NSString *displayError = RunnerResolveWindowDisplayID(resolvedWindow, &displayID); + if (displayError != nil) return displayError; NSInteger interfaceOrientation = ((RunnerMsgSendInteger)objc_msgSend)(application, bridge->interfaceOrientationSelector); NSInteger targetProcessID = @@ -330,11 +321,18 @@ + (NSInteger)interfaceOrientationForApplication:(id)application { [bridge->core.recordClass alloc], bridge->initRecordSelector, recordName, + displayID, interfaceOrientation ); if (eventRecord == nil) { return @"private XCTest event synthesis failed: could not create event record"; } + NSLog( + @"AGENT_DEVICE_RUNNER_SYNTHESIZED_RECORD name=%@ displayID=%lu interfaceOrientation=%ld", + recordName, + (unsigned long)displayID, + (long)interfaceOrientation + ); ((RunnerMsgSendSetInteger)objc_msgSend)( eventRecord, bridge->core.setTargetProcessIDSelector, @@ -384,25 +382,6 @@ static id RunnerSwipePointerPath( return path; } -static id RunnerContinuousDragPointerPath( - const RunnerGestureEventBridge *bridge, - CGPoint start, - CGPoint end, - double durationMs -) { - // This is velocity shaping, not just interpolation density: smoothstep's endpoint slope is zero, - // while a planned linear segment reaches lift with nonzero velocity unless a destination hold - // follows it. UIKit uses finger-up velocity for scroll deceleration. See ADR 0013 and issue #1586. - return RunnerSampledDragPointerPath( - bridge, - start, - end, - durationMs, - RunnerSmoothstepProgress, - RunnerContinuousDragFrameCount(durationMs) - ); -} - static id RunnerControlledScrollPointerPath( const RunnerGestureEventBridge *bridge, CGPoint start, @@ -451,14 +430,6 @@ static id RunnerSampledDragPointerPath( return path; } -static double RunnerSmoothstepProgress(double t) { - return t * t * (3.0 - 2.0 * t); -} - -NSInteger RunnerContinuousDragFrameCount(double durationMs) { - return MAX(3, (NSInteger)(durationMs / RunnerDragSampleIntervalMs)); -} - NSInteger RunnerControlledScrollFrameCount(double durationMs) { return MIN( RunnerControlledScrollMaxFrameCount, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSystemSurfaceHostPolicy.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSystemSurfaceHostPolicy.swift index 97dd753e28..9db900e9e3 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSystemSurfaceHostPolicy.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSystemSurfaceHostPolicy.swift @@ -30,51 +30,3 @@ enum SystemSurfaceHostRegistry { host(forBundleId: bundleId) != nil } } - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS -import XCTest - -private struct SystemSurfaceHostFixture: Decodable { - struct Host: Decodable { - let bundleId: String - let kind: String - } - let hosts: [Host] -} - -extension RunnerTests { - func testSystemSurfaceHostRegistryMirrorsGoldenFixture() throws { - let fixtureURL = URL(fileURLWithPath: #filePath) - .deletingLastPathComponent() // AgentDeviceRunnerUITests - .deletingLastPathComponent() // AgentDeviceRunner - .deletingLastPathComponent() // runner - .deletingLastPathComponent() // apple - .deletingLastPathComponent() // repo root - .appendingPathComponent("contracts") - .appendingPathComponent("fixtures") - .appendingPathComponent("ios-system-surface-hosts.json") - let fixture = try JSONDecoder().decode( - SystemSurfaceHostFixture.self, - from: Data(contentsOf: fixtureURL) - ) - let registry = SystemSurfaceHostRegistry.hosts.map { [$0.bundleId, $0.kind.rawValue] } - let golden = fixture.hosts.map { [$0.bundleId, $0.kind] } - XCTAssertEqual(registry, golden, "SystemSurfaceHostRegistry drifted from the golden fixture") - } - - func testSystemSurfaceHostRegistryRecognizesRegisteredHosts() { - XCTAssertTrue(SystemSurfaceHostRegistry.isSystemSurfaceHost("com.apple.SafariViewService")) - XCTAssertTrue(SystemSurfaceHostRegistry.isSystemSurfaceHost("com.apple.PassbookUIService")) - XCTAssertFalse(SystemSurfaceHostRegistry.isSystemSurfaceHost("com.example.app")) - XCTAssertFalse(SystemSurfaceHostRegistry.isSystemSurfaceHost(nil)) - XCTAssertEqual( - SystemSurfaceHostRegistry.host(forBundleId: "com.apple.SafariViewService")?.kind, - .webAuth - ) - XCTAssertEqual( - SystemSurfaceHostRegistry.host(forBundleId: "com.apple.PassbookUIService")?.kind, - .payment - ) - } -} -#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTapPointPolicy.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTapPointPolicy.swift index 379f78b734..33dde24083 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTapPointPolicy.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTapPointPolicy.swift @@ -25,53 +25,3 @@ enum TapPointPolicy { && centerY >= windowFrame.minY && centerY <= windowFrame.maxY } } - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS -private struct TapPointPolicyFixture: Decodable { - struct Frame: Decodable { - let x: Double - let y: Double - let width: Double - let height: Double - - var cgRect: CGRect { - CGRect(x: x, y: y, width: width, height: height) - } - } - - let name: String - let elementFrame: Frame - let windowFrame: Frame - let allowed: Bool -} - -extension RunnerTests { - // Golden parity table (ADR 0011 Layer 2): every case in - // contracts/fixtures/tap-point-policy.json must agree with the vitest twin - // (tap-point-policy-parity.test.ts). Add cases there, never fork the rule. - func testTapPointPolicyMatchesGoldenParityTable() throws { - let fixtureURL = URL(fileURLWithPath: #filePath) - .deletingLastPathComponent() // AgentDeviceRunnerUITests - .deletingLastPathComponent() // AgentDeviceRunner - .deletingLastPathComponent() // runner - .deletingLastPathComponent() // apple - .deletingLastPathComponent() // repo root - .appendingPathComponent("contracts") - .appendingPathComponent("fixtures") - .appendingPathComponent("tap-point-policy.json") - let data = try Data(contentsOf: fixtureURL) - let cases = try JSONDecoder().decode([TapPointPolicyFixture].self, from: data) - XCTAssertFalse(cases.isEmpty, "parity table must not be empty") - for fixture in cases { - XCTAssertEqual( - TapPointPolicy.isAllowed( - elementFrame: fixture.elementFrame.cgRect, - windowFrame: fixture.windowFrame.cgRect - ), - fixture.allowed, - fixture.name - ) - } - } -} -#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift index 9d41034d68..df933a24ab 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift @@ -55,14 +55,10 @@ extension RunnerTests { static func privateAXCustomActionCoverage(_ raw: Any?) -> SnapshotCustomActionCoverage? { guard let coverage = raw as? [String: Any], let read = (coverage[RunnerAXSnapshotCustomActionsReadKey] as? NSNumber)?.intValue, - let candidates = (coverage[RunnerAXSnapshotCustomActionsCandidatesKey] as? NSNumber)?.intValue + let candidates = (coverage[RunnerAXSnapshotCustomActionsCandidatesKey] as? NSNumber)?.intValue, + let truncated = (coverage[RunnerAXSnapshotCustomActionsTruncatedKey] as? NSNumber)?.intValue, + let blocked = (coverage[RunnerAXSnapshotCustomActionsBlockedKey] as? NSNumber)?.boolValue else { return nil } - // read/candidates are the ratio and must both be present; the truncation - // count is additive, so an older bridge that omits it reads as zero rather - // than voiding the whole coverage. - let truncated = - (coverage[RunnerAXSnapshotCustomActionsTruncatedKey] as? NSNumber)?.intValue ?? 0 - let blocked = (coverage[RunnerAXSnapshotCustomActionsBlockedKey] as? NSNumber)?.boolValue ?? false return SnapshotCustomActionCoverage( read: read, candidates: candidates, truncated: truncated, blocked: blocked) } @@ -114,14 +110,16 @@ extension RunnerTests { /// remembered depth deliberately does NOT refresh the TTL, so expiry re-probes the full /// requested depth once per window instead of capping this screen class forever. func recordPrivateAXAcceptedDepth( + bundleId: String?, + processIdentifier: Int?, exactDepthRequested: Bool, effectiveDepth: Int, attemptDepths: [Int] ) { guard !exactDepthRequested, effectiveDepth != attemptDepths.first else { return } rememberPrivateAXAcceptedDepth( - bundleId: currentBundleId, - processIdentifier: currentAppProcessIdentifier, + bundleId: bundleId, + processIdentifier: processIdentifier, depth: effectiveDepth ) } @@ -164,11 +162,12 @@ extension RunnerTests { } func privateAXSnapshotAcquisition( - app: XCUIApplication, + target: SnapshotCaptureTarget, hint: CaptureHint, deadline: Date = .distantFuture ) -> SnapshotAcquisition? { #if os(iOS) && targetEnvironment(simulator) + let app = target.app let requestedDepth = hint.rawTraversalDepth ?? 64 // An explicit --depth request is honored as asked: no accepted-depth // memory, no frontier extension past it. @@ -177,8 +176,8 @@ extension RunnerTests { exactDepthRequested ? nil : rememberedPrivateAXAcceptedDepth( - bundleId: currentBundleId, - processIdentifier: currentAppProcessIdentifier + bundleId: target.bundleId, + processIdentifier: target.processIdentifier ) let attemptDepths = Self.privateAXAttemptDepths( requestedDepth: requestedDepth, @@ -206,6 +205,8 @@ extension RunnerTests { return nil } recordPrivateAXAcceptedDepth( + bundleId: target.bundleId, + processIdentifier: target.processIdentifier, exactDepthRequested: exactDepthRequested, effectiveDepth: effectiveDepth, attemptDepths: attemptDepths @@ -216,8 +217,11 @@ extension RunnerTests { } let rootFrame = privateAXRect(root["frame"]) - let geometry = privateAXSnapshotGeometry(app: app, rootFrame: rootFrame) - let viewport = geometry.viewport + let viewport = privateAXSnapshotViewport( + app: app, + bundleId: target.bundleId, + rootFrame: rootFrame + ) let nodes = privateAXAcquisition( rawRoot: root, hint: hint @@ -254,8 +258,7 @@ extension RunnerTests { customActions: Self.privateAXCustomActionCoverage( response[RunnerAXSnapshotCustomActionsKey] ), - viewport: viewport, - interfaceOrientation: geometry.interfaceOrientation + viewport: viewport ) #else return nil @@ -266,40 +269,36 @@ extension RunnerTests { /// grinding on this screen class. Under penalty it reliably burns its full timeout and /// falls back anyway (~1s added to every private AX capture on the Bluesky bench feed), /// so honor the penalty here the same way capture plans do. - func shouldReadPrivateAXViewportViaXCTest() -> Bool { - !hasAbandonedMainThreadWork() && !isSnapshotXCTestChannelPenalized(bundleId: currentBundleId) + func shouldReadPrivateAXViewportViaXCTest(bundleId: String?) -> Bool { + !hasAbandonedMainThreadWork() && !isSnapshotXCTestChannelPenalized(bundleId: bundleId) } - /// The geometry this tier may anchor a rotation on. The bridge's own root frame is one more - /// reported box rather than the app's frame, so a capture anchored on it reports no interface - /// orientation and normalizes nothing: rotated system surfaces then stay as reported, which the - /// consumers already treat as geometry they cannot measure (#2612). - private func privateAXSnapshotGeometry( + /// The app's reported viewport when XCTest can read it, else the bridge's own root frame declared + /// `.derived`, which cannot anchor a rotation: rotated system surfaces then stay as reported (#2612). + private func privateAXSnapshotViewport( app: XCUIApplication, + bundleId: String?, rootFrame: CGRect - ) -> (viewport: CGRect, interfaceOrientation: Int) { - let fallback = rootFrame.isEmpty ? CGRect.infinite : rootFrame - guard shouldReadPrivateAXViewportViaXCTest() else { - return (fallback, RunnerInterfaceOrientation.unknown) + ) -> SnapshotViewport { + let fallback = SnapshotViewport.derived(box: rootFrame) + guard shouldReadPrivateAXViewportViaXCTest(bundleId: bundleId) else { + return fallback } do { - let anchor = try runMainThreadWork( + let reported = try runMainThreadWork( "private_ax_viewport", timeout: 1, timeoutError: snapshotMainThreadTimeoutError("reading private AX viewport") ) { - ( - viewport: self.safeSnapshotViewport(app: app), - interfaceOrientation: self.capturedInterfaceOrientation(app: app) - ) + self.safeSnapshotViewport(app: app, readingOrientation: true) } - if anchor.viewport.isInfinite || anchor.viewport.isNull || anchor.viewport.isEmpty { - return (fallback, RunnerInterfaceOrientation.unknown) + if case .missing = reported { + return fallback } - return (anchor.viewport, anchor.interfaceOrientation) + return reported } catch { NSLog("AGENT_DEVICE_RUNNER_PRIVATE_AX_VIEWPORT_FALLBACK=%@", String(describing: error)) - return (fallback, RunnerInterfaceOrientation.unknown) + return fallback } } @@ -321,555 +320,3 @@ extension RunnerTests { return nil } } - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS -// MARK: - In-bundle unit tests - -extension RunnerTests { - func testPrivateAXAttemptDepthsAppliesRememberedDepth() { - XCTAssertEqual( - Self.privateAXAttemptDepths(requestedDepth: 64, rememberedDepth: nil), - [64, 56, 40, 24, 12] - ) - XCTAssertEqual( - Self.privateAXAttemptDepths(requestedDepth: 64, rememberedDepth: 56), - [56, 40, 24, 12] - ) - XCTAssertEqual(Self.privateAXAttemptDepths(requestedDepth: 64, rememberedDepth: 12), [12]) - // Remembered at/above the requested depth changes nothing. - XCTAssertEqual( - Self.privateAXAttemptDepths(requestedDepth: 64, rememberedDepth: 64), - [64, 56, 40, 24, 12] - ) - // A shallower explicit request keeps its own rungs; deeper stale memory is ignored. - XCTAssertEqual(Self.privateAXAttemptDepths(requestedDepth: 24, rememberedDepth: 56), [24, 12]) - } - - /// Executed producer contract for the #1627 review blocker: a frontier whose - /// live element vanished, and one whose re-rooted request fails, must BOTH - /// count as missed — an all-miss extension reporting itself drained would - /// present a capped capture as complete. Goes red if either miss-path - /// increment in extendSnapshotFrontiers is removed. - func testDeepExtensionCountsMissedFrontiers() { - // Element vanished (list churn between serialization and extension): the - // fabricated snapshot answers nil for accessibilityElement — missed, and - // no request call is consumed. (An explicit nil property: bare NSObject - // resolves the key through a UIKit category and would take the call path.) - let orphan = RunnerAXSnapshotFrontier() - orphan.snapshot = FrontierSnapshotWithoutElementForTesting() - orphan.node = NSMutableDictionary() - // Re-rooted request fails: the element resolves but the client cannot - // serve requestSnapshotForElement — one consumed call AND a miss. - let unreachable = RunnerAXSnapshotFrontier() - unreachable.snapshot = FrontierSnapshotWithElementForTesting() - unreachable.node = NSMutableDictionary() - - var nodeCount = 0 - var truncated = ObjCBool(false) - let outcome = RunnerAXSnapshotBridge.extend( - NSMutableArray(array: [orphan, unreachable]), - axClient: NSObject(), - attributes: [], - maxDepth: 56, - maxNodes: 5_000, - nodeCount: &nodeCount, - truncated: &truncated, - callsAllowed: 8, - mergedLeaves: nil, - deadline: nil - ) - - XCTAssertEqual(outcome?[RunnerAXSnapshotDeepExtensionMissedKey] as? Int, 2) - XCTAssertEqual(outcome?[RunnerAXSnapshotDeepExtensionCallsKey] as? Int, 1) - XCTAssertEqual(outcome?[RunnerAXSnapshotDeepExtensionPendingKey] as? Int, 0) - XCTAssertEqual(outcome?[RunnerAXSnapshotDeepExtensionNodesAddedKey] as? Int, 0) - XCTAssertFalse(truncated.boolValue) - // And the consumer verdict over exactly this outcome: still depth-limited. - XCTAssertTrue( - Self.privateAXDepthLimited( - effectiveDepth: 56, requestedDepth: 64, pendingFrontiers: 0, missedFrontiers: 2)) - } - - func testPrivateAXDepthLimitedRequiresEveryFrontierResolved() { - // Un-capped capture is never depth-limited, extension or not. - XCTAssertFalse( - Self.privateAXDepthLimited( - effectiveDepth: 64, requestedDepth: 64, pendingFrontiers: nil, missedFrontiers: nil)) - // Capped with no extension outcome (never ran) stays depth-limited. - XCTAssertTrue( - Self.privateAXDepthLimited( - effectiveDepth: 56, requestedDepth: 64, pendingFrontiers: nil, missedFrontiers: nil)) - // Fully drained extension clears the verdict. - XCTAssertFalse( - Self.privateAXDepthLimited( - effectiveDepth: 56, requestedDepth: 64, pendingFrontiers: 0, missedFrontiers: 0)) - // Budget exhaustion (pending frontiers) keeps it. - XCTAssertTrue( - Self.privateAXDepthLimited( - effectiveDepth: 56, requestedDepth: 64, pendingFrontiers: 2, missedFrontiers: 0)) - // The #1627 review blocker: an all-miss extension (elements vanished or - // re-rooted requests failed) resolved nothing — it must NOT present the - // capture as complete just because the queue emptied. - XCTAssertTrue( - Self.privateAXDepthLimited( - effectiveDepth: 56, requestedDepth: 64, pendingFrontiers: 0, missedFrontiers: 8)) - } - - func testPrivateAXAcceptedDepthMemoryMatchesBundleProcessAndExpires() { - defer { clearPrivateAXAcceptedDepth(reason: "test-cleanup") } - - rememberPrivateAXAcceptedDepth(bundleId: "xyz.blueskyweb.app", processIdentifier: 111, depth: 56) - XCTAssertEqual( - rememberedPrivateAXAcceptedDepth(bundleId: "xyz.blueskyweb.app", processIdentifier: 111), - 56 - ) - XCTAssertNil(rememberedPrivateAXAcceptedDepth(bundleId: "com.other.app", processIdentifier: 111)) - // A relaunch changes the PID; the new process must re-probe the full depth even inside the - // TTL, and an unknown current PID (post-invalidation) must never match. - XCTAssertNil(rememberedPrivateAXAcceptedDepth(bundleId: "xyz.blueskyweb.app", processIdentifier: 222)) - XCTAssertNil(rememberedPrivateAXAcceptedDepth(bundleId: "xyz.blueskyweb.app", processIdentifier: nil)) - - // Expired memory stops applying (the expiry re-probes the full requested depth). - privateAXAcceptedDepthUntil = Date(timeIntervalSinceNow: -1) - XCTAssertNil(rememberedPrivateAXAcceptedDepth(bundleId: "xyz.blueskyweb.app", processIdentifier: 111)) - } - - func testPrivateAXAcceptedDepthMemoryRequiresProcessIdentifierToRecord() { - defer { clearPrivateAXAcceptedDepth(reason: "test-cleanup") } - - rememberPrivateAXAcceptedDepth(bundleId: "xyz.blueskyweb.app", processIdentifier: nil, depth: 56) - XCTAssertNil( - rememberedPrivateAXAcceptedDepth(bundleId: "xyz.blueskyweb.app", processIdentifier: nil) - ) - } - - func testViewportReadSkippedWhileXCTestChannelPenalized() { - // Pins the viewport fast path (#1587 review): every penalized private AX capture used to burn - // the full 1s main-thread timeout on a doomed viewport read before falling back. - currentBundleId = "xyz.blueskyweb.app" - defer { - currentBundleId = nil - clearSnapshotXCTestChannelPenalty(reason: "test-cleanup") - abandonedMainThreadWorkCount = 0 - } - - XCTAssertTrue(shouldReadPrivateAXViewportViaXCTest()) - - penalizeSnapshotXCTestChannel(bundleId: "xyz.blueskyweb.app", reason: "test") - XCTAssertFalse(shouldReadPrivateAXViewportViaXCTest()) - - clearSnapshotXCTestChannelPenalty(reason: "test") - XCTAssertTrue(shouldReadPrivateAXViewportViaXCTest()) - - abandonedMainThreadWorkCount = 1 - XCTAssertFalse(shouldReadPrivateAXViewportViaXCTest()) - } - - /// The wire field must reach both capture options AND the backend pin: custom - /// actions are only readable through the private AX client, so a capture that - /// asked for them but planned the XCTest tree backend would return a payload - /// that structurally cannot carry them. - func testCustomActionsRequestPinsPrivateAXBackend() throws { - let asked = try JSONDecoder().decode( - Command.self, from: Data(#"{"command":"snapshot","customActions":true}"#.utf8)) - let options = Self.presentationOptions(from: asked) - XCTAssertTrue(options.customActions) - XCTAssertEqual(options.preferredBackend, SnapshotBackendKind.privateAX.rawValue) - XCTAssertTrue( - Self.snapshotXCTestChannelTreatedAsPenalized( - penalized: false, preferredBackend: options.preferredBackend)) - - // An explicit pin is never overwritten by the implied one. - let pinned = try JSONDecoder().decode( - Command.self, - from: Data(#"{"command":"snapshot","customActions":true,"preferredBackend":"tree"}"#.utf8)) - XCTAssertEqual(Self.presentationOptions(from: pinned).preferredBackend, "tree") - - // And the default capture neither asks nor pins. - let bare = try JSONDecoder().decode(Command.self, from: Data(#"{"command":"snapshot"}"#.utf8)) - XCTAssertFalse(Self.presentationOptions(from: bare).customActions) - XCTAssertNil(Self.presentationOptions(from: bare).preferredBackend) - } - - /// A request-pinned backend degraded nothing, so its verdict must not claim - /// slow accessibility work — that reason drives a user-facing warning. - func testRequestPinnedBackendReportsItsOwnReason() { - let requested = Self.xcTestChannelStateFirstFailure( - .deferredToIndependentBackend, requestPinnedBackend: true) - XCTAssertEqual(requested?.code, "requested-backend") - XCTAssertFalse(requested?.reason.contains("slow accessibility work") ?? true) - - // The circuit breaker's own deferral keeps its established code and wording. - let breaker = Self.xcTestChannelStateFirstFailure(.deferredToIndependentBackend) - XCTAssertEqual(breaker?.code, "deferred") - - // The bounded probe and the healthy plan are untouched by the new flag. - XCTAssertEqual( - Self.xcTestChannelStateFirstFailure(.boundedXCTestProbe, requestPinnedBackend: true)?.code, - "budget") - XCTAssertNil(Self.xcTestChannelStateFirstFailure(.normal, requestPinnedBackend: true)) - } - - /// The disclosure only exists if the counts survive the bridge boundary, and - /// "did not ask" must stay distinguishable from "read none". - func testCustomActionCoverageParsesOnlyCompletePairs() { - let coverage = Self.privateAXCustomActionCoverage([ - RunnerAXSnapshotCustomActionsReadKey: 12, - RunnerAXSnapshotCustomActionsCandidatesKey: 19, - ]) - XCTAssertEqual(coverage?.read, 12) - XCTAssertEqual(coverage?.candidates, 19) - - // Absent key = the capture never asked; it must not read as (0, 0), which - // would warn "0 of 0" on every default capture. - XCTAssertNil(Self.privateAXCustomActionCoverage(nil)) - // A half-present pair cannot express a ratio, so it is dropped whole. - XCTAssertNil( - Self.privateAXCustomActionCoverage([RunnerAXSnapshotCustomActionsReadKey: 12])) - } - - /// The AX call cannot be cancelled once issued, so the read deadline frees - /// only the caller — the call keeps running. Without containment, repeating - /// `snapshot --actions` against a wedged element would stack orphaned reads, - /// all sharing one XCAXClient. This pins the containment: one serial queue and - /// a single-flight refusal that adds no work while a read is outstanding. - func testHungCustomActionReadIsContainedAndRecovers() { - let hung = HungAXClientForTesting() - let element = NSObject() - let dispatchesBefore = RunnerAXSnapshotBridge.customActionReadDispatchCount() - let blockedBefore = RunnerAXSnapshotBridge.customActionReadBlockedCount() - defer { hung.release() } - - // 1. First read wedges. The caller is freed by the deadline, but the call is - // still out there, so it stays counted in flight. - var completed = ObjCBool(true) - let firstStarted = Date() - let first = RunnerAXSnapshotBridge.customActionNames( - forElement: element, axClient: hung, completed: &completed) - XCTAssertNil(first) - XCTAssertFalse(completed.boolValue) - XCTAssertGreaterThanOrEqual(-firstStarted.timeIntervalSinceNow, 0.9) - XCTAssertEqual(RunnerAXSnapshotBridge.customActionReadsInFlight(), 1) - XCTAssertEqual( - RunnerAXSnapshotBridge.customActionReadDispatchCount(), dispatchesBefore + 1) - - // 2. Repeats do NOT accumulate: no new dispatch, still exactly one in - // flight, and every repeat is refused by single-flight admission. - for _ in 0..<5 { - XCTAssertNil( - RunnerAXSnapshotBridge.customActionNames( - forElement: element, axClient: hung, completed: &completed)) - XCTAssertFalse(completed.boolValue) - } - XCTAssertEqual(RunnerAXSnapshotBridge.customActionReadsInFlight(), 1) - XCTAssertEqual( - RunnerAXSnapshotBridge.customActionReadDispatchCount(), dispatchesBefore + 1) - XCTAssertEqual(RunnerAXSnapshotBridge.customActionReadBlockedCount(), blockedBefore + 5) - - // 3. A capture in that state discloses the skip rather than presenting the - // unread elements as action-free — and spends no read budget doing it. - let leaf = RunnerAXSnapshotFrontier() - leaf.snapshot = FrontierSnapshotWithElementForTesting() - leaf.node = NSMutableDictionary(dictionary: ["label": "feedItem", "children": []]) - let coverage = RunnerAXSnapshotBridge.annotateCustomActions( - onMergedLeaves: [leaf], axClient: hung, limit: 12, rootFrame: .zero, deadline: nil) - XCTAssertEqual(coverage[RunnerAXSnapshotCustomActionsBlockedKey] as? Bool, true) - XCTAssertEqual(coverage[RunnerAXSnapshotCustomActionsReadKey] as? Int, 0) - XCTAssertEqual(coverage[RunnerAXSnapshotCustomActionsCandidatesKey] as? Int, 1) - XCTAssertEqual( - RunnerAXSnapshotBridge.customActionReadDispatchCount(), dispatchesBefore + 1) - - // And the rendered verdict names the hang, not the scroll remedy. - let blockedWarnings = Self.customActionCoverageWarnings( - Self.privateAXCustomActionCoverage(coverage)!) - XCTAssertEqual(blockedWarnings.count, 1) - XCTAssertTrue(blockedWarnings[0].contains("still hung")) - XCTAssertFalse(blockedWarnings[0].contains("Scroll")) - - // 4. Recovery: once the wedged call returns, reads resume by themselves. - hung.release() - let recovered = expectation(description: "in-flight drains") - DispatchQueue.global().async { - while RunnerAXSnapshotBridge.customActionReadsInFlight() > 0 { - usleep(20_000) - } - recovered.fulfill() - } - wait(for: [recovered], timeout: 5) - - completed = ObjCBool(false) - XCTAssertNil( - RunnerAXSnapshotBridge.customActionNames( - forElement: element, axClient: hung, completed: &completed)) - // Completed (the fake answers nil actions), which is the point: the pass is - // live again rather than latched off. - XCTAssertTrue(completed.boolValue) - XCTAssertEqual( - RunnerAXSnapshotBridge.customActionReadDispatchCount(), dispatchesBefore + 2) - } - - /// The element budget bounds how many elements we read; these caps bound what - /// any ONE element can put in the response. Clipping must be reported, since - /// a clipped list looks exactly like a complete one. - func testActionNamesAreCappedPerElementAndReported() { - var truncated = ObjCBool(true) - - // Under both caps: untouched, nothing to report. - let small = ["Reply", "Repost"] - XCTAssertEqual( - RunnerAXSnapshotBridge.cappedActionNames(small, truncated: &truncated), small) - XCTAssertFalse(truncated.boolValue) - - // More actions than the per-element cap: clipped to the first 8, reported. - let many = (1...20).map { "Action \($0)" } - let cappedMany = RunnerAXSnapshotBridge.cappedActionNames(many, truncated: &truncated) - XCTAssertEqual(cappedMany.count, 8) - XCTAssertEqual(cappedMany.first, "Action 1") - XCTAssertTrue(truncated.boolValue) - - // A single very long name is shortened, reported, and stays one string. - let long = String(repeating: "a", count: 500) - let cappedLong = RunnerAXSnapshotBridge.cappedActionNames([long], truncated: &truncated) - XCTAssertEqual(cappedLong.count, 1) - XCTAssertTrue(truncated.boolValue) - XCTAssertLessThan(cappedLong[0].count, long.count) - XCTAssertTrue(cappedLong[0].hasSuffix("…")) - - // Empty input is not "truncated". - XCTAssertEqual(RunnerAXSnapshotBridge.cappedActionNames([], truncated: &truncated), []) - XCTAssertFalse(truncated.boolValue) - } - - /// A capped pass must say so; a complete one must stay silent. - func testPartialCustomActionPassIsDisclosedAndCompleteOneIsNot() { - let partial = SnapshotQuality( - state: "recovered", backend: "private-ax", reason: nil, reasonCode: "requested-backend", - effectiveDepth: nil, collapsedLeafIndexes: nil, - customActions: SnapshotCustomActionCoverage(read: 12, candidates: 19, truncated: 0, blocked: false)) - let message = Self.legacyQualityMessage(partial) - XCTAssertTrue(message?.contains("12 of 19 merged elements") == true) - XCTAssertTrue(message?.contains("remaining 7") == true) - XCTAssertTrue(message?.contains("Scroll them into view") == true) - - // Every candidate read: nothing to disclose, and a healthy capture stays silent. - let complete = SnapshotQuality( - state: "healthy", backend: "private-ax", reason: nil, reasonCode: nil, - effectiveDepth: nil, collapsedLeafIndexes: nil, - customActions: SnapshotCustomActionCoverage(read: 19, candidates: 19, truncated: 0, blocked: false)) - XCTAssertNil(Self.legacyQualityMessage(complete)) - - // A healthy capture with an incomplete pass still discloses — the guard must - // not key the disclosure off degradation state. - let healthyButCapped = SnapshotQuality( - state: "healthy", backend: "private-ax", reason: nil, reasonCode: nil, - effectiveDepth: nil, collapsedLeafIndexes: nil, - customActions: SnapshotCustomActionCoverage(read: 12, candidates: 19, truncated: 0, blocked: false)) - XCTAssertTrue( - Self.legacyQualityMessage(healthyButCapped)?.contains("12 of 19") == true) - } - - /// Action names annotated by the bridge must survive into the emitted node — - /// the whole point of the capture is that the merged card names its hidden - /// affordances. - func testPrivateAXNodesCarryAnnotatedCustomActions() { - let tree: [String: Any] = [ - "type": Int(XCUIElement.ElementType.application.rawValue), - "label": "Blue Sky", - "frame": ["x": 0, "y": 0, "width": 390, "height": 844], - "children": [ - [ - "type": Int(XCUIElement.ElementType.link.rawValue), - "label": "feedItem-by-whiskers.test", - "frame": ["x": 0, "y": 100, "width": 390, "height": 200], - "actions": ["Reply", "Repost", "Open post options menu"], - "children": [], - ], - [ - "type": Int(XCUIElement.ElementType.button.rawValue), - "label": "Compose", - "frame": ["x": 300, "y": 700, "width": 60, "height": 60], - "children": [], - ], - ], - ] - let nodes = privateAXAcquisition( - rawRoot: tree, - hint: CaptureHint( - projection: .regular, depth: nil, regularPresentedDepth: nil, - interactiveOnly: false, customActions: false) - ) - - let card = nodes.first { $0.label == "feedItem-by-whiskers.test" } - XCTAssertEqual(card?.actions, ["Reply", "Repost", "Open post options menu"]) - // A node the bridge did not annotate stays absent, not empty. - XCTAssertNil(nodes.first { $0.label == "Compose" }?.actions) - } - - func testPrivateAXAcquisitionDoesNotInterpretScope() { - let tree: [String: Any] = [ - "type": 1, "label": "App", - "children": [ - [ - "type": 9, "identifier": "homeScreen", - "children": [ - ["type": 48, "label": "Post body without the scope text", "children": []] - ], - ], - ["type": 9, "label": "unrelated sibling", "children": []], - ], - ] - // Scope never reaches acquisition: the hint derived for a scoped request carries no scope, - // and the backend has no way to interpret one. - let nodes = privateAXAcquisition( - rawRoot: tree, - hint: SnapshotPresentation.captureHint( - for: PresentationOptions( - interactiveOnly: false, - depth: nil, - scope: "homeScreen", - raw: false - ) - ) - ) - - let labels = nodes.compactMap { $0.label ?? $0.identifier } - XCTAssertTrue(labels.contains("homeScreen")) - // Descendants of the matched scope are included even when they do not contain the text. - XCTAssertTrue(labels.contains("Post body without the scope text")) - XCTAssertTrue(labels.contains("unrelated sibling")) - } - - func testPrivateAXInteractiveFiltersLoginLikeHiddenDrawer() throws { - let tree: [String: Any] = [ - "type": Int(XCUIElement.ElementType.application.rawValue), - "label": "Blue Sky", - "frame": ["x": 0, "y": 0, "width": 390, "height": 844], - "children": [ - [ - "type": Int(XCUIElement.ElementType.scrollView.rawValue), - "frame": ["x": 0, "y": 0, "width": 390, "height": 844], - "children": [ - [ - "type": Int(XCUIElement.ElementType.image.rawValue), - "label": "Callstack", - "frame": ["x": 145, "y": 104, "width": 100, "height": 100], - "children": [], - ], - [ - "type": Int(XCUIElement.ElementType.staticText.rawValue), - "label": "Welcome back", - "frame": ["x": 32, "y": 260, "width": 326, "height": 32], - "children": [], - ], - [ - "type": Int(XCUIElement.ElementType.textField.rawValue), - "label": "Email", - "identifier": "login.email", - "frame": ["x": 32, "y": 348, "width": 326, "height": 48], - "children": [], - ], - [ - "type": Int(XCUIElement.ElementType.secureTextField.rawValue), - "label": "Password", - "identifier": "login.password", - "frame": ["x": 32, "y": 412, "width": 326, "height": 48], - "children": [], - ], - [ - "type": Int(XCUIElement.ElementType.button.rawValue), - "label": "Sign in", - "identifier": "login.submit", - "frame": ["x": 32, "y": 492, "width": 326, "height": 52], - "children": [], - ], - [ - "type": Int(XCUIElement.ElementType.link.rawValue), - "label": "Forgot password?", - "frame": ["x": 128, "y": 568, "width": 134, "height": 32], - "children": [], - ], - [ - "type": Int(XCUIElement.ElementType.button.rawValue), - "label": "Admin settings", - "frame": ["x": -260, "y": 184, "width": 220, "height": 44], - "children": [], - ], - [ - "type": Int(XCUIElement.ElementType.other.rawValue), - "frame": ["x": 16, "y": 184, "width": 220, "height": 44], - "children": [], - ], - ], - ] - ], - ] - let viewport = CGRect(x: 0, y: 0, width: 390, height: 844) - let hint = CaptureHint( - projection: .regular, depth: nil, regularPresentedDepth: nil, - interactiveOnly: true, customActions: false) - let acquired = SnapshotGeometrySpace.normalized( - nodes: privateAXAcquisition(rawRoot: tree, hint: hint), - viewport: viewport, - interfaceOrientation: RunnerInterfaceOrientation.portrait - ) - // Acquisition serializes the drawer too; the shared fold is what hides it (#1797). - XCTAssertTrue(acquired.compactMap(\.label).contains("Admin settings")) - - let capture = try SnapshotPresentation.presentRegular( - SnapshotAcquisition( - hint: hint, nodes: acquired, truncated: false, effectiveDepth: nil, viewport: viewport), - options: PresentationOptions(interactiveOnly: true, depth: nil, scope: nil, raw: false), - policy: .cursorProjected - ) - let labels = capture.nodes.compactMap { $0.label } - XCTAssertEqual( - labels, - ["Blue Sky", "Callstack", "Welcome back", "Email", "Password", "Sign in", "Forgot password?"] - ) - XCTAssertFalse(labels.contains("Admin settings")) - } -} - -/// Stands in for an AX client whose `attributesForElement:` never returns — -/// the wedged-server case the containment exists for. `release()` lets the -/// hung call finish so recovery is observable. -private final class HungAXClientForTesting: NSObject { - private let gate = DispatchSemaphore(value: 0) - private let releasedOnce = NSLock() - private var released = false - - @objc(attributesForElement:attributes:error:) - func attributes(forElement element: Any, attributes: Any, error: NSErrorPointer) -> Any? { - releasedOnce.lock() - let alreadyReleased = released - releasedOnce.unlock() - // Once the wedge clears, the server answers normally again — that is what - // makes the recovery leg a recovery rather than a second hang. - if alreadyReleased { - return nil - } - gate.wait() - return nil - } - - func release() { - releasedOnce.lock() - defer { releasedOnce.unlock() } - guard !released else { return } - released = true - gate.signal() - } -} - -/// Minimal snapshot stand-in whose accessibilityElement resolves (so the -/// extension proceeds to the request) while the paired fake client cannot -/// serve it — the failed-re-root miss path. -private final class FrontierSnapshotWithElementForTesting: NSObject { - @objc let accessibilityElement = NSObject() -} - -/// The vanished-element case: KVC resolves the property and gets nil. -private final class FrontierSnapshotWithoutElementForTesting: NSObject { - @objc let accessibilityElement: NSObject? = nil -} -#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift index 9e06410820..c7bc8312fc 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift @@ -68,15 +68,25 @@ extension RunnerTests { guard waitUntilAlertButtonHittable(button, deadline: deadline) else { return alertVerificationResponse(.timedOut, action: action, activated: false) } - // The hittable read above is this activation's readiness gate, so XCTest's pre-synthesis wait - // adds nothing and can cost more than the command has: the tap would land after the deadline + let buttonFrame = button.frame + NSLog( + "AGENT_DEVICE_RUNNER_ALERT_ACTIVATION action=%@ label=%@ frame=(%.1f,%.1f,%.1f,%.1f) point=(%.1f,%.1f)", + action, + button.label, + buttonFrame.origin.x, buttonFrame.origin.y, buttonFrame.size.width, buttonFrame.size.height, + buttonFrame.midX, buttonFrame.midY + ) + // The hittable read above is this activation's readiness gate, so XCTest's pre-synthesis waits + // add nothing and can cost more than the command has: the tap would land after the deadline // expired and the alert would be answered by a button the caller was told nothing about // (#2546). The post-tap settle stays, because the verification below reads the alert this tap // replaces; an alert that dismisses and presents an identical replacement passes through a // window with no alert, and a first read landing there reports a dismissal nothing proved. var outcome = RunnerInteractionOutcome.performed - withBoundedInteractionIdleTimeoutIfSupported(alert.ownerApp, waits: .preEventSkipped) { - outcome = activateElement(app: alert.ownerApp, element: button, action: "alert \(action)") + withUIInterruptionHandlingDisabledIfSupported(alert.ownerApp) { + withBoundedInteractionIdleTimeoutIfSupported(alert.ownerApp, waits: .preEventSkipped) { + outcome = activateElement(app: alert.ownerApp, element: button, action: "alert \(action)") + } } if let response = unsupportedResponse(for: outcome) { return response @@ -106,6 +116,22 @@ extension RunnerTests { ) } + /// Before each event XCTest looks for SpringBoard elements over the target and hands them to its + /// interruption handler, which waits up to 15 s for a notification banner to leave and taps a + /// button of its own choosing on any other alert. An alert command answers exactly the alert it + /// resolved, with the button it chose, before its deadline, so it opts out of both. + private func withUIInterruptionHandlingDisabledIfSupported(_ target: XCUIApplication, operation: () -> Void) { + let key = "doesNotHandleUIInterruptions" + guard target.responds(to: NSSelectorFromString("setDoesNotHandleUIInterruptions:")) else { + operation() + return + } + let previous = target.value(forKey: key) as? NSNumber + target.setValue(true, forKey: key) + defer { target.setValue(previous?.boolValue ?? false, forKey: key) } + operation() + } + private func runnerAlert(_ modal: ResolvedBlockingSystemModal) -> RunnerAlert? { let buttons = modal.actions.filter { isEnabledElement($0) } guard !buttons.isEmpty else { @@ -135,22 +161,26 @@ extension RunnerTests { elements.first { isVisibleElement($0) } } + /// The marker is matched inside XCTest's query, so the screen is read once per query. Reading each + /// descendant instead costs one round trip per element, and on a screen whose tree changes while + /// it is read (a loading web view) each vanished element adds XCTest's retry cycle. `containing` + /// also matches a window that is itself the marker. private func firstDismissPopupWindow(in app: XCUIApplication) -> XCUIElement? { - safeElementsQuery { - app.windows.allElementsBoundByIndex - }.first { window in - if !isVisibleElement(window) { return false } - if isDismissPopupMarker(window.label) || isDismissPopupMarker(window.identifier) { - return true - } - return safeElementsQuery { - window.descendants(matching: .any).allElementsBoundByIndex - }.contains { descendant in - isDismissPopupMarker(descendant.label) || isDismissPopupMarker(descendant.identifier) - } - } + firstExistingElement(in: safeElementsQuery { + app.windows.containing(Self.dismissPopupMarker).allElementsBoundByIndex + }) } + /// The one definition of a popover's dismiss region: a label or identifier that reads "dismiss + /// popup", in any case, with any surrounding whitespace. XCTest queries take it as a format predicate. + private static let dismissPopupMarkerPattern = #"\s*dismiss popup\s*"# + private static let dismissPopupMarker = NSPredicate( + format: "label MATCHES[c] %@ OR identifier MATCHES[c] %@", + dismissPopupMarkerPattern, + dismissPopupMarkerPattern + ) + private static let dismissPopupMarkerText = NSPredicate(format: "SELF MATCHES[c] %@", dismissPopupMarkerPattern) + private func chooseAlertButton(_ buttons: [XCUIElement], action: String) -> XCUIElement? { if action == "accept" { if let accept = buttons.first(where: { isAcceptButton($0.label) }) { @@ -162,7 +192,7 @@ extension RunnerTests { return buttons.first(where: { isDismissButton($0.label) }) ?? buttons.last } - private func isAcceptButton(_ label: String) -> Bool { + func isAcceptButton(_ label: String) -> Bool { let normalized = label.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() return [ "ok", @@ -175,12 +205,6 @@ extension RunnerTests { ].contains(normalized) || normalized.hasPrefix("confirm") } -#if AGENT_DEVICE_RUNNER_UNIT_TESTS - func testAlertAcceptTreatsOpenAsAffirmative() { - XCTAssertTrue(isAcceptButton("Open")) - } -#endif - private func isDismissButton(_ label: String) -> Bool { [ "cancel", @@ -265,7 +289,7 @@ extension RunnerTests { return hittable } - private func isDismissPopupMarker(_ label: String) -> Bool { - label.trimmingCharacters(in: .whitespacesAndNewlines).caseInsensitiveCompare("dismiss popup") == .orderedSame + func isDismissPopupMarker(_ text: String) -> Bool { + Self.dismissPopupMarkerText.evaluate(with: text) } } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AlertObservation.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AlertObservation.swift index f6d4cb5335..6ef66572c9 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AlertObservation.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AlertObservation.swift @@ -68,9 +68,8 @@ extension RunnerTests { } private func containsDismissPopupMarker(_ snapshot: XCUIElementSnapshot) -> Bool { - [snapshot.label, snapshot.identifier].contains { - $0.trimmingCharacters(in: .whitespacesAndNewlines).caseInsensitiveCompare("dismiss popup") == .orderedSame - } || snapshot.children.contains { containsDismissPopupMarker($0) } + [snapshot.label, snapshot.identifier].contains { isDismissPopupMarker($0) } || + snapshot.children.contains { containsDismissPopupMarker($0) } } private func alertPresentation(_ snapshot: XCUIElementSnapshot) -> RunnerAlertPresentation { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+BlockingSystemModalResolution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+BlockingSystemModalResolution.swift index 2b92c09bdc..687d36d8b8 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+BlockingSystemModalResolution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+BlockingSystemModalResolution.swift @@ -79,39 +79,3 @@ enum RemoteHostedSystemModalPolicy { state == .runningForeground } } - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS -extension RunnerTests { - func testRemoteHostProbeRunsOnlyWhenSpringboardModalHasNoActions() { - XCTAssertTrue(RemoteHostedSystemModalPolicy.shouldProbeRemoteHost(springboardActionCount: 0)) - XCTAssertFalse(RemoteHostedSystemModalPolicy.shouldProbeRemoteHost(springboardActionCount: 1)) - XCTAssertFalse(RemoteHostedSystemModalPolicy.shouldProbeRemoteHost(springboardActionCount: 3)) - } - - func testRemoteHostStateGateFailsClosedToForeground() { - XCTAssertTrue(RemoteHostedSystemModalPolicy.isEligibleHostState(.runningForeground)) - XCTAssertFalse(RemoteHostedSystemModalPolicy.isEligibleHostState(.runningBackground)) - XCTAssertFalse(RemoteHostedSystemModalPolicy.isEligibleHostState(.notRunning)) - XCTAssertFalse(RemoteHostedSystemModalPolicy.isEligibleHostState(.unknown)) - } - - // No SpringBoard host (`hasSpringBoardSystemModalHost`) means modal resolution must return - // `.absent` without probing com.apple.springboard (#1351). Written for tvOS, where no lane - // ever executed it; `resolveBlockingSystemModal` takes that decision at RUNTIME off the same - // flag on macOS, so the host lane runs the real branch on every PR. - // - // Its former sibling `testBlockingSystemAlertSnapshotIsNilOnTvOS` is deleted rather than - // widened: `blockingSystemAlertSnapshot` is `#if os(macOS) return nil`, so on the only lane - // that could run it the assertion would pin a compile-time literal — a green that no change - // to the runner could turn red. The runtime gate it meant to cover is this test's subject, - // and the nil it returns on macOS is the compiler's business, not a test's. - #if os(tvOS) || os(macOS) - func testResolveBlockingSystemModalIsAbsentWithoutSpringBoardHost() { - guard case .absent = resolveBlockingSystemModal(deadline: .distantFuture) else { - XCTFail("blocking system-modal resolution must be .absent without a SpringBoard host") - return - } - } - #endif -} -#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandDispatch.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandDispatch.swift new file mode 100644 index 0000000000..f58f712686 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandDispatch.swift @@ -0,0 +1,630 @@ +import XCTest +import AgentDeviceSnapshotPresentation + +extension RunnerTests { + // MARK: - Main Thread Dispatch + + func executeAccepted(command: Command) throws -> Response { + commandJournal.start(command: command) + pendingTargetActivation = nil + do { + let response = try executeDispatched(command: command) + commandJournal.finish(command: command, response: response) + guard let fact = pendingTargetActivation else { return response } + // Stamped after `finish`, like the uptime anchor: a journal-replayed result carries no + // activation fact, because the command that paid for it is the one being replayed, not one + // that just repaired foreground (#2682). + pendingTargetActivation = nil + return response.stampingTargetActivation(fact) + } catch { + pendingTargetActivation = nil + commandJournal.fail(command: command, error: error) + throw error + } + } + + func executeStatus(command: Command) -> Response { + guard + let statusCommandId = command.statusCommandId?.trimmedNonEmpty + else { + return Response( + ok: false, + error: ErrorPayload( + code: "INVALID_ARGS", + message: "status requires statusCommandId", + hint: "Set statusCommandId to the commandId of the runner command to inspect." + ) + ) + } + return Response(ok: true, data: commandJournal.status(normalizedCommandId: statusCommandId)) + } + + func executeUptime() -> Response { + // Placeholder value: the transport layer (jsonResponse) overwrites currentUptimeMs with a + // fresher send-time stamp on every ok response; kept so direct callers still get a value. + Response( + ok: true, + data: DataPayload(currentUptimeMs: currentUptimeMs()) + ) + } + + struct ActiveCommandContext { + let app: XCUIApplication + /// Set when `app` is a system surface served in place over the still-bound session app (#2438). + var systemSurface: SystemSurfaceHost? = nil + } + + enum ActiveCommandPreparation { + case response(Response) + case context(ActiveCommandContext) + } + + private func runnerBusyResponse(command: Command, abandonedForSeconds: TimeInterval) -> Response { + NSLog( + "AGENT_DEVICE_RUNNER_BUSY command=%@ commandId=%@ abandonedForSeconds=%.1f", + command.command.rawValue, + command.commandId ?? "", + abandonedForSeconds + ) + return Response( + ok: false, + error: ErrorPayload( + code: "RUNNER_BUSY", + message: + "The iOS runner is still finishing a previous command that exceeded its execution watchdog (usually an accessibility capture on a heavy or animating screen).", + hint: + "Wait a few seconds and retry. If snapshots keep failing on this screen, use screenshot as visual truth and interact by coordinates, or navigate to another screen." + ) + ) + } + + private func runnerWedgedResponse(command: Command, abandonedForSeconds: TimeInterval) -> Response { + NSLog( + "AGENT_DEVICE_RUNNER_WEDGED command=%@ commandId=%@ abandonedForSeconds=%.1f", + command.command.rawValue, + command.commandId ?? "", + abandonedForSeconds + ) + return Response( + ok: false, + error: ErrorPayload( + code: "RUNNER_WEDGED", + message: + "The iOS runner main thread has been stuck in abandoned work for \(Int(abandonedForSeconds)) seconds and cannot recover on its own.", + hint: + "The runner session will be restarted. Retry the command after the restart; if this screen keeps wedging captures, use screenshot as visual truth and interact by coordinates." + ) + ) + } + + private func runnerUnavailableResponse(command: Command) -> Response? { + switch currentMainThreadBusyState() { + case .idle: + return nil + case .busy(let abandonedForSeconds): + return runnerBusyResponse(command: command, abandonedForSeconds: abandonedForSeconds) + case .wedged(let abandonedForSeconds): + return runnerWedgedResponse(command: command, abandonedForSeconds: abandonedForSeconds) + } + } + + func executeDispatched(command: Command) throws -> Response { + // XCTest work cannot be cancelled mid-flight: once the watchdog abandons a main-queue + // block, queueing more main-thread commands behind it only buries the runner deeper. + // Refuse fast instead so the daemon backs off while the abandoned work drains; past the + // wedge threshold, escalate so the daemon recycles this runner (#1105). + if let unavailable = runnerUnavailableResponse(command: command) { + return unavailable + } + let alertDeadline = command.command == .alert + ? Date().addingTimeInterval(Self.alertCommandTimeout(timeoutMs: command.timeoutMs)) + : nil + // Resolve this before the command's outer main-thread block. If the bounded probe abandons + // slow XCTest enumeration, return the established recoverable response instead of queueing + // command preparation behind work that may outlive the 30-second command watchdog. + let routeToSpringboard = shouldRouteToSpringboardBlockingSystemModal(command) + if let unavailable = runnerUnavailableResponse(command: command) { + return unavailable + } + if command.command == .snapshot { + return try executeSnapshotDispatched(command: command) + } + if command.command == .alert, let deadline = alertDeadline { + return try runMainThreadWork( + "command_execution", + timeout: max(0.001, deadline.timeIntervalSinceNow), + timeoutError: mainThreadExecutionTimeoutError + ) { + try self.executeOnMainSafely( + command: command, + alertDeadline: deadline, + routeToSpringboard: routeToSpringboard + ) + } + } + return try runMainThreadWork( + "command_execution", + timeout: mainThreadExecutionTimeout, + timeoutError: mainThreadExecutionTimeoutError + ) { + try self.executeOnMainSafely(command: command, routeToSpringboard: routeToSpringboard) + } + } + + // MARK: - Command Handling + + private func executeOnMainSafely( + command: Command, + alertDeadline: Date? = nil, + routeToSpringboard: Bool + ) throws -> Response { + var hasRetried = false + while true { + var response: Response? + var swiftError: Error? + let failureCountBefore = currentXCTestFailureCount() + let exceptionMessage = RunnerObjCExceptionCatcher.catchException({ + do { + response = try self.executeOnMain( + command: command, + alertDeadline: alertDeadline, + routeToSpringboard: routeToSpringboard + ) + } catch { + swiftError = error + } + }) + + if let exceptionMessage { + invalidateCachedTarget(reason: "objc_exception") + if !hasRetried, shouldRetryException(command, message: exceptionMessage) { + NSLog( + "AGENT_DEVICE_RUNNER_RETRY command=%@ reason=objc_exception", + command.command.rawValue + ) + hasRetried = true + sleepFor(retryCooldown) + continue + } + throw NSError( + domain: RunnerErrorDomain.exception, + code: RunnerErrorCode.objcException, + userInfo: [NSLocalizedDescriptionKey: exceptionMessage] + ) + } + if let swiftError { + throw swiftError + } + guard let response else { + throw NSError( + domain: RunnerErrorDomain.general, + code: RunnerErrorCode.commandReturnedNoResponse, + userInfo: [NSLocalizedDescriptionKey: "command returned no response"] + ) + } +#if AGENT_DEVICE_RUNNER_UNIT_TESTS + // #1605 merge gate: the REAL gesture already executed above; recording a + // production-shaped issue here makes the per-command failure-count + // conversion below fire exactly as in the field (bsky-24: activation + // lands, bookkeeping records a failure). Compiled out of production. + if consumeInjectedTapRecordedFailureForTesting(command: command.command) { + record( + XCTIssue( + type: .assertionFailure, + compactDescription: "Injected tap recorded-failure (#1605 corroboration merge gate)" + ) + ) + } +#endif + if didRecordXCTestFailure(since: failureCountBefore), + let failureResponse = xctestRecordedFailureResponse(command: command, response: response) + { + invalidateCachedTarget(reason: "xctest_recorded_failure") + return failureResponse + } + if !hasRetried, shouldRetryCommand(command), shouldRetryResponse(response) { + NSLog( + "AGENT_DEVICE_RUNNER_RETRY command=%@ reason=response_unavailable", + command.command.rawValue + ) + hasRetried = true + invalidateCachedTarget(reason: "response_unavailable") + sleepFor(retryCooldown) + continue + } + return response + } + } + + /// The dispatched snapshot recovery loop: read-only retry + XCTest-recorded-failure invalidation, + /// matching what `executeOnMainSafely` gives the generic path. `perform` runs the capture and its + /// own bounded main-thread work. + func executeDispatchedWithRecovery( + command: Command, + perform: () throws -> Response + ) throws -> Response { + var hasRetried = false + while true { + let failureCountBefore = try runMainThreadWork( + "recorded_failure_count", + timeout: mainThreadExecutionTimeout, + timeoutError: mainThreadExecutionTimeoutError + ) { + self.currentXCTestFailureCount() + } + let response = try perform() + // Recovered independently — re-entering main for bookkeeping would queue behind the still- + // abandoned XCTest query and re-stall the command (#1244), so skip it until that work drains. + if hasAbandonedMainThreadWork() { + NSLog( + "AGENT_DEVICE_RUNNER_DISPATCH_RECOVERY_SKIPPED_XCTEST_OCCUPIED command=%@", + command.command.rawValue + ) + return response + } + let recordedFailureResponse = try runMainThreadWork( + "recorded_failure_count", + timeout: mainThreadExecutionTimeout, + timeoutError: mainThreadExecutionTimeoutError + ) { + self.didRecordXCTestFailure(since: failureCountBefore) + ? self.xctestRecordedFailureResponse(command: command, response: response) + : nil + } + if let recordedFailureResponse { + try runMainThreadWork( + "target_invalidation", + timeout: mainThreadExecutionTimeout, + timeoutError: mainThreadExecutionTimeoutError + ) { + self.invalidateCachedTarget(reason: "xctest_recorded_failure") + } + return recordedFailureResponse + } + if !hasRetried, shouldRetryCommand(command), shouldRetryResponse(response) { + NSLog( + "AGENT_DEVICE_RUNNER_RETRY command=%@ reason=response_unavailable", + command.command.rawValue + ) + hasRetried = true + try runMainThreadWork( + "target_invalidation", + timeout: mainThreadExecutionTimeout, + timeoutError: mainThreadExecutionTimeoutError + ) { + self.invalidateCachedTarget(reason: "response_unavailable") + self.sleepFor(self.retryCooldown) + } + continue + } + return response + } + } + + private func executeOnMain( + command: Command, + alertDeadline: Date?, + routeToSpringboard: Bool + ) throws -> Response { + let preparation = prepareActiveCommandContext( + command: command, + routeToSpringboard: routeToSpringboard + ) + let activeApp: XCUIApplication + switch preparation { + case .response(let response): + return response + case .context(let context): + activeApp = context.app + } + + switch command.command { + case .status: + return executeStatus(command: command) + case .targetReset: + return resetTargetAfterExternalRelaunch() + case .shutdown: + stopRecordingIfNeeded() + return Response(ok: true, data: DataPayload(message: "shutdown")) + case .recordStart: + guard + let requestedOutPath = command.outPath?.trimmingCharacters(in: .whitespacesAndNewlines), + !requestedOutPath.isEmpty + else { + return Response(ok: false, error: ErrorPayload(message: "recordStart requires outPath")) + } + let hasAppBundleId = !(command.appBundleId? + .trimmingCharacters(in: .whitespacesAndNewlines) + .isEmpty ?? true) + guard hasAppBundleId else { + return Response(ok: false, error: ErrorPayload(message: "recordStart requires appBundleId")) + } + if activeRecording != nil { + return Response(ok: false, error: ErrorPayload(message: "recording already in progress")) + } + if let requestedFps = command.fps, (requestedFps < minRecordingFps || requestedFps > maxRecordingFps) { + return Response(ok: false, error: ErrorPayload(message: "recordStart fps must be between \(minRecordingFps) and \(maxRecordingFps)")) + } + do { + let resolvedOutPath = resolveRecordingOutPath(requestedOutPath) + let fpsLabel = command.fps.map(String.init) ?? String(RunnerTests.defaultRecordingFps) + NSLog( + "AGENT_DEVICE_RUNNER_RECORD_START requestedOutPath=%@ resolvedOutPath=%@ fps=%@", + requestedOutPath, + resolvedOutPath, + fpsLabel + ) + let recorder = ScreenRecorder( + outputPath: resolvedOutPath, + fps: command.fps.map { Int32($0) } + ) + try startRecording(recorder) { [weak self] in + guard let self else { return .failure(.unresolvedScreen) } + return self.captureRunnerFrameResult(app: activeApp) + } + activeRecording = recorder + return Response(ok: true, data: DataPayload(message: "recording started")) + } catch { + activeRecording = nil + return Response(ok: false, error: Self.recordingStartErrorPayload(for: error)) + } + case .recordStop: + guard let recorder = activeRecording else { + // The runner protocol is the durable cleanup primitive. A daemon may crash after the + // native stop succeeds but before it commits the resource transition, so exact-owner + // recovery must be able to repeat this command safely. Public `record stop` still owns + // its user-facing no-active validation through the daemon session manifest. + return Response(ok: true, data: DataPayload(message: "recording already stopped")) + } + do { + try recorder.stop() + activeRecording = nil + return Response(ok: true, data: DataPayload(message: "recording stopped")) + } catch { + activeRecording = nil + return Response(ok: false, error: ErrorPayload(message: "failed to stop recording: \(error.localizedDescription)")) + } + case .uptime: + return executeUptime() + case .activate: + guard + let bundleId = command.appBundleId?.trimmingCharacters(in: .whitespacesAndNewlines), + !bundleId.isEmpty + else { + return Response(ok: false, error: ErrorPayload(message: "activate requires appBundleId")) + } + // prepareActiveCommandContext already activated this bundle. Keep this case as the + // explicit acknowledgement after that preflight, not as a second activation. + return Response(ok: true, data: DataPayload(message: "app activated")) + case .terminate: + guard + let bundleId = command.appBundleId?.trimmingCharacters(in: .whitespacesAndNewlines), + !bundleId.isEmpty + else { + return Response(ok: false, error: ErrorPayload(message: "terminate requires appBundleId")) + } + XCUIApplication(bundleIdentifier: bundleId).terminate() + if currentBundleId == bundleId { + invalidateCachedTarget(reason: "target_terminated") + } + return Response(ok: true, data: DataPayload(message: "app terminated")) + default: + break + } + return try executeOnMainPrepared( + command: command, + activeApp: activeApp, + alertDeadline: alertDeadline + ) + } + + func prepareActiveCommandContext( + command: Command, + routeToSpringboard: Bool = false + ) -> ActiveCommandPreparation { + var activeApp = currentApp ?? app + var systemSurface: SystemSurfaceHost? = nil + if routeToSpringboard { + activeApp = springboard + } else if shouldSkipAppActivationPreflight(command) { + activeApp = resolveAppWithoutActivation(command: command) + } else if let presented = presentedSystemSurfaceHost() { + // Serve and drive the presented surface IN PLACE: never activate it (that cancels what it + // presents) and never adopt it as the cached session target, so once it is gone the next + // command resolves back to the still-bound session app (#2438). + activeApp = presented.app + systemSurface = presented.host + if isInteractionCommand(command.command) { + applyInteractionStabilizationIfNeeded() + } + } else if !isRunnerLifecycleCommand(command.command) { + let normalizedBundleId = command.appBundleId? + .trimmingCharacters(in: .whitespacesAndNewlines) + let requestedBundleId = (normalizedBundleId?.isEmpty == true) ? nil : normalizedBundleId + if let bundleId = requestedBundleId, + let notRunning = notRunningReadResponse(command: command, bundleId: bundleId) + { + return .response(notRunning) + } + if let bundleId = requestedBundleId { + if currentBundleId != bundleId || currentApp == nil { + _ = activateTarget(bundleId: bundleId, reason: "bundle_changed") + } else { + refreshCachedTargetIfProcessChanged(bundleId: bundleId) + } + } else { + // Do not reuse stale bundle targets when the caller does not explicitly request one. + invalidateCachedTarget(reason: "missing_app_bundle") + } + + activeApp = currentApp ?? app + if let bundleId = requestedBundleId, targetNeedsActivation(activeApp) { + activeApp = activateTarget(bundleId: bundleId, reason: "stale_target") + } else if requestedBundleId == nil, targetNeedsActivation(activeApp) { + ensureRunnerHostAppActive(reason: "missing_app_bundle") + activeApp = app + } + + let skipExistenceWait = canUseFastForegroundAppGuard( + activeApp: activeApp, + requestedBundleId: requestedBundleId + ) + if !skipExistenceWait && !activeApp.waitForExistence(timeout: appExistenceTimeout) { + if let bundleId = requestedBundleId { + activeApp = activateTarget(bundleId: bundleId, reason: "missing_after_wait") + guard activeApp.waitForExistence(timeout: appExistenceTimeout) else { + return .response(Response(ok: false, error: .targetAppUnavailable(bundleId: bundleId))) + } + } else { + return .response(Response(ok: false, error: .targetAppUnavailable(bundleId: nil))) + } + } + + if isInteractionCommand(command.command) { + if let bundleId = requestedBundleId, activeApp.state != .runningForeground { + activeApp = activateTarget(bundleId: bundleId, reason: "interaction_foreground_guard") + } else if requestedBundleId == nil, activeApp.state != .runningForeground { + ensureRunnerHostAppActive(reason: "interaction_missing_app_bundle") + activeApp = app + } + let skipInteractionExistenceWait = canUseFastForegroundAppGuard( + activeApp: activeApp, + requestedBundleId: requestedBundleId + ) + if !skipInteractionExistenceWait && !activeApp.waitForExistence(timeout: 2) { + return .response( + Response(ok: false, error: .targetAppUnavailable(bundleId: requestedBundleId)) + ) + } + applyInteractionStabilizationIfNeeded() + } + } + return .context(ActiveCommandContext(app: activeApp, systemSurface: systemSurface)) + } + + /// A registered system surface host that is genuinely on screen, or nil. Presence is foreground + /// state, not tree content: a torn-down host still serves a rich tree, and it can only be + /// foreground-with-a-stale-tree if something activated it, which the open guard refuses. `state` + /// never activates and is cheap when the host is absent. See docs/adr/0004. + private func presentedSystemSurfaceHost() -> (host: SystemSurfaceHost, app: XCUIApplication)? { +#if os(iOS) + for host in SystemSurfaceHostRegistry.hosts { + let candidate = XCUIApplication(bundleIdentifier: host.bundleId) + if candidate.state == .runningForeground { + return (host, candidate) + } + } + return nil +#else + return nil +#endif + } + + func currentXCTestFailureCount() -> Int { + return testRun?.failureCount ?? 0 + } + + func didRecordXCTestFailure(since failureCountBefore: Int) -> Bool { + return currentXCTestFailureCount() > failureCountBefore + } + + func xctestRecordedFailureResponse(command: Command, response: Response) -> Response? { + guard response.ok else { return nil } + if response.data?.runnerFatal == true { + return nil + } + guard !isReadOnlyCommand(command), !isRunnerLifecycleCommand(command.command) else { + return nil + } + return Response( + ok: false, + error: ErrorPayload( + code: "XCTEST_RECORDED_FAILURE", + message: "XCTest recorded a failure while executing \(command.command.rawValue); the action may not have been performed.", + hint: "The iOS runner session was invalidated. Re-observe with a fresh snapshot before retrying; if the accessibility tree is unavailable, use screenshot plus coordinate commands instead of retrying the tap blindly." + ) + ) + } + + func shouldSkipAppActivationPreflight(_ command: Command) -> Bool { +#if os(iOS) + if command.command == .alert { + return true + } + // A hardware Action Button press belongs to the system, not to the session app: the Shortcut or + // App Intent behind it is expected to run whether that app is foregrounded, backgrounded, or + // terminated, and activating first would foreground exactly what the press should leave alone. + // The press keeps its recorded-failure conversion, which `isLifecycle` would have removed + // (#2699, #2702 review). + if command.command == .actionButton { + return true + } + // Coordinate-only synthesized taps can run after an AX-fatal foreground screen because they do not + // need app activation, window lookup, keyboard lookup, or element resolution. Selector/text + // interactions intentionally stay on the normal AX path because they need an element query. + // Scroll/drag/sequence keep the normal foreground guard and stabilization path. + guard command.text == nil, command.selectorKey == nil else { return false } + guard hasCachedTargetForActivationSkip(command: command) else { return false } + return isCoordinateOnlyTap(command) +#else + return false +#endif + } + + func shouldRouteToSpringboardBlockingSystemModal( + _ command: Command + ) -> Bool { +#if os(iOS) + guard isCoordinateOnlyTap(command) else { + return false + } + #if AGENT_DEVICE_RUNNER_UNIT_TESTS + if let override = blockingSystemModalPresenceOverrideForTesting { + return override + } + #endif + let probeDeadline = Date().addingTimeInterval(systemModalProbeBudget) + // Routing runs on the command queue, so this hands the probe a target rather than a bundle id it + // read across the main boundary: the penalty an abandoned probe arms carries the identity main + // holds once the probe starts, not one whose write was still queued behind the block that + // occupied main (#2781). + return boundedBlockingSystemAlertSnapshot( + deadline: probeDeadline, + penaltyTarget: .mainOwnedTarget + ) != nil +#else + return false +#endif + } + + private func isCoordinateOnlyTap(_ command: Command) -> Bool { + return command.command == .tap + && command.text == nil + && command.selectorKey == nil + && command.x != nil + && command.y != nil + } + + private func hasCachedTargetForActivationSkip(command: Command) -> Bool { + guard let currentApp, currentApp.state == .runningForeground else { return false } + guard let bundleId = command.appBundleId?.trimmingCharacters(in: .whitespacesAndNewlines), + !bundleId.isEmpty + else { + return true + } + return currentBundleId == bundleId + } + + func resolveAppWithoutActivation(command: Command) -> XCUIApplication { + guard let bundleId = command.appBundleId? + .trimmingCharacters(in: .whitespacesAndNewlines), + !bundleId.isEmpty + else { + return currentApp ?? app + } + if currentBundleId == bundleId, let currentApp { + return currentApp + } + return XCUIApplication(bundleIdentifier: bundleId) + } +} diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index e05ac00ae6..491b7962ad 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -1,1628 +1,7 @@ import XCTest import AgentDeviceSnapshotPresentation -#if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS) -import ObjectiveC.runtime - -private final class RunnerSynthesizedSwipeFailureStub: NSObject { - @objc(synthesizeSwipeWithApplication:x:y:x2:y2:durationMs:) - class func synthesizeSwipe( - application: XCUIApplication, - x: Double, - y: Double, - x2: Double, - y2: Double, - durationMs: Double - ) -> String? { - "forced private synthesis failure" - } -} - -private final class RunnerSynthesizedTapFailureStub: NSObject { - @objc(synthesizeTapWithApplication:x:y:) - class func synthesizeTap(application: XCUIApplication, x: Double, y: Double) -> String? { - "forced private synthesis failure" - } -} -#endif - -extension RunnerTests { - // MARK: - Main Thread Dispatch - - private func currentUptimeMs() -> Double { - ProcessInfo.processInfo.systemUptime * 1000 - } - - private func measureGesture(_ action: () -> Void) -> (gestureStartUptimeMs: Double, gestureEndUptimeMs: Double) { - let gestureStartUptimeMs = currentUptimeMs() - action() - return (gestureStartUptimeMs, currentUptimeMs()) - } - - func synthesizedSwipeFallbackHoldDuration(durationMs: Double) -> TimeInterval { - min(max((durationMs / 5.0) / 1000.0, 0.016), 0.120) - } - - func coordinateDragHoldDuration() -> TimeInterval { - 0.050 - } - - func unsupportedResponse(for outcome: RunnerInteractionOutcome) -> Response? { - switch outcome { - case .performed: - return nil - case .unsupported(let message, let hint): - return Response( - ok: false, - error: ErrorPayload(code: "UNSUPPORTED_OPERATION", message: message, hint: hint) - ) - } - } - - /// Optional visualization frame returned with a gesture response. - enum GestureFrame { - case none - case touch(TouchVisualizationFrame?) - case drag(DragVisualizationFrame) - } - - struct GestureFallback { - let strategy: String - let message: String - let hint: String? - } - - private func gestureFallback(strategy: String, from outcome: RunnerInteractionOutcome) -> GestureFallback? { - switch outcome { - case .performed: - return nil - case .unsupported(let message, let hint): - return GestureFallback(strategy: strategy, message: message, hint: hint) - } - } - - - /// Runs a gesture action with uniform timing capture. Touch gestures pass `idleTimeout: true` - /// (the default) to run inside the scroll idle-timeout + quiescence-skip wrapper; synthesis - /// pointer-plan gestures pass `false` because RunnerSynthesizedGesture governs their - /// own timing. Returns the captured timing and the action's outcome. - /// - /// NOTE: a new SYNTHESIS gesture must pass `idleTimeout: false` — the default `true` would wrap - /// it in the scroll idle-timeout/quiescence-skip path and change its runtime behavior. - func performGesture( - _ app: XCUIApplication, - idleTimeout: Bool = true, - _ action: () -> RunnerInteractionOutcome - ) -> (timing: (gestureStartUptimeMs: Double, gestureEndUptimeMs: Double), outcome: RunnerInteractionOutcome) { - var outcome = RunnerInteractionOutcome.performed - let timing = measureGesture { - if idleTimeout { - withBoundedInteractionIdleTimeoutIfSupported(app, waits: .bothSkipped) { - outcome = action() - } - } else { - outcome = action() - } - } - return (timing, outcome) - } - - /// Single factory for the success payload every gesture returns (message + gesture timing + - /// an optional touch/drag visualization frame), so the field shape lives in one place. - private func gestureResponse( - message: String, - timing: (gestureStartUptimeMs: Double, gestureEndUptimeMs: Double), - frame: GestureFrame = .none, - fallback: GestureFallback? = nil, - maestroNonHittableCoordinateFallbackUsed: Bool? = nil - ) -> Response { - let data: DataPayload - switch frame { - case .none: - data = DataPayload( - message: message, - gestureStartUptimeMs: timing.gestureStartUptimeMs, - gestureEndUptimeMs: timing.gestureEndUptimeMs, - gestureFallback: fallback?.strategy, - gestureFallbackMessage: fallback?.message, - gestureFallbackHint: fallback?.hint, - maestroNonHittableCoordinateFallbackUsed: maestroNonHittableCoordinateFallbackUsed - ) - case .touch(let f): - data = DataPayload( - message: message, - gestureStartUptimeMs: timing.gestureStartUptimeMs, - gestureEndUptimeMs: timing.gestureEndUptimeMs, - x: f?.x, - y: f?.y, - referenceWidth: f?.referenceWidth, - referenceHeight: f?.referenceHeight, - gestureFallback: fallback?.strategy, - gestureFallbackMessage: fallback?.message, - gestureFallbackHint: fallback?.hint, - maestroNonHittableCoordinateFallbackUsed: maestroNonHittableCoordinateFallbackUsed - ) - case .drag(let f): - data = DataPayload( - message: message, - gestureStartUptimeMs: timing.gestureStartUptimeMs, - gestureEndUptimeMs: timing.gestureEndUptimeMs, - x: f.x, - y: f.y, - x2: f.x2, - y2: f.y2, - referenceWidth: f.referenceWidth, - referenceHeight: f.referenceHeight, - gestureFallback: fallback?.strategy, - gestureFallbackMessage: fallback?.message, - gestureFallbackHint: fallback?.hint - ) - } - return Response(ok: true, data: data) - } - - /// Gesture plans already return canonical centroid endpoints from the portable runtime. - /// Keep runner timing/fallback diagnostics, but do not leak the coordinate-drag adapter's - /// visualization frame into only the fast-fling response shape. - private func canonicalPlannedGestureResponse(_ response: Response) -> Response { - guard response.ok, let data = response.data else { return response } - return Response( - ok: true, - data: DataPayload( - message: data.message, - gestureStartUptimeMs: data.gestureStartUptimeMs, - gestureEndUptimeMs: data.gestureEndUptimeMs, - gestureFallback: data.gestureFallback, - gestureFallbackMessage: data.gestureFallbackMessage, - gestureFallbackHint: data.gestureFallbackHint - ) - ) - } - - private func plannedGestureResponse( - plan: RunnerGesturePlan, - timing: (gestureStartUptimeMs: Double, gestureEndUptimeMs: Double), - outcome: RunnerInteractionOutcome - ) -> Response { - if let response = unsupportedResponse(for: outcome) { - return response - } - return gestureResponse(message: plan.intent, timing: timing) - } - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS - func testGestureResponseIncludesSynthesizedTapFallbackDiagnostics() { - let response = gestureResponse( - message: "tapped", - timing: (gestureStartUptimeMs: 1, gestureEndUptimeMs: 2), - fallback: GestureFallback( - strategy: "xctest-coordinate-tap", - message: "Runner synthesized coordinate tap is unavailable", - hint: "Using XCTest coordinate tap fallback." - ) - ) - - XCTAssertEqual(response.ok, true) - XCTAssertEqual(response.data?.gestureFallback, "xctest-coordinate-tap") - XCTAssertEqual( - response.data?.gestureFallbackMessage, - "Runner synthesized coordinate tap is unavailable" - ) - XCTAssertEqual(response.data?.gestureFallbackHint, "Using XCTest coordinate tap fallback.") - } - - func testGestureResponseIncludesMaestroNonHittableFallbackUsage() { - let response = gestureResponse( - message: "tapped via non-hittable coordinate fallback", - timing: (gestureStartUptimeMs: 1, gestureEndUptimeMs: 2), - frame: .touch(nil), - maestroNonHittableCoordinateFallbackUsed: true - ) - - XCTAssertEqual(response.data?.maestroNonHittableCoordinateFallbackUsed, true) - } - - func testCanonicalPlannedGestureResponseOmitsDragFrameAndPreservesDiagnostics() { - let response = gestureResponse( - message: "fling", - timing: (gestureStartUptimeMs: 1, gestureEndUptimeMs: 2), - frame: .drag( - DragVisualizationFrame( - x: 160, - y: 150, - x2: 40, - y2: 150, - referenceWidth: 200, - referenceHeight: 300 - ) - ), - fallback: GestureFallback( - strategy: "xctest-coordinate-drag", - message: "Private synthesis unavailable", - hint: "Using XCTest coordinate fallback." - ) - ) - - let canonical = canonicalPlannedGestureResponse(response) - - XCTAssertEqual(canonical.data?.gestureStartUptimeMs, 1) - XCTAssertEqual(canonical.data?.gestureEndUptimeMs, 2) - XCTAssertEqual(canonical.data?.gestureFallback, "xctest-coordinate-drag") - XCTAssertEqual(canonical.data?.gestureFallbackMessage, "Private synthesis unavailable") - XCTAssertEqual(canonical.data?.gestureFallbackHint, "Using XCTest coordinate fallback.") - XCTAssertNil(canonical.data?.x) - XCTAssertNil(canonical.data?.y) - XCTAssertNil(canonical.data?.x2) - XCTAssertNil(canonical.data?.y2) - XCTAssertNil(canonical.data?.referenceWidth) - XCTAssertNil(canonical.data?.referenceHeight) - } - -#if os(iOS) - func testSinglePointerFlingFallsBackToXCTestCoordinateDragWhenPrivateSynthesisFails() throws { - let selector = NSSelectorFromString( - "synthesizeSwipeWithApplication:x:y:x2:y2:durationMs:" - ) - guard - let synthesizedSwipeMethod = class_getClassMethod(RunnerSynthesizedGesture.self, selector), - let failureStubMethod = class_getClassMethod(RunnerSynthesizedSwipeFailureStub.self, selector) - else { - XCTFail("unable to install synthesized swipe failure stub") - return - } - let originalImplementation = method_getImplementation(synthesizedSwipeMethod) - method_setImplementation( - synthesizedSwipeMethod, - method_getImplementation(failureStubMethod) - ) - app.launch() - runnerAccessibilityHealth = .healthy - defer { - method_setImplementation(synthesizedSwipeMethod, originalImplementation) - invalidateCachedTarget(reason: "unit_test_cleanup") - app.terminate() - } - let command = try runnerCommandFixture( - """ - {"command":"gesture","commandId":"gesture-fling-fallback","gesturePlan":{"topology":"single","intent":"fling","executionProfile":"endpoint-hold","durationMs":100,"viewport":{"x":0,"y":0,"width":200,"height":300},"pointers":[{"pointerId":0,"samples":[{"offsetMs":0,"point":{"x":160,"y":150}},{"offsetMs":100,"point":{"x":40,"y":150}}]}]}} - """ - ) - - let response = try executeOnMainPrepared(command: command, activeApp: app) - - XCTAssertTrue(response.ok) - XCTAssertEqual(response.data?.message, "fling") - XCTAssertEqual(response.data?.gestureFallback, "xctest-coordinate-drag") - XCTAssertEqual(response.data?.gestureFallbackMessage, "forced private synthesis failure") - XCTAssertEqual( - response.data?.gestureFallbackHint, - "Private XCTest event synthesis is required for AX-free coordinate drag on iOS; update Xcode if this persists." - ) - XCTAssertNil(response.data?.x) - XCTAssertNil(response.data?.y) - XCTAssertNil(response.data?.x2) - XCTAssertNil(response.data?.y2) - } - - func testSelectorTapFallsBackToXCTestCoordinateWhenPrivateSynthesisFails() throws { - let selector = NSSelectorFromString("synthesizeTapWithApplication:x:y:") - guard - let synthesizedTapMethod = class_getClassMethod(RunnerSynthesizedGesture.self, selector), - let failureStubMethod = class_getClassMethod(RunnerSynthesizedTapFailureStub.self, selector) - else { - XCTFail("unable to install synthesized tap failure stub") - return - } - let originalImplementation = method_getImplementation(synthesizedTapMethod) - method_setImplementation( - synthesizedTapMethod, - method_getImplementation(failureStubMethod) - ) - app.launch() - currentApp = app - runnerAccessibilityHealth = .healthy - defer { - method_setImplementation(synthesizedTapMethod, originalImplementation) - invalidateCachedTarget(reason: "unit_test_cleanup") - app.terminate() - } - let command = try runnerCommandFixture( - #"{"command":"tap","commandId":"selector-tap-fallback","selectorKey":"label","selectorValue":"Agent Device Runner","synthesized":true}"# - ) - - let response = try executeOnMainPrepared(command: command, activeApp: app) - - XCTAssertTrue(response.ok) - XCTAssertEqual(response.data?.message, "tapped") - XCTAssertEqual(response.data?.gestureFallback, "xctest-coordinate-tap") - XCTAssertEqual(response.data?.gestureFallbackMessage, "forced private synthesis failure") - XCTAssertEqual( - response.data?.gestureFallbackHint, - "Falling back to XCTest coordinate tap may be slower and can still need a healthy accessibility tree." - ) - } -#endif - -#if os(iOS) - func testTypeWithoutResolvedInputReturnsTypedFailureBeforeDispatchingText() throws { - let command = try runnerCommandFixture( - #"{"command":"type","commandId":"type-without-focus","text":"hello"}"# - ) - - let response = executeTypeCommand( - activeApp: XCUIApplication(bundleIdentifier: "com.example.agentdevice.missing-input"), - command: command - ) - - XCTAssertFalse(response.ok) - XCTAssertEqual(response.error?.code, "TEXT_INPUT_NOT_FOCUSED") - XCTAssertEqual( - response.error?.hint, - "Focus a visible text input, then retry type or fill. If the input is not exposed by accessibility, use a coordinate focus command before typing." - ) - } - - func testBareTypeUsesTappedInputWhenSoftwareKeyboardIsHidden() throws { - // The fixture uses a real text responder with an empty input view to model hardware-keyboard input. - app.launchArguments = ["--agent-device-text-entry-regression"] - app.launch() - defer { - invalidateCachedTarget(reason: "unit_test_cleanup") - app.terminate() - } - XCTAssertTrue(app.waitForExistence(timeout: appExistenceTimeout)) - - let textField = app.textFields["agent-device-hardware-keyboard-input"] - XCTAssertTrue(textField.waitForExistence(timeout: appExistenceTimeout)) - let frame = textField.frame - XCTAssertFalse(frame.isEmpty) - - let tapCommand = try runnerCommandFixture( - #"{"command":"tap","commandId":"tap-hardware-keyboard-input","selectorKey":"id","selectorValue":"agent-device-hardware-keyboard-input"}"# - ) - let tapResponse = try executeOnMainPrepared(command: tapCommand, activeApp: app) - XCTAssertTrue(tapResponse.ok, String(describing: tapResponse.error)) - // A precondition, not a product claim. The fixture's empty `inputView` is what keeps the - // keyboard down, but nothing in this bundle owns the simulator's own keyboard settings, so an - // ambient flip that raised one here would be an environment fact — and reporting it as a - // failed assertion is what made this read as a product regression on unrelated PRs (#1874). - try XCTSkipIf( - isKeyboardVisible(app: app), - "software keyboard is up: this simulator cannot exercise the hidden-keyboard responder path" - ) - - let failureCountBefore = currentXCTestFailureCount() - let typeCommand = try runnerCommandFixture( - #"{"command":"type","commandId":"type-hardware-keyboard","text":"hardware-keyboard"}"# - ) - let typeResponse = executeTypeCommand(activeApp: app, command: typeCommand) - - XCTAssertTrue(typeResponse.ok, String(describing: typeResponse.error)) - XCTAssertFalse(didRecordXCTestFailure(since: failureCountBefore)) - XCTAssertEqual(typeResponse.data?.textEntryRoute, "synthesized-first-responder") - XCTAssertEqual(String(describing: textField.value ?? ""), "hardware-keyboard") - - let secondFailureCountBefore = currentXCTestFailureCount() - let secondTypeCommand = try runnerCommandFixture( - #"{"command":"type","commandId":"type-hardware-keyboard-again","text":"-again"}"# - ) - let secondTypeResponse = executeTypeCommand(activeApp: app, command: secondTypeCommand) - - XCTAssertFalse(secondTypeResponse.ok) - XCTAssertEqual(secondTypeResponse.error?.code, "TEXT_INPUT_NOT_FOCUSED") - XCTAssertFalse(didRecordXCTestFailure(since: secondFailureCountBefore)) - XCTAssertEqual(String(describing: textField.value ?? ""), "hardware-keyboard") - } - - // `waitForTextEntryReadiness`'s hardware-keyboard fallback returns early only on confirmed - // focus (#1874), and `keyboardFocusConfirmed` reads that from the app-wide focus predicate this - // bundle otherwise refuses to trust. Two XCTest facts it rests on, neither a repository - // invariant: the predicate reports a responder that shows NO software keyboard at all, and it - // names the element well enough to tell the tapped field from another one. The fixture field is - // the exact shape the fallback exists for — a real responder with an empty `inputView` — so this - // is where both are observable. If either regressed, readiness would silently stop taking the - // fallback and spend the full readinessTimeout on every hardware-keyboard field, which no other - // assertion would notice. - func testHardwareKeyboardResponderConfirmsItsOwnKeyboardFocus() throws { - app.launchArguments = ["--agent-device-text-entry-regression"] - app.launch() - defer { - invalidateCachedTarget(reason: "unit_test_cleanup") - app.terminate() - } - XCTAssertTrue(app.waitForExistence(timeout: appExistenceTimeout)) - - let textField = app.textFields["agent-device-hardware-keyboard-input"] - XCTAssertTrue(textField.waitForExistence(timeout: appExistenceTimeout)) - let otherElement = app.staticTexts["Agent Device Runner"] - XCTAssertTrue(otherElement.waitForExistence(timeout: appExistenceTimeout)) - XCTAssertFalse( - keyboardFocusConfirmed(app: app, element: textField), - "an untapped field must not confirm focus, or the fallback would fire immediately" - ) - - let tapCommand = try runnerCommandFixture( - #"{"command":"tap","commandId":"tap-focus-confirmation","selectorKey":"id","selectorValue":"agent-device-hardware-keyboard-input"}"# - ) - let tapResponse = try executeOnMainPrepared(command: tapCommand, activeApp: app) - XCTAssertTrue(tapResponse.ok, String(describing: tapResponse.error)) - try XCTSkipIf( - isKeyboardVisible(app: app), - "software keyboard is up: this simulator cannot exercise the hidden-keyboard responder path" - ) - - let deadline = Date().addingTimeInterval(TextEntryTiming.readinessTimeout) - var confirmed = keyboardFocusConfirmed(app: app, element: textField) - while !confirmed && Date() < deadline { - sleepFor(TextEntryTiming.pollInterval) - confirmed = keyboardFocusConfirmed(app: app, element: textField) - } - XCTAssertTrue(confirmed, "a tapped responder must confirm its own keyboard focus") - XCTAssertFalse( - keyboardFocusConfirmed(app: app, element: otherElement), - "focus held by another element must read as a refusal, never as this element's focus" - ) - } - - func testBareDelayedTypeFailsWhenTappedInputDisappearsMidCommand() throws { - app.launchArguments = [ - "--agent-device-text-entry-regression", - "--agent-device-text-entry-disappear-after-input", - ] - app.launch() - defer { - invalidateCachedTarget(reason: "unit_test_cleanup") - app.terminate() - } - XCTAssertTrue(app.waitForExistence(timeout: appExistenceTimeout)) - - let textField = app.textFields["agent-device-hardware-keyboard-input"] - XCTAssertTrue(textField.waitForExistence(timeout: appExistenceTimeout)) - let tapCommand = try runnerCommandFixture( - #"{"command":"tap","commandId":"tap-disappearing-input","selectorKey":"id","selectorValue":"agent-device-hardware-keyboard-input"}"# - ) - let tapResponse = try executeOnMainPrepared(command: tapCommand, activeApp: app) - XCTAssertTrue(tapResponse.ok, String(describing: tapResponse.error)) - - let failureCountBefore = currentXCTestFailureCount() - let typeCommand = try runnerCommandFixture( - #"{"command":"type","commandId":"type-disappearing-input","text":"ab","delayMs":50}"# - ) - let typeResponse = executeTypeCommand(activeApp: app, command: typeCommand) - - XCTAssertFalse(didRecordXCTestFailure(since: failureCountBefore)) - XCTAssertFalse(typeResponse.ok) - XCTAssertEqual(typeResponse.error?.code, "TEXT_INPUT_NOT_FOCUSED") - XCTAssertFalse(textField.exists) - } -#endif - - func testInjectedTapRecordedFailureGateIsTapOnlyAndCountGated() { - // The seam's recording side cannot run in-bundle (a real XCTIssue would - // fail this very test run — same constraint the record(_:) suppression - // tests document); the live daemon proof covers it. This pins the gate. - XCTAssertFalse(RunnerTests.shouldInjectTapRecordedFailure(command: .tap, remaining: 0)) - XCTAssertTrue(RunnerTests.shouldInjectTapRecordedFailure(command: .tap, remaining: 1)) - XCTAssertFalse(RunnerTests.shouldInjectTapRecordedFailure(command: .type, remaining: 1)) - XCTAssertFalse(RunnerTests.shouldInjectTapRecordedFailure(command: .snapshot, remaining: 1)) - } - - func testXCTestRecordedFailureResponseFailsMutatingSuccesses() throws { - let command = try runnerCommandFixture(#"{"command":"tap","commandId":"tap-1"}"#) - let response = Response(ok: true, data: DataPayload(message: "tapped")) - - let failureResponse = xctestRecordedFailureResponse(command: command, response: response) - - XCTAssertEqual(failureResponse?.ok, false) - XCTAssertEqual(failureResponse?.error?.code, "XCTEST_RECORDED_FAILURE") - XCTAssertEqual( - failureResponse?.error?.message, - "XCTest recorded a failure while executing tap; the action may not have been performed." - ) - } - - func testXCTestRecordedFailureResponseFailsActionButtonSuccess() throws { - // The Action Button press carries no settle and no post-action observation, so this conversion is - // the only evidence the press landed. That is why the press is not classified runner-lifecycle: - // `isLifecycle` would silence the conversion here (#2699, #2702 review). - let command = try runnerCommandFixture(#"{"command":"actionButton","commandId":"action-button-1"}"#) - let response = Response(ok: true, data: DataPayload(message: "actionButton")) - - let failureResponse = xctestRecordedFailureResponse(command: command, response: response) - - XCTAssertEqual(failureResponse?.ok, false) - XCTAssertEqual(failureResponse?.error?.code, "XCTEST_RECORDED_FAILURE") - XCTAssertEqual( - failureResponse?.error?.message, - "XCTest recorded a failure while executing actionButton; the action may not have been performed." - ) - } - - func testXCTestRecordedFailureResponseDoesNotWrapReadOnlyOrRunnerFatalResponses() throws { - let snapshotCommand = try runnerCommandFixture(#"{"command":"snapshot","commandId":"snapshot-1"}"#) - let tapCommand = try runnerCommandFixture(#"{"command":"tap","commandId":"tap-1"}"#) - let runnerFatalResponse = Response( - ok: true, - data: DataPayload(runnerFatal: true, runnerFatalReason: "ax_snapshot_unavailable") - ) - - XCTAssertNil( - xctestRecordedFailureResponse( - command: snapshotCommand, - response: Response(ok: true, data: DataPayload(nodes: [], truncated: false)) - ) - ) - XCTAssertNil(xctestRecordedFailureResponse(command: tapCommand, response: runnerFatalResponse)) - } - - // Simulator-only from here to the matching #endif: these launch the host app, route through - // SpringBoard, or assert the iOS-only alert/system-modal branches. Tests outside the - // `os(iOS)` regions in this file are pure runner decisions and also run on the macOS host - // lane (ci.yml) — see the classification convention in RunnerTests.swift. -#if os(iOS) - func testMissingBundleCommandInvalidatesCompleteCachedTargetState() throws { - app.launch() - currentApp = app - currentBundleId = "com.example.stale-target" - currentAppProcessIdentifier = 42 - snapshotXCTestPenaltyWarmupExemptionPending = true - defer { - invalidateCachedTarget(reason: "unit_test_cleanup") - app.terminate() - } - let command = try runnerCommandFixture( - #"{"command":"snapshot","commandId":"snapshot-without-bundle"}"# - ) - - _ = prepareActiveCommandContext(command: command) - - XCTAssertNil(currentApp) - XCTAssertNil(currentBundleId) - XCTAssertNil(currentAppProcessIdentifier) - XCTAssertFalse(snapshotXCTestPenaltyWarmupExemptionPending) - } - - func testSkipAppActivationPreflightIncludesForegroundCachedCoordinateOnlyTaps() throws { - app.launch() - currentApp = app - currentBundleId = nil - defer { - currentApp = nil - currentBundleId = nil - app.terminate() - } - let tap = try runnerCommandFixture( - #"{"command":"tap","commandId":"tap-1","x":10,"y":20}"# - ) - - XCTAssertTrue(shouldSkipAppActivationPreflight(tap)) - } - - func testSkipAppActivationPreflightRejectsMissingChangedAndBackgroundTargets() throws { - let coordinateTap = try runnerCommandFixture( - #"{"command":"tap","commandId":"tap-1","x":10,"y":20}"# - ) - currentApp = nil - currentBundleId = nil - XCTAssertFalse(shouldSkipAppActivationPreflight(coordinateTap)) - - app.launch() - currentApp = app - currentBundleId = "com.example.current" - defer { - currentApp = nil - currentBundleId = nil - app.terminate() - } - let changedBundleTap = try runnerCommandFixture( - #"{"command":"tap","commandId":"tap-2","appBundleId":"com.example.other","x":10,"y":20}"# - ) - - XCTAssertFalse(shouldSkipAppActivationPreflight(changedBundleTap)) - - app.terminate() - currentApp = app - currentBundleId = nil - - XCTAssertFalse(shouldSkipAppActivationPreflight(coordinateTap)) - } - - func testActionButtonPressSkipsAppActivationPreflightWithoutBeingRunnerLifecycle() throws { - currentApp = nil - currentBundleId = nil - let press = try runnerCommandFixture(#"{"command":"actionButton","commandId":"action-button-1"}"#) - - // The skip is its own decision, reached without the lifecycle flag that would also drop the - // recorded-failure conversion; no cached target and no foreground app is required for it. - XCTAssertFalse(isRunnerLifecycleCommand(.actionButton)) - XCTAssertTrue(shouldSkipAppActivationPreflight(press)) - } - - func testPrepareActiveCommandContextRoutesBlockingSystemModalToSpringboard() throws { - blockingSystemModalPresenceOverrideForTesting = true - currentApp = nil - currentBundleId = nil - defer { - blockingSystemModalPresenceOverrideForTesting = nil - currentApp = nil - currentBundleId = nil - } - let tap = try runnerCommandFixture( - #"{"command":"tap","commandId":"tap-1","x":10,"y":20}"# - ) - - let preparation = prepareActiveCommandContext( - command: tap, - routeToSpringboard: shouldRouteToSpringboardBlockingSystemModal(tap) - ) - - guard case .context(let context) = preparation else { - XCTFail("expected command context") - return - } - XCTAssertTrue(context.app === springboard) - } - - func testExecuteDispatchedReturnsBusyBeforeBlockingSystemModalProbeDrains() throws { - app.launch() - currentApp = app - currentBundleId = nil - defer { - currentApp = nil - currentBundleId = nil - systemModalProbeOverrideForTesting = nil - clearSnapshotXCTestChannelPenalty(reason: "test-cleanup") - app.terminate() - } - - final class ResultBox { - var response: Response? - var error: Error? - var commandRecoveredBeforeRelease = false - var wasBusyBeforeRelease = false - var hadAbandonedProbeBeforeRelease = false - var drained = false - } - let box = ResultBox() - let probeStarted = expectation(description: "system-modal routing probe started") - let verificationFinished = expectation(description: "command recovery and modal probe drain verified") - let probeReleaseGate = DispatchSemaphore(value: 0) - let commandFinishedGate = DispatchSemaphore(value: 0) - systemModalProbeOverrideForTesting = { _ in - probeStarted.fulfill() - _ = probeReleaseGate.wait(timeout: .now() + 15) - return DataPayload(message: "late system modal") - } - - let command = try runnerCommandFixture( - #"{"command":"tap","commandId":"bounded-modal-routing","x":10,"y":20}"# - ) - DispatchQueue(label: "agent-device.runner.tests.modal-routing-probe").async { - do { - box.response = try self.executeDispatched(command: command) - } catch { - box.error = error - } - commandFinishedGate.signal() - } - DispatchQueue(label: "agent-device.runner.tests.modal-routing-probe-verifier").async { - let commandWait = commandFinishedGate.wait( - timeout: .now() + self.systemModalProbeBudget + 3 - ) - box.commandRecoveredBeforeRelease = commandWait == .success - && box.error == nil - && box.response?.error?.code == "RUNNER_BUSY" - if case .busy = self.currentMainThreadBusyState() { - box.wasBusyBeforeRelease = true - } - box.hadAbandonedProbeBeforeRelease = self.hasAbandonedMainThreadWork() - - // The XCTest main thread is blocked inside the injected probe, so this verifier owns the - // ordered release after recording the command result and abandoned-work state above. - probeReleaseGate.signal() - let deadline = Date().addingTimeInterval(5) - while self.hasAbandonedMainThreadWork(), Date() < deadline { - self.sleepFor(0.002) - } - box.drained = !self.hasAbandonedMainThreadWork() - verificationFinished.fulfill() - } - - wait(for: [probeStarted, verificationFinished], timeout: 15) - XCTAssertTrue( - box.commandRecoveredBeforeRelease, - "the public coordinate tap must return RUNNER_BUSY before the blocked modal probe drains" - ) - XCTAssertTrue(box.wasBusyBeforeRelease) - XCTAssertTrue(box.hadAbandonedProbeBeforeRelease) - XCTAssertTrue(box.drained) - guard case .idle = currentMainThreadBusyState() else { - return XCTFail("expected the runner to become idle after the routing probe drained") - } - XCTAssertFalse(hasAbandonedMainThreadWork()) - } - - func testSkipAppActivationPreflightRejectsSelectorAndMixedSequenceGestures() throws { - app.launch() - currentApp = app - currentBundleId = nil - defer { - currentApp = nil - currentBundleId = nil - app.terminate() - } - let selectorTap = try runnerCommandFixture( - #"{"command":"tap","commandId":"tap-1","selectorKey":"label","selectorValue":"Search","synthesized":true}"# - ) - let standardDrag = try runnerCommandFixture( - #"{"command":"drag","commandId":"drag-1","x":10,"y":20,"x2":30,"y2":40}"# - ) - let mixedSequence = try runnerCommandFixture( - """ - {"command":"sequence","commandId":"seq-1","steps":[ - {"kind":"tap","x":10,"y":20,"synthesized":true}, - {"kind":"doubleTap","x":30,"y":40} - ]} - """ - ) - - XCTAssertFalse(shouldSkipAppActivationPreflight(selectorTap)) - XCTAssertFalse(shouldSkipAppActivationPreflight(standardDrag)) - XCTAssertFalse(shouldSkipAppActivationPreflight(mixedSequence)) - } - - // Launches nothing, but still simulator-only: `shouldSkipAppActivationPreflight` is - // `#if os(iOS) …guards… #else return false #endif`, so on macOS this asserts a compile-time - // literal and no edit to the iOS body could make it red. Its five siblings above and below - // are gated for the same reason. - func testSkipAppActivationPreflightRequiresCachedForegroundTarget() throws { - currentApp = nil - currentBundleId = nil - let scroll = try runnerCommandFixture( - #"{"command":"scroll","commandId":"scroll-1","direction":"down","pixels":400}"# - ) - - XCTAssertFalse(shouldSkipAppActivationPreflight(scroll)) - } - - func testSkipAppActivationPreflightKeepsDragScrollAndSequenceOnForegroundGuard() throws { - app.launch() - currentApp = app - currentBundleId = nil - defer { - currentApp = nil - currentBundleId = nil - app.terminate() - } - let drag = try runnerCommandFixture( - #"{"command":"drag","commandId":"drag-1","x":10,"y":20,"x2":30,"y2":40,"synthesized":true}"# - ) - let scroll = try runnerCommandFixture( - #"{"command":"scroll","commandId":"scroll-1","direction":"down","pixels":400}"# - ) - let sequence = try runnerCommandFixture( - """ - {"command":"sequence","commandId":"seq-1","steps":[ - {"kind":"tap","x":10,"y":20,"synthesized":true}, - {"kind":"longPress","x":10,"y":200,"durationMs":300} - ]} - """ - ) - - XCTAssertFalse(shouldSkipAppActivationPreflight(drag)) - XCTAssertFalse(shouldSkipAppActivationPreflight(scroll)) - XCTAssertFalse(shouldSkipAppActivationPreflight(sequence)) - } - - func testSkipAppActivationPreflightIncludesAlertCommands() throws { - let alert = try runnerCommandFixture( - #"{"command":"alert","commandId":"alert-1","action":"get"}"# - ) - - XCTAssertTrue(shouldSkipAppActivationPreflight(alert)) - } -#endif - - func testExecuteDispatchedReturnsBusyBeforeMainThreadFastPath() throws { - let command = try runnerCommandFixture(#"{"command":"snapshot","commandId":"snapshot-busy"}"#) - abandonedMainThreadWorkCount = 1 - abandonedMainThreadWorkSince = Date(timeIntervalSinceNow: -2) - defer { - abandonedMainThreadWorkCount = 0 - abandonedMainThreadWorkSince = nil - } - - let response = try executeDispatched(command: command) - - XCTAssertFalse(response.ok) - XCTAssertEqual(response.error?.code, "RUNNER_BUSY") - XCTAssertTrue(response.error?.message.contains("previous command") == true) - } - - func testExecuteDispatchedReturnsWedgedBeforeMainThreadFastPath() throws { - let command = try runnerCommandFixture(#"{"command":"snapshot","commandId":"snapshot-wedged"}"#) - abandonedMainThreadWorkCount = 1 - abandonedMainThreadWorkSince = Date(timeIntervalSinceNow: -(mainThreadWedgeThreshold + 1)) - defer { - abandonedMainThreadWorkCount = 0 - abandonedMainThreadWorkSince = nil - } - - let response = try executeDispatched(command: command) - - XCTAssertFalse(response.ok) - XCTAssertEqual(response.error?.code, "RUNNER_WEDGED") - XCTAssertTrue(response.error?.hint?.contains("runner session will be restarted") == true) - } - - func testPostSnapshotDelayMarkDoesNotQueueBehindAbandonedMainThreadWork() { - abandonedMainThreadWorkCount = 1 - defer { - abandonedMainThreadWorkCount = 0 - needsPostSnapshotInteractionDelay = false - } - - let finished = expectation(description: "off-main caller finished") - DispatchQueue(label: "agent-device.runner.tests.post-snapshot-delay").async { - self.setNeedsPostSnapshotInteractionDelay() - finished.fulfill() - } - - wait(for: [finished], timeout: 1) - mainThreadWorkLock.lock() - let abandonedWorkCount = abandonedMainThreadWorkCount - mainThreadWorkLock.unlock() - XCTAssertEqual(abandonedWorkCount, 1, "the skipped mark must not add an abandoned unit") - XCTAssertFalse(needsPostSnapshotInteractionDelay) - } - - func testSnapshotFailureInvalidationQueuesBehindAbandonedMainThreadWorkWithoutWaiting() { - currentBundleId = "com.example.stale-target" - defer { currentBundleId = nil } - - final class ResultBox { - var elapsed: TimeInterval? - var bundleStillCachedWhileBlocked: Bool? - var abandonedWhileBlocked: Int? - } - let box = ResultBox() - let mainBlocked = DispatchSemaphore(value: 0) - let releaseMain = DispatchSemaphore(value: 0) - let finished = expectation(description: "invalidation returned while main was blocked") - - DispatchQueue(label: "agent-device.runner.tests.snapshot-invalidation").async { - _ = try? self.runMainThreadWork( - "command_execution", - timeout: 0, - timeoutError: self.mainThreadExecutionTimeoutError - ) { - mainBlocked.signal() - _ = releaseMain.wait(timeout: .now() + 5) - return true - } - _ = mainBlocked.wait(timeout: .now() + 2) - let startedAt = Date() - self.invalidateCachedTargetAfterSnapshotFailure() - box.elapsed = Date().timeIntervalSince(startedAt) - box.bundleStillCachedWhileBlocked = self.currentBundleId != nil - self.mainThreadWorkLock.lock() - box.abandonedWhileBlocked = self.abandonedMainThreadWorkCount - self.mainThreadWorkLock.unlock() - releaseMain.signal() - finished.fulfill() - } - - wait(for: [finished], timeout: 8) - let drainDeadline = Date().addingTimeInterval(2) - while hasAbandonedMainThreadWork() || currentBundleId != nil, Date() < drainDeadline { - sleepFor(0.005) - } - - XCTAssertLessThan( - box.elapsed ?? .infinity, - 0.5, - "the failed capture must not wait behind abandoned main-thread work" - ) - XCTAssertEqual( - box.bundleStillCachedWhileBlocked, - true, - "the drop must queue behind the blocked main thread, not run early" - ) - XCTAssertEqual(box.abandonedWhileBlocked, 1, "the deferred drop must not add an abandoned unit") - XCTAssertFalse(hasAbandonedMainThreadWork()) - XCTAssertNil(currentBundleId, "the drop must run once the main thread frees") - } -#endif - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS - func execute(command: Command) throws -> Response { - if command.command == .status { - return executeStatus(command: command) - } - if command.command == .uptime { - return executeUptime() - } - commandJournal.accept(command: command) - return try executeAccepted(command: command) - } -#endif - - func executeAccepted(command: Command) throws -> Response { - commandJournal.start(command: command) - pendingTargetActivation = nil - do { - let response = try executeDispatched(command: command) - commandJournal.finish(command: command, response: response) - guard let fact = pendingTargetActivation else { return response } - // Stamped after `finish`, like the uptime anchor: a journal-replayed result carries no - // activation fact, because the command that paid for it is the one being replayed, not one - // that just repaired foreground (#2682). - pendingTargetActivation = nil - return response.stampingTargetActivation(fact) - } catch { - pendingTargetActivation = nil - commandJournal.fail(command: command, error: error) - throw error - } - } - - func executeStatus(command: Command) -> Response { - guard - let statusCommandId = command.statusCommandId?.trimmedNonEmpty - else { - return Response( - ok: false, - error: ErrorPayload( - code: "INVALID_ARGS", - message: "status requires statusCommandId", - hint: "Set statusCommandId to the commandId of the runner command to inspect." - ) - ) - } - return Response(ok: true, data: commandJournal.status(normalizedCommandId: statusCommandId)) - } - - func executeUptime() -> Response { - // Placeholder value: the transport layer (jsonResponse) overwrites currentUptimeMs with a - // fresher send-time stamp on every ok response; kept so direct callers still get a value. - Response( - ok: true, - data: DataPayload(currentUptimeMs: currentUptimeMs()) - ) - } - - struct ActiveCommandContext { - let app: XCUIApplication - /// Set when `app` is a system surface served in place over the still-bound session app (#2438). - var systemSurface: SystemSurfaceHost? = nil - } - - enum ActiveCommandPreparation { - case response(Response) - case context(ActiveCommandContext) - } - - private func runnerBusyResponse(command: Command, abandonedForSeconds: TimeInterval) -> Response { - NSLog( - "AGENT_DEVICE_RUNNER_BUSY command=%@ commandId=%@ abandonedForSeconds=%.1f", - command.command.rawValue, - command.commandId ?? "", - abandonedForSeconds - ) - return Response( - ok: false, - error: ErrorPayload( - code: "RUNNER_BUSY", - message: - "The iOS runner is still finishing a previous command that exceeded its execution watchdog (usually an accessibility capture on a heavy or animating screen).", - hint: - "Wait a few seconds and retry. If snapshots keep failing on this screen, use screenshot as visual truth and interact by coordinates, or navigate to another screen." - ) - ) - } - - private func runnerWedgedResponse(command: Command, abandonedForSeconds: TimeInterval) -> Response { - NSLog( - "AGENT_DEVICE_RUNNER_WEDGED command=%@ commandId=%@ abandonedForSeconds=%.1f", - command.command.rawValue, - command.commandId ?? "", - abandonedForSeconds - ) - return Response( - ok: false, - error: ErrorPayload( - code: "RUNNER_WEDGED", - message: - "The iOS runner main thread has been stuck in abandoned work for \(Int(abandonedForSeconds)) seconds and cannot recover on its own.", - hint: - "The runner session will be restarted. Retry the command after the restart; if this screen keeps wedging captures, use screenshot as visual truth and interact by coordinates." - ) - ) - } - - private func runnerUnavailableResponse(command: Command) -> Response? { - switch currentMainThreadBusyState() { - case .idle: - return nil - case .busy(let abandonedForSeconds): - return runnerBusyResponse(command: command, abandonedForSeconds: abandonedForSeconds) - case .wedged(let abandonedForSeconds): - return runnerWedgedResponse(command: command, abandonedForSeconds: abandonedForSeconds) - } - } - - private func executeDispatched(command: Command) throws -> Response { - // XCTest work cannot be cancelled mid-flight: once the watchdog abandons a main-queue - // block, queueing more main-thread commands behind it only buries the runner deeper. - // Refuse fast instead so the daemon backs off while the abandoned work drains; past the - // wedge threshold, escalate so the daemon recycles this runner (#1105). - if let unavailable = runnerUnavailableResponse(command: command) { - return unavailable - } - let alertDeadline = command.command == .alert - ? Date().addingTimeInterval(Self.alertCommandTimeout(timeoutMs: command.timeoutMs)) - : nil - if Thread.isMainThread { - let routeToSpringboard = shouldRouteToSpringboardBlockingSystemModal(command) - return try executeOnMainSafely( - command: command, - alertDeadline: alertDeadline, - routeToSpringboard: routeToSpringboard - ) - } - // Resolve this before the command's outer main-thread block. If the bounded probe abandons - // slow XCTest enumeration, return the established recoverable response instead of queueing - // command preparation behind work that may outlive the 30-second command watchdog. - let routeToSpringboard = shouldRouteToSpringboardBlockingSystemModal(command) - if let unavailable = runnerUnavailableResponse(command: command) { - return unavailable - } - if command.command == .snapshot { - return try executeSnapshotDispatched(command: command) - } - if command.command == .alert, let deadline = alertDeadline { - return try runMainThreadWork( - "command_execution", - timeout: max(0.001, deadline.timeIntervalSinceNow), - timeoutError: mainThreadExecutionTimeoutError - ) { - try self.executeOnMainSafely( - command: command, - alertDeadline: deadline, - routeToSpringboard: routeToSpringboard - ) - } - } - return try runMainThreadWork( - "command_execution", - timeout: mainThreadExecutionTimeout, - timeoutError: mainThreadExecutionTimeoutError - ) { - try self.executeOnMainSafely(command: command, routeToSpringboard: routeToSpringboard) - } - } - - // MARK: - Command Handling - - private func executeOnMainSafely( - command: Command, - alertDeadline: Date? = nil, - routeToSpringboard: Bool - ) throws -> Response { - var hasRetried = false - while true { - var response: Response? - var swiftError: Error? - let failureCountBefore = currentXCTestFailureCount() - let exceptionMessage = RunnerObjCExceptionCatcher.catchException({ - do { - response = try self.executeOnMain( - command: command, - alertDeadline: alertDeadline, - routeToSpringboard: routeToSpringboard - ) - } catch { - swiftError = error - } - }) - - if let exceptionMessage { - invalidateCachedTarget(reason: "objc_exception") - if !hasRetried, shouldRetryException(command, message: exceptionMessage) { - NSLog( - "AGENT_DEVICE_RUNNER_RETRY command=%@ reason=objc_exception", - command.command.rawValue - ) - hasRetried = true - sleepFor(retryCooldown) - continue - } - throw NSError( - domain: RunnerErrorDomain.exception, - code: RunnerErrorCode.objcException, - userInfo: [NSLocalizedDescriptionKey: exceptionMessage] - ) - } - if let swiftError { - throw swiftError - } - guard let response else { - throw NSError( - domain: RunnerErrorDomain.general, - code: RunnerErrorCode.commandReturnedNoResponse, - userInfo: [NSLocalizedDescriptionKey: "command returned no response"] - ) - } -#if AGENT_DEVICE_RUNNER_UNIT_TESTS - // #1605 merge gate: the REAL gesture already executed above; recording a - // production-shaped issue here makes the per-command failure-count - // conversion below fire exactly as in the field (bsky-24: activation - // lands, bookkeeping records a failure). Compiled out of production. - if consumeInjectedTapRecordedFailureForTesting(command: command.command) { - record( - XCTIssue( - type: .assertionFailure, - compactDescription: "Injected tap recorded-failure (#1605 corroboration merge gate)" - ) - ) - } -#endif - if didRecordXCTestFailure(since: failureCountBefore), - let failureResponse = xctestRecordedFailureResponse(command: command, response: response) - { - invalidateCachedTarget(reason: "xctest_recorded_failure") - return failureResponse - } - if !hasRetried, shouldRetryCommand(command), shouldRetryResponse(response) { - NSLog( - "AGENT_DEVICE_RUNNER_RETRY command=%@ reason=response_unavailable", - command.command.rawValue - ) - hasRetried = true - invalidateCachedTarget(reason: "response_unavailable") - sleepFor(retryCooldown) - continue - } - return response - } - } - - private func executeSnapshotDispatched(command: Command) throws -> Response { - try executeDispatchedWithRecovery(command: command) { - try self.executeSnapshotDispatchedOnce(command: command) - } - } - - /// The dispatched snapshot recovery loop: read-only retry + XCTest-recorded-failure invalidation, - /// matching what `executeOnMainSafely` gives the generic path. `perform` runs the capture and its - /// own bounded main-thread work. - func executeDispatchedWithRecovery( - command: Command, - perform: () throws -> Response - ) throws -> Response { - var hasRetried = false - while true { - let failureCountBefore = try runMainThreadWork( - "recorded_failure_count", - timeout: mainThreadExecutionTimeout, - timeoutError: mainThreadExecutionTimeoutError - ) { - self.currentXCTestFailureCount() - } - let response = try perform() - // Recovered independently — re-entering main for bookkeeping would queue behind the still- - // abandoned XCTest query and re-stall the command (#1244), so skip it until that work drains. - if hasAbandonedMainThreadWork() { - NSLog( - "AGENT_DEVICE_RUNNER_DISPATCH_RECOVERY_SKIPPED_XCTEST_OCCUPIED command=%@", - command.command.rawValue - ) - return response - } - let recordedFailureResponse = try runMainThreadWork( - "recorded_failure_count", - timeout: mainThreadExecutionTimeout, - timeoutError: mainThreadExecutionTimeoutError - ) { - self.didRecordXCTestFailure(since: failureCountBefore) - ? self.xctestRecordedFailureResponse(command: command, response: response) - : nil - } - if let recordedFailureResponse { - try runMainThreadWork( - "target_invalidation", - timeout: mainThreadExecutionTimeout, - timeoutError: mainThreadExecutionTimeoutError - ) { - self.invalidateCachedTarget(reason: "xctest_recorded_failure") - } - return recordedFailureResponse - } - if !hasRetried, shouldRetryCommand(command), shouldRetryResponse(response) { - NSLog( - "AGENT_DEVICE_RUNNER_RETRY command=%@ reason=response_unavailable", - command.command.rawValue - ) - hasRetried = true - try runMainThreadWork( - "target_invalidation", - timeout: mainThreadExecutionTimeout, - timeoutError: mainThreadExecutionTimeoutError - ) { - self.invalidateCachedTarget(reason: "response_unavailable") - self.sleepFor(self.retryCooldown) - } - continue - } - return response - } - } - - private func executeSnapshotDispatchedOnce(command: Command) throws -> Response { - let preparation = try runMainThreadWork( - "command_preparation", - timeout: mainThreadExecutionTimeout, - timeoutError: mainThreadExecutionTimeoutError - ) { - try self.prepareActiveCommandContextSafely(command: command, routeToSpringboard: false) - } - switch preparation { - case .response(let response): - return response - case .context(let context): - return try executeSnapshotPrepared( - command: command, - activeApp: context.app, - systemSurface: context.systemSurface - ) - } - } - - /// Pure command→options projection, extracted so the runner unit bundle can - /// prove the decoded wire field actually reaches presentation options (#1634 P2). - static func presentationOptions(from command: Command) -> PresentationOptions { - let customActions = command.customActions ?? false - return PresentationOptions( - interactiveOnly: command.interactiveOnly ?? false, - depth: command.depth, - scope: command.scope, - raw: command.raw ?? false, - // Custom actions are only readable through the private AX client, so - // asking for them pins that backend rather than silently returning a - // capture that structurally cannot carry them. An explicit pin wins. - preferredBackend: command.preferredBackend - ?? (customActions ? SnapshotBackendKind.privateAX.rawValue : nil), - customActions: customActions - ) - } - - private func executeSnapshotPrepared( - command: Command, - activeApp: XCUIApplication, - systemSurface: SystemSurfaceHost? = nil - ) throws -> Response { - let options = Self.presentationOptions(from: command) - do { - var payload: DataPayload - if options.raw { - payload = try snapshotRaw(app: activeApp, options: options) - } else { - payload = try snapshotFast(app: activeApp, options: options) - } - if let systemSurface { - payload.systemSurface = SystemSurfaceProvenancePayload( - bundleId: systemSurface.bundleId, - kind: systemSurface.kind.rawValue - ) - } - setNeedsPostSnapshotInteractionDelay() - return Response(ok: true, data: payload) - } catch let failure as SnapshotCaptureFailure { - invalidateCachedTargetAfterSnapshotFailure() - return Response( - ok: false, - error: ErrorPayload( - code: failure.code, - message: failure.message, - hint: failure.hint - ) - ) - } - } - - private func setNeedsPostSnapshotInteractionDelay() { - if Thread.isMainThread { - needsPostSnapshotInteractionDelay = true - return - } - guard !hasAbandonedMainThreadWork() else { - NSLog("AGENT_DEVICE_RUNNER_POST_SNAPSHOT_DELAY_MARK_SKIPPED_XCTEST_OCCUPIED") - return - } - do { - try runMainThreadWork( - "post_snapshot_delay_mark", - timeout: 1, - timeoutError: mainThreadExecutionTimeoutError - ) { - self.needsPostSnapshotInteractionDelay = true - } - } catch { - NSLog("AGENT_DEVICE_RUNNER_POST_SNAPSHOT_DELAY_MARK_FAILED=%@", String(describing: error)) - } - } - - private func invalidateCachedTargetAfterSnapshotFailure() { - if Thread.isMainThread { - invalidateCachedTarget(reason: "ax_snapshot_failure") - return - } - // Abandoned work ahead of this hop cannot be cancelled: queue the drop behind it without - // waiting, so the failed capture answers now and the next command still finds the target gone. - guard !hasAbandonedMainThreadWork() else { - NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_INVALIDATION_DEFERRED_XCTEST_OCCUPIED") - DispatchQueue.main.async { - self.invalidateCachedTarget(reason: "ax_snapshot_failure") - } - return - } - do { - try runMainThreadWork( - "target_invalidation", - timeout: 1, - timeoutError: mainThreadExecutionTimeoutError - ) { - self.invalidateCachedTarget(reason: "ax_snapshot_failure") - } - } catch { - NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_INVALIDATION_FAILED=%@", String(describing: error)) - } - } - - private func prepareActiveCommandContextSafely( - command: Command, - routeToSpringboard: Bool - ) throws -> ActiveCommandPreparation { - var preparation: ActiveCommandPreparation? - let exceptionMessage = RunnerObjCExceptionCatcher.catchException({ - preparation = self.prepareActiveCommandContext( - command: command, - routeToSpringboard: routeToSpringboard - ) - }) - if let exceptionMessage { - throw NSError( - domain: RunnerErrorDomain.exception, - code: RunnerErrorCode.objcException, - userInfo: [NSLocalizedDescriptionKey: exceptionMessage] - ) - } - guard let preparation else { - throw NSError( - domain: RunnerErrorDomain.general, - code: RunnerErrorCode.commandReturnedNoResponse, - userInfo: [NSLocalizedDescriptionKey: "snapshot preflight returned no response"] - ) - } - return preparation - } - - private func executeOnMain( - command: Command, - alertDeadline: Date?, - routeToSpringboard: Bool - ) throws -> Response { - let preparation = prepareActiveCommandContext( - command: command, - routeToSpringboard: routeToSpringboard - ) - let activeApp: XCUIApplication - switch preparation { - case .response(let response): - return response - case .context(let context): - activeApp = context.app - } - - switch command.command { - case .status: - return executeStatus(command: command) - case .targetReset: - return resetTargetAfterExternalRelaunch() - case .shutdown: - stopRecordingIfNeeded() - return Response(ok: true, data: DataPayload(message: "shutdown")) - case .recordStart: - guard - let requestedOutPath = command.outPath?.trimmingCharacters(in: .whitespacesAndNewlines), - !requestedOutPath.isEmpty - else { - return Response(ok: false, error: ErrorPayload(message: "recordStart requires outPath")) - } - let hasAppBundleId = !(command.appBundleId? - .trimmingCharacters(in: .whitespacesAndNewlines) - .isEmpty ?? true) - guard hasAppBundleId else { - return Response(ok: false, error: ErrorPayload(message: "recordStart requires appBundleId")) - } - if activeRecording != nil { - return Response(ok: false, error: ErrorPayload(message: "recording already in progress")) - } - if let requestedFps = command.fps, (requestedFps < minRecordingFps || requestedFps > maxRecordingFps) { - return Response(ok: false, error: ErrorPayload(message: "recordStart fps must be between \(minRecordingFps) and \(maxRecordingFps)")) - } - do { - let resolvedOutPath = resolveRecordingOutPath(requestedOutPath) - let fpsLabel = command.fps.map(String.init) ?? String(RunnerTests.defaultRecordingFps) - NSLog( - "AGENT_DEVICE_RUNNER_RECORD_START requestedOutPath=%@ resolvedOutPath=%@ fps=%@", - requestedOutPath, - resolvedOutPath, - fpsLabel - ) - let recorder = ScreenRecorder( - outputPath: resolvedOutPath, - fps: command.fps.map { Int32($0) } - ) - try recorder.start { [weak self] in - return self?.captureRunnerFrame() - } - activeRecording = recorder - return Response(ok: true, data: DataPayload(message: "recording started")) - } catch { - activeRecording = nil - return Response(ok: false, error: ErrorPayload(message: "failed to start recording: \(error.localizedDescription)")) - } - case .recordStop: - guard let recorder = activeRecording else { - // The runner protocol is the durable cleanup primitive. A daemon may crash after the - // native stop succeeds but before it commits the resource transition, so exact-owner - // recovery must be able to repeat this command safely. Public `record stop` still owns - // its user-facing no-active validation through the daemon session manifest. - return Response(ok: true, data: DataPayload(message: "recording already stopped")) - } - do { - try recorder.stop() - activeRecording = nil - return Response(ok: true, data: DataPayload(message: "recording stopped")) - } catch { - activeRecording = nil - return Response(ok: false, error: ErrorPayload(message: "failed to stop recording: \(error.localizedDescription)")) - } - case .uptime: - return executeUptime() - case .activate: - guard - let bundleId = command.appBundleId?.trimmingCharacters(in: .whitespacesAndNewlines), - !bundleId.isEmpty - else { - return Response(ok: false, error: ErrorPayload(message: "activate requires appBundleId")) - } - // prepareActiveCommandContext already activated this bundle. Keep this case as the - // explicit acknowledgement after that preflight, not as a second activation. - return Response(ok: true, data: DataPayload(message: "app activated")) - case .terminate: - guard - let bundleId = command.appBundleId?.trimmingCharacters(in: .whitespacesAndNewlines), - !bundleId.isEmpty - else { - return Response(ok: false, error: ErrorPayload(message: "terminate requires appBundleId")) - } - XCUIApplication(bundleIdentifier: bundleId).terminate() - if currentBundleId == bundleId { - invalidateCachedTarget(reason: "target_terminated") - } - return Response(ok: true, data: DataPayload(message: "app terminated")) - default: - break - } - return try executeOnMainPrepared( - command: command, - activeApp: activeApp, - alertDeadline: alertDeadline - ) - } - - private func prepareActiveCommandContext( - command: Command, - routeToSpringboard: Bool = false - ) -> ActiveCommandPreparation { - var activeApp = currentApp ?? app - var systemSurface: SystemSurfaceHost? = nil - if routeToSpringboard { - activeApp = springboard - } else if shouldSkipAppActivationPreflight(command) { - activeApp = resolveAppWithoutActivation(command: command) - } else if let presented = presentedSystemSurfaceHost() { - // Serve and drive the presented surface IN PLACE: never activate it (that cancels what it - // presents) and never adopt it as the cached session target, so once it is gone the next - // command resolves back to the still-bound session app (#2438). - activeApp = presented.app - systemSurface = presented.host - if isInteractionCommand(command.command) { - applyInteractionStabilizationIfNeeded() - } - } else if !isRunnerLifecycleCommand(command.command) { - let normalizedBundleId = command.appBundleId? - .trimmingCharacters(in: .whitespacesAndNewlines) - let requestedBundleId = (normalizedBundleId?.isEmpty == true) ? nil : normalizedBundleId - if let bundleId = requestedBundleId { - if currentBundleId != bundleId || currentApp == nil { - _ = activateTarget(bundleId: bundleId, reason: "bundle_changed") - } else { - refreshCachedTargetIfProcessChanged(bundleId: bundleId) - } - } else { - // Do not reuse stale bundle targets when the caller does not explicitly request one. - invalidateCachedTarget(reason: "missing_app_bundle") - } - - activeApp = currentApp ?? app - if let bundleId = requestedBundleId, targetNeedsActivation(activeApp) { - activeApp = activateTarget(bundleId: bundleId, reason: "stale_target") - } else if requestedBundleId == nil, targetNeedsActivation(activeApp) { - ensureRunnerHostAppActive(reason: "missing_app_bundle") - activeApp = app - } - - let skipExistenceWait = canUseFastForegroundAppGuard( - activeApp: activeApp, - requestedBundleId: requestedBundleId, - command: command.command - ) - if !skipExistenceWait && !activeApp.waitForExistence(timeout: appExistenceTimeout) { - if let bundleId = requestedBundleId { - activeApp = activateTarget(bundleId: bundleId, reason: "missing_after_wait") - guard activeApp.waitForExistence(timeout: appExistenceTimeout) else { - return .response(Response(ok: false, error: ErrorPayload(message: "app '\(bundleId)' is not available"))) - } - } else { - return .response(Response(ok: false, error: ErrorPayload(message: "runner app is not available"))) - } - } - - if isInteractionCommand(command.command) { - if let bundleId = requestedBundleId, activeApp.state != .runningForeground { - activeApp = activateTarget(bundleId: bundleId, reason: "interaction_foreground_guard") - } else if requestedBundleId == nil, activeApp.state != .runningForeground { - ensureRunnerHostAppActive(reason: "interaction_missing_app_bundle") - activeApp = app - } - let skipInteractionExistenceWait = canUseFastForegroundAppGuard( - activeApp: activeApp, - requestedBundleId: requestedBundleId, - command: command.command - ) - if !skipInteractionExistenceWait && !activeApp.waitForExistence(timeout: 2) { - if let bundleId = requestedBundleId { - return .response(Response(ok: false, error: ErrorPayload(message: "app '\(bundleId)' is not available"))) - } - return .response(Response(ok: false, error: ErrorPayload(message: "runner app is not available"))) - } - applyInteractionStabilizationIfNeeded() - } - } - return .context(ActiveCommandContext(app: activeApp, systemSurface: systemSurface)) - } - - /// A registered system surface host that is genuinely on screen, or nil. Presence is foreground - /// state, not tree content: a torn-down host still serves a rich tree, and it can only be - /// foreground-with-a-stale-tree if something activated it, which the open guard refuses. `state` - /// never activates and is cheap when the host is absent. See docs/adr/0004. - private func presentedSystemSurfaceHost() -> (host: SystemSurfaceHost, app: XCUIApplication)? { -#if os(iOS) - for host in SystemSurfaceHostRegistry.hosts { - let candidate = XCUIApplication(bundleIdentifier: host.bundleId) - if candidate.state == .runningForeground { - return (host, candidate) - } - } - return nil -#else - return nil -#endif - } - +extension RunnerTests { func executeOnMainPrepared( command: Command, activeApp: XCUIApplication, @@ -1633,7 +12,8 @@ extension RunnerTests { clearRememberedTextEntryTap() } switch command.command { - case .status, .activate, .terminate, .targetReset, .shutdown, .recordStart, .recordStop, .uptime: + case .status, .activate, .terminate, .targetReset, .shutdown, .recordStart, .recordStop, .uptime, + .snapshot: return Response( ok: false, error: ErrorPayload( @@ -1698,20 +78,15 @@ extension RunnerTests { app: activeApp, policy: synthesizedGesturePolicy(policyKind) ) - let (timing, outcome) = performGesture(activeApp, idleTimeout: false) { + switch performSynthesizedGesture(activeApp, kind: policyKind, context: context, synthesize: { synthesizedTapAt( app: activeApp, x: touchPoint.x, y: touchPoint.y, context: context ) - } - if case .performed = outcome { - logSynthesizedGesturePolicyDecision( - kind: policyKind, - context: context, - fallbackAttempted: false - ) + }) { + case .performed(let timing): if isTextEntry { waitForTextEntryReadinessAfterTap(app: activeApp, element: element) } @@ -1727,13 +102,12 @@ extension RunnerTests { ? match.usedNonHittableFallback : nil ) + case .xctestFallback(let message, let hint): + fallback = GestureFallback(strategy: "xctest-coordinate-tap", message: message, hint: hint) + case .refused(_, let message, let hint): + clearRememberedTextEntryTap() + return unsupportedResponse(message: message, hint: hint) } - logSynthesizedGesturePolicyDecision( - kind: policyKind, - context: context, - fallbackAttempted: true - ) - fallback = gestureFallback(strategy: "xctest-coordinate-tap", from: outcome) } let (timing, outcome) = performGesture(activeApp) { if expectedPoint != nil || match.usedNonHittableFallback { @@ -1792,16 +166,18 @@ extension RunnerTests { app: activeApp, policy: synthesizedGesturePolicy(policyKind) ) - let (timing, outcome) = performGesture(activeApp, idleTimeout: false) { + switch performSynthesizedGesture(activeApp, kind: policyKind, context: context, synthesize: { synthesizedTapAt(app: activeApp, x: x, y: y, context: context) - } - if case .performed = outcome { - logSynthesizedGesturePolicyDecision(kind: policyKind, context: context, fallbackAttempted: false) + }) { + case .performed(let timing): rememberTextEntryTap(textInput) return gestureResponse(message: "tapped", timing: timing) + case .xctestFallback(let message, let hint): + fallback = GestureFallback(strategy: "xctest-coordinate-tap", message: message, hint: hint) + case .refused(_, let message, let hint): + clearRememberedTextEntryTap() + return unsupportedResponse(message: message, hint: hint) } - logSynthesizedGesturePolicyDecision(kind: policyKind, context: context, fallbackAttempted: true) - fallback = gestureFallback(strategy: "xctest-coordinate-tap", from: outcome) } let touchFrame = resolvedTouchVisualizationFrame(app: activeApp, x: x, y: y) let (timing, outcome) = performGesture(activeApp) { tapAt(app: activeApp, x: x, y: y) } @@ -1867,9 +243,7 @@ extension RunnerTests { x2: x2, y2: y2, durationMs: defaults.durationMs, - synthesized: command.synthesized == true, - message: "dragged", - synthesizedPolicyKind: .synthesizedDrag + message: "dragged" ) case .scroll: // Fused frame-resolve + drag scroll for non-tvOS. On iOS this intentionally stays on the @@ -2051,10 +425,7 @@ extension RunnerTests { return Response(ok: false, error: ErrorPayload(message: "readText did not resolve text")) } return Response(ok: true, data: DataPayload(text: text)) - case .snapshot: - return try executeSnapshotPrepared(command: command, activeApp: activeApp) case .screenshot: - let screenshot: XCUIScreenshot #if os(macOS) // macOS keeps the app-targeted capture behavior for window-level screenshots. if let bundleId = command.appBundleId, !bundleId.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { @@ -2064,6 +435,7 @@ extension RunnerTests { // Brief wait for the app transition animation to complete sleepFor(0.5) } + let screenshot: XCUIScreenshot if command.fullscreen == true { screenshot = XCUIScreen.main.screenshot() } else if let bundleId = command.appBundleId, !bundleId.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { @@ -2071,37 +443,67 @@ extension RunnerTests { } else { screenshot = XCUIScreen.main.screenshot() } -#else - screenshot = XCUIScreen.main.screenshot() -#endif - guard let pngData = runnerPngData(for: screenshot.image) else { - return Response(ok: false, error: ErrorPayload(message: "Failed to encode screenshot as PNG")) - } - if command.inlineScreenshot == true { + return screenshotResponse( + image: screenshot.image, + inlineScreenshot: command.inlineScreenshot == true + ) + #elseif os(iOS) + // A foldable lights one panel at a time while `XCUIScreen.main` names one fixed panel, so an + // app on the other panel is captured as a valid PNG of nothing. The capture comes from the + // display owning a window, and a display that no window can name fails this required capture + // instead of answering with the main screen (#2728). + // + // `.screenshot` is a runner-lifecycle command, which skips the preflight that resolves the + // session app, so on a fresh runner process `activeApp` is still the runner host app. The + // requested bundle id is resolved here and never activated: an observation must not change + // which app is foregrounded in order to see which display it is on (#2728). + switch captureObservedScreen(app: resolveAppWithoutActivation(command: command)) { + case .failure(let failure): return Response( - ok: true, - data: DataPayload(imageBase64: pngData.base64EncodedString()) + ok: false, + error: ErrorPayload(code: failure.rawValue, message: failure.message, hint: failure.hint) + ) + case .success(let captured): + // The facts travel to the host in the response and to the operator in runner.log, because a + // capture whose density is in question is argued from what the capture itself measured. + NSLog( + "AGENT_DEVICE_RUNNER_SCREEN_CAPTURE display=%lu pixels=%ldx%ld pixelsPerPoint=%g", + captured.displayID, + captured.pixelWidth, + captured.pixelHeight, + captured.pixelsPerPoint + ) + return screenshotResponse( + image: captured.image, + inlineScreenshot: command.inlineScreenshot == true, + metadata: ScreenshotMetadataPayload( + displayID: captured.displayID, + pixelWidth: captured.pixelWidth, + pixelHeight: captured.pixelHeight, + pixelsPerPoint: captured.pixelsPerPoint + ) ) } - let fileName = "screenshot-\(Int(Date().timeIntervalSince1970 * 1000)).png" - let filePath = (NSTemporaryDirectory() as NSString).appendingPathComponent(fileName) - do { - try pngData.write(to: URL(fileURLWithPath: filePath)) - } catch { - return Response(ok: false, error: ErrorPayload(message: "Failed to write screenshot: \(error.localizedDescription)")) - } -#if os(macOS) - return Response(ok: true, data: DataPayload(message: filePath)) #else - // Return path relative to app container root (tmp/ maps to NSTemporaryDirectory) - return Response(ok: true, data: DataPayload(message: "tmp/\(fileName)")) + return screenshotResponse( + image: XCUIScreen.main.screenshot().image, + inlineScreenshot: command.inlineScreenshot == true + ) #endif - case .back, .backInApp: - if tapInAppBackControl(app: activeApp) { - let message = command.command == .back ? "back" : "backInApp" - return Response(ok: true, data: DataPayload(message: message)) + case .backInApp: + switch tapInAppBackControl(app: activeApp) { + case .performed: + return Response(ok: true, data: DataPayload(message: "backInApp")) + case .unavailable: + return Response( + ok: false, + error: ErrorPayload(message: "in-app back control is not available") + ) + case .unverified(let error): + // The fallback gesture ran but the display refused to be sampled. Reporting the typed refusal + // keeps an unknown outcome from being laundered into a definitive "no back control" (#2728). + return Response(ok: false, error: error) } - return Response(ok: false, error: ErrorPayload(message: "in-app back control is not available")) case .backSystem: if performSystemBackAction(app: activeApp) { return Response(ok: true, data: DataPayload(message: "backSystem")) @@ -2209,10 +611,8 @@ extension RunnerTests { x2: last.x, y2: last.y, durationMs: plan.durationMs, - synthesized: true, message: plan.intent, - synthesizedPolicyKind: .synthesizedDrag, - synthesizedProfile: .fastSwipe + synthesized: (profile: .fastSwipe, policyKind: .synthesizedDrag) ) ) case .sampled: @@ -2231,603 +631,4 @@ extension RunnerTests { return executeSequence(command: command, activeApp: activeApp) } } - - private func invalidScrollDirectionResponse(commandName: String) -> Response { - Response( - ok: false, - error: ErrorPayload( - code: "INVALID_ARGS", - message: "\(commandName) requires direction up|down|left|right" - ) - ) - } - - private func scrollDurationIsValid(_ durationMs: Double?) -> Bool { - guard let durationMs else { return true } - return durationMs.isFinite && durationMs >= 0 && durationMs <= 10000 - } - - private func invalidScrollDurationResponse(commandName: String) -> Response { - return Response( - ok: false, - error: ErrorPayload( - code: "INVALID_ARGS", - message: "\(commandName) durationMs must be between 0 and 10000" - ) - ) - } - - private func executeScrollDragGesture( - activeApp: XCUIApplication, - x: Double, - y: Double, - x2: Double, - y2: Double, - durationMs: Double, - message: String, - context: SynthesizedCoordinateContext, - releaseBehavior: ScrollReleaseBehavior? - ) -> Response { -#if os(iOS) - return executeDragGesture( - activeApp: activeApp, - x: x, - y: y, - x2: x2, - y2: y2, - durationMs: durationMs, - synthesized: true, - message: message, - synthesizedContext: context, - synthesizedPolicyKind: .scroll, - synthesizedProfile: scrollDragProfile(releaseBehavior: releaseBehavior) - ) -#else - return executeDragGesture( - activeApp: activeApp, - x: x, - y: y, - x2: x2, - y2: y2, - durationMs: durationMs, - synthesized: false, - message: message, - synthesizedPolicyKind: .scroll - ) -#endif - } - - /// Shared drag execution for explicit drag commands. The iOS synthesized lane keeps its - /// fallback policy explicit; viewport scrolling owns a separate single drag specification. - private func executeDragGesture( - activeApp: XCUIApplication, - x: Double, - y: Double, - x2: Double, - y2: Double, - durationMs: Double?, - synthesized: Bool, - message: String, - synthesizedContext: SynthesizedCoordinateContext? = nil, - synthesizedPolicyKind: SynthesizedGesturePolicyKind, - synthesizedProfile: SynthesizedDragProfile = .continuous - ) -> Response { - let durationMs = durationMs ?? runnerDefaultDragDurationMs - let commandName = dragCommandName(message: message) - guard x.isFinite, y.isFinite, x2.isFinite, y2.isFinite else { - return Response( - ok: false, - error: ErrorPayload(code: "INVALID_ARGS", message: "\(commandName) requires finite coordinates") - ) - } - if synthesized, let synthesizedResponse = executeSynthesizedDragGesture( - activeApp: activeApp, - x: x, - y: y, - x2: x2, - y2: y2, - durationMs: durationMs, - message: message, - context: synthesizedContext, - policyKind: synthesizedPolicyKind, - profile: synthesizedProfile - ) { - return synthesizedResponse - } - let dragPoints = keyboardAvoidingDragPoints(app: activeApp, x: x, y: y, x2: x2, y2: y2) - let dragFrame = resolvedDragVisualizationFrame( - app: activeApp, - x: dragPoints.x, - y: dragPoints.y, - x2: dragPoints.x2, - y2: dragPoints.y2 - ) - var fallback: GestureFallback? - if synthesized { - let durationMs = min(max(durationMs, 16), 10000) - let context = synthesizedCoordinateContext( - app: activeApp, - policy: synthesizedGesturePolicy(synthesizedPolicyKind) - ) - let (timing, outcome) = performGesture(activeApp, idleTimeout: false) { - synthesizedDragAt( - app: activeApp, - x: dragPoints.x, - y: dragPoints.y, - x2: dragPoints.x2, - y2: dragPoints.y2, - durationMs: durationMs, - profile: synthesizedProfile, - context: context - ) - } - if case .performed = outcome { - return gestureResponse(message: message, timing: timing, frame: .drag(dragFrame)) - } - fallback = gestureFallback(strategy: "xctest-coordinate-drag", from: outcome) - } - let holdDuration = synthesized - ? synthesizedSwipeFallbackHoldDuration(durationMs: durationMs) - : coordinateDragHoldDuration() - let (timing, outcome) = performGesture(activeApp) { - dragAt( - app: activeApp, - x: dragPoints.x, - y: dragPoints.y, - x2: dragPoints.x2, - y2: dragPoints.y2, - holdDuration: holdDuration - ) - } - if let response = unsupportedResponse(for: outcome) { - return response - } - return gestureResponse( - message: message, - timing: timing, - frame: .drag(dragFrame), - fallback: fallback - ) - } - - private func executeSynthesizedDragGesture( - activeApp: XCUIApplication, - x: Double, - y: Double, - x2: Double, - y2: Double, - durationMs: Double, - message: String, - context: SynthesizedCoordinateContext?, - policyKind: SynthesizedGesturePolicyKind, - profile: SynthesizedDragProfile - ) -> Response? { -#if os(iOS) - let policy = synthesizedGesturePolicy(policyKind) - let context = context ?? synthesizedCoordinateContext(app: activeApp, policy: policy) - guard let plan = axFreeSynthesizedDragPlan( - app: activeApp, - x: x, - y: y, - x2: x2, - y2: y2, - context: context - ) - else { - if context?.allowsXCTestCoordinateFallback == true { - logSynthesizedGesturePolicyDecision(kind: policyKind, context: context, fallbackAttempted: true) - return executeCoordinateDragFallback( - activeApp: activeApp, - x: x, - y: y, - x2: x2, - y2: y2, - durationMs: durationMs, - message: message, - fallback: nil - ) - } - logSynthesizedGesturePolicyDecision(kind: policyKind, context: context, fallbackAttempted: false) - return Response( - ok: false, - error: ErrorPayload( - code: "INVALID_ARGS", - message: "\(dragCommandName(message: message)) could not resolve a finite synthesized coordinate frame" - ) - ) - } - let durationMs = min(max(durationMs, 16), 10000) - let dragFrame = axFreeDragVisualizationFrame( - x: plan.points.x, - y: plan.points.y, - x2: plan.points.x2, - y2: plan.points.y2, - referenceFrame: plan.referenceFrame - ) - let (timing, outcome) = performGesture(activeApp, idleTimeout: false) { - synthesizedDragAt( - app: activeApp, - x: plan.points.x, - y: plan.points.y, - x2: plan.points.x2, - y2: plan.points.y2, - durationMs: durationMs, - profile: profile, - context: plan.context - ) - } - if case .performed = outcome { - logSynthesizedGesturePolicyDecision(kind: policyKind, context: plan.context, fallbackAttempted: false) - return gestureResponse(message: message, timing: timing, frame: .drag(dragFrame)) - } - if plan.context.allowsXCTestCoordinateFallback { - logSynthesizedGesturePolicyDecision(kind: policyKind, context: plan.context, fallbackAttempted: true) - return executeCoordinateDragFallback( - activeApp: activeApp, - x: plan.points.x, - y: plan.points.y, - x2: plan.points.x2, - y2: plan.points.y2, - durationMs: durationMs, - message: message, - fallback: gestureFallback(strategy: "xctest-coordinate-drag", from: outcome) - ) - } - logSynthesizedGesturePolicyDecision(kind: policyKind, context: plan.context, fallbackAttempted: false) - return unsupportedResponse(for: outcome) -#else - return nil -#endif - } - - private func executeCoordinateDragFallback( - activeApp: XCUIApplication, - x: Double, - y: Double, - x2: Double, - y2: Double, - durationMs: Double, - message: String, - fallback: GestureFallback? - ) -> Response { - let dragPoints = keyboardAvoidingDragPoints(app: activeApp, x: x, y: y, x2: x2, y2: y2) - let dragFrame = resolvedDragVisualizationFrame( - app: activeApp, - x: dragPoints.x, - y: dragPoints.y, - x2: dragPoints.x2, - y2: dragPoints.y2 - ) - let holdDuration = synthesizedSwipeFallbackHoldDuration(durationMs: durationMs) - let (timing, outcome) = performGesture(activeApp) { - dragAt( - app: activeApp, - x: dragPoints.x, - y: dragPoints.y, - x2: dragPoints.x2, - y2: dragPoints.y2, - holdDuration: holdDuration - ) - } - if let response = unsupportedResponse(for: outcome) { - return response - } - return gestureResponse( - message: message, - timing: timing, - frame: .drag(dragFrame), - fallback: fallback - ) - } - - /// Adds the #2500 avoidance evidence to a scroll response. Only the frame resolver knows whether - /// it trimmed the swipe for a keyboard, and only `scroll` has this evidence to carry, so it is - /// attached where the frame was resolved rather than threaded through every gesture response. - /// The refusal a keyboard forces. It performs no gesture: swiping into the keys would leave the - /// surface where it was, which the daemon's no-progress fingerprint reads as a stuck container - /// (#2499) and an agent reads as a broken scroll. The TS owner maps the code to the - /// `scroll_keyboard_occludes_surface` reason and the "dismiss the keyboard" hint. - private func scrollKeyboardOccludedResponse( - direction: String, - keyboardMinY: Double, - visibleHeight: Double - ) -> Response { - return Response( - ok: false, - error: ErrorPayload( - code: ScrollViewportPolicy.occlusionRunnerCode, - message: String( - format: - "scroll %@ refused: the keyboard leaves %.0fpt of visible surface above it, too little to swipe", - direction, - visibleHeight - ) - ) - ) - } - - private func dragCommandName(message: String) -> String { - return message == "scrolled" ? "scroll" : "drag" - } - - func currentXCTestFailureCount() -> Int { - return testRun?.failureCount ?? 0 - } - - func didRecordXCTestFailure(since failureCountBefore: Int) -> Bool { - return currentXCTestFailureCount() > failureCountBefore - } - - private func xctestRecordedFailureResponse(command: Command, response: Response) -> Response? { - guard response.ok else { return nil } - if response.data?.runnerFatal == true { - return nil - } - guard !isReadOnlyCommand(command), !isRunnerLifecycleCommand(command.command) else { - return nil - } - return Response( - ok: false, - error: ErrorPayload( - code: "XCTEST_RECORDED_FAILURE", - message: "XCTest recorded a failure while executing \(command.command.rawValue); the action may not have been performed.", - hint: "The iOS runner session was invalidated. Re-observe with a fresh snapshot before retrying; if the accessibility tree is unavailable, use screenshot plus coordinate commands instead of retrying the tap blindly." - ) - ) - } - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS - func runnerCommandFixture(_ json: String) throws -> Command { - try JSONDecoder().decode(Command.self, from: Data(json.utf8)) - } -#endif - - private func shouldSkipAppActivationPreflight(_ command: Command) -> Bool { -#if os(iOS) - if command.command == .alert { - return true - } - // A hardware Action Button press belongs to the system, not to the session app: the Shortcut or - // App Intent behind it is expected to run whether that app is foregrounded, backgrounded, or - // terminated, and activating first would foreground exactly what the press should leave alone. - // The press keeps its recorded-failure conversion, which `isLifecycle` would have removed - // (#2699, #2702 review). - if command.command == .actionButton { - return true - } - // Coordinate-only synthesized taps can run after an AX-fatal foreground screen because they do not - // need app activation, window lookup, keyboard lookup, or element resolution. Selector/text - // interactions intentionally stay on the normal AX path because they need an element query. - // Scroll/drag/sequence keep the normal foreground guard and stabilization path. - guard command.text == nil, command.selectorKey == nil else { return false } - guard hasCachedTargetForActivationSkip(command: command) else { return false } - return isCoordinateOnlyTap(command) -#else - return false -#endif - } - - private func shouldRouteToSpringboardBlockingSystemModal( - _ command: Command - ) -> Bool { -#if os(iOS) - guard isCoordinateOnlyTap(command) else { - return false - } - #if AGENT_DEVICE_RUNNER_UNIT_TESTS - if let override = blockingSystemModalPresenceOverrideForTesting { - return override - } - #endif - let probeDeadline = Date().addingTimeInterval(systemModalProbeBudget) - // `runMainThreadWork` executes inline for a main-thread caller, so that path cannot use its - // timeout machinery. Direct main-thread dispatch keeps the prior synchronous modal check; - // normal off-main command dispatch uses the bounded probe and post-probe busy recovery. - if Thread.isMainThread { - return firstBlockingSystemModal( - in: springboard, - deadline: probeDeadline - ) != nil - } - return boundedBlockingSystemAlertSnapshot( - deadline: probeDeadline - ) != nil -#else - return false -#endif - } - - private func isCoordinateOnlyTap(_ command: Command) -> Bool { - return command.command == .tap - && command.text == nil - && command.selectorKey == nil - && command.x != nil - && command.y != nil - } - - private func hasCachedTargetForActivationSkip(command: Command) -> Bool { - guard let currentApp, currentApp.state == .runningForeground else { return false } - guard let bundleId = command.appBundleId?.trimmingCharacters(in: .whitespacesAndNewlines), - !bundleId.isEmpty - else { - return true - } - return currentBundleId == bundleId - } - - private func resolveAppWithoutActivation(command: Command) -> XCUIApplication { - guard let bundleId = command.appBundleId? - .trimmingCharacters(in: .whitespacesAndNewlines), - !bundleId.isEmpty - else { - return currentApp ?? app - } - if currentBundleId == bundleId, let currentApp { - return currentApp - } - return XCUIApplication(bundleIdentifier: bundleId) - } - - func executeTypeCommand(activeApp: XCUIApplication, command: Command) -> Response { - guard let text = command.text else { - return Response(ok: false, error: ErrorPayload(message: "type requires text")) - } - let delaySeconds = Double(max(command.delayMs ?? 0, 0)) / 1000.0 - let textEntryMode = resolveTextEntryMode(command) - let target: TextEntryTarget - var resolvedCoordinateContext: SynthesizedCoordinateContext? - var maestroNonHittableCoordinateFallbackUsed: Bool? - if command.allowNonHittableCoordinateFallback == true, - command.x != nil, - command.y != nil - { - // The shared runtime has already resolved this node as non-hittable and - // deliberately selected Maestro's coordinate compatibility route. - maestroNonHittableCoordinateFallbackUsed = true - } - let focusStartedAt = Date() -#if os(iOS) - let xCTestChannelPenalized = isSnapshotXCTestChannelPenalized(bundleId: currentBundleId) - var resolvedCoordinateTarget: TextEntryTarget? - if Self.shouldUseResolvedCoordinateTextEntryRoute( - repairMode: textEntryMode, - hasX: command.x != nil, - hasY: command.y != nil, - xCTestChannelPenalized: xCTestChannelPenalized - ), let x = command.x, let y = command.y { - let policyKind = SynthesizedGesturePolicyKind.coordinateTap - let context = synthesizedCoordinateContext( - app: activeApp, - policy: synthesizedGesturePolicy(policyKind) - ) - let (_, outcome) = performGesture(activeApp, idleTimeout: false) { - synthesizedTapAt(app: activeApp, x: x, y: y, context: context) - } - if Self.shouldFallbackFromSynthesizedTextEntryFocus(outcome) { - logSynthesizedGesturePolicyDecision( - kind: policyKind, - context: context, - fallbackAttempted: true - ) - } else { - logSynthesizedGesturePolicyDecision( - kind: policyKind, - context: context, - fallbackAttempted: false - ) - resolvedCoordinateContext = context - resolvedCoordinateTarget = TextEntryTarget( - element: nil, - refreshPoint: CGPoint(x: x, y: y), - prefersFocusedElement: false - ) - } - } -#else - let xCTestChannelPenalized = false - let resolvedCoordinateTarget: TextEntryTarget? = nil -#endif - if let resolvedCoordinateTarget { - target = resolvedCoordinateTarget - } else if let selectorKey = command.selectorKey, let selectorValue = command.selectorValue { - // Released daemons may still send selector-keyed type commands even though current - // daemons resolve fill selectors through the runtime tree before reaching the runner. - let match = findElement( - app: activeApp, - selectorKey: selectorKey, - selectorValue: selectorValue, - allowNonHittableFallback: command.allowNonHittableCoordinateFallback == true - ) - if match.isAmbiguous { - return Response(ok: false, error: ErrorPayload(code: "AMBIGUOUS_MATCH", message: "selector matched multiple elements")) - } - guard let element = match.element else { - return Response(ok: false, error: ErrorPayload(code: "NO_MATCH", message: "selector did not match an element")) - } - guard isTextEntryElement(element) else { - return Response(ok: false, error: ErrorPayload(code: "INVALID_TARGET", message: "selector did not match a text input")) - } - if command.allowNonHittableCoordinateFallback == true { - maestroNonHittableCoordinateFallbackUsed = match.usedNonHittableFallback - } - target = focusTextInputForTextEntry(app: activeApp, element: element) - } else { - target = focusTextInputForTextEntry(app: activeApp, x: command.x, y: command.y) - } - NSLog( - "AGENT_DEVICE_RUNNER_TEXT_ENTRY_PHASE commandId=%@ phase=focus durationMs=%.1f chars=%d mode=%@", - command.commandId ?? "", - Date().timeIntervalSince(focusStartedAt) * 1000.0, - text.count, - textEntryModeName(textEntryMode) - ) - if textEntryMode == .replacement { -#if os(iOS) - let canReplaceResolvedFirstResponder = Self.shouldUseSynthesizedFirstResponderReplacement( - hasResolvedElement: target.element != nil, - hasRefreshPoint: target.refreshPoint != nil, - xCTestChannelPenalized: xCTestChannelPenalized - ) -#else - let canReplaceResolvedFirstResponder = false -#endif - guard target.element != nil || canReplaceResolvedFirstResponder else { - let message = - (command.x != nil && command.y != nil) - ? "no text input found at the provided coordinates to clear" - : "no focused text input to clear" - return Response(ok: false, error: ErrorPayload(message: message)) - } - } - let textResult = typeTextReliably( - app: activeApp, - target: target, - text: text, - delaySeconds: delaySeconds, - repairMode: textEntryMode, - xCTestChannelPenalized: xCTestChannelPenalized, - synthesizer: PrivateXCTestTextEntrySynthesizer(), - commandId: command.commandId - ) - if let failure = textResult.failure { - return Response( - ok: false, - error: ErrorPayload(code: failure.rawValue, message: failure.message, hint: failure.hint) - ) - } - if textResult.verified == false { - let expected = textResult.expectedText ?? "" - let observed = textResult.observedText ?? "" - return Response( - ok: false, - error: ErrorPayload( - code: "TEXT_ENTRY_MISMATCH", - message: "text entry verification failed: expected \"\(expected)\", observed \"\(observed)\"" - ) - ) - } - let point = target.refreshPoint - let frame: CGRect - if let resolvedCoordinateContext { - frame = resolvedCoordinateContext.referenceFrame - } else if point != nil { - frame = activeApp.frame - } else { - // Bare `type` has no coordinate response to normalize. Avoid serializing the - // application AX tree only to emit unused reference dimensions. - frame = .zero - } - return Response( - ok: true, - data: DataPayload( - message: textResult.repaired ? "typed after repair" : "typed", - x: point.map { Double($0.x) }, - y: point.map { Double($0.y) }, - referenceWidth: frame.isEmpty ? nil : Double(frame.width), - referenceHeight: frame.isEmpty ? nil : Double(frame.height), - maestroNonHittableCoordinateFallbackUsed: maestroNonHittableCoordinateFallbackUsed, - textEntryRoute: textResult.textEntryRoute - ) - ) - } } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandJournal.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandJournal.swift index f15acf8f9b..c91955116d 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandJournal.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandJournal.swift @@ -136,7 +136,7 @@ final class RunnerCommandJournal { case .snapshot, .screenshot: return false case .tap, .mouseClick, .longPress, .drag, - .remotePress, .type, .swipe, .scroll, .desktopScroll, .findText, .querySelector, .readText, .back, + .remotePress, .type, .swipe, .scroll, .desktopScroll, .findText, .querySelector, .readText, .backInApp, .backSystem, .home, .rotate, .appSwitcher, .actionButton, .keyboardDismiss, .keyboardReturn, .alert, .sequence, .gesture, .gestureViewport, .recordStart, .recordStop, .status, .uptime, .activate, .terminate, .targetReset, .shutdown: @@ -144,340 +144,3 @@ final class RunnerCommandJournal { } } } - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS -extension RunnerTests { - func testUptimeBypassesCommandJournal() throws { - let command = runnerJournalCommand("uptime", id: "uptime-probe") - - let response = try execute(command: command) - let status = commandJournal.status(normalizedCommandId: "uptime-probe") - - XCTAssertEqual(response.ok, true) - XCTAssertNotNil(response.data?.currentUptimeMs) - XCTAssertEqual(status.lifecycleState, RunnerCommandLifecycleState.notAccepted.rawValue) - } - - func testStampingCurrentUptimePreservesPayload() { - let stamped = Response(ok: true, data: DataPayload(message: "recording started")) - .stampingCurrentUptimeMs(123.5) - - XCTAssertEqual(stamped.ok, true) - XCTAssertEqual(stamped.data?.message, "recording started") - XCTAssertEqual(stamped.data?.currentUptimeMs, 123.5) - } - - func testStampingCurrentUptimeCreatesPayloadWhenNil() { - let stamped = Response(ok: true).stampingCurrentUptimeMs(456.0) - - XCTAssertEqual(stamped.ok, true) - XCTAssertEqual(stamped.data?.currentUptimeMs, 456.0) - } - - func testStampingCurrentUptimeSkipsErrorResponses() { - let response = Response(ok: false, error: ErrorPayload(message: "boom")) - let stamped = response.stampingCurrentUptimeMs(789.0) - - XCTAssertEqual(stamped.ok, false) - XCTAssertNil(stamped.data) - XCTAssertEqual(stamped.error?.message, "boom") - } - - func testStampingCurrentMainThreadBusyPreservesPayload() { - let stamped = Response(ok: true, data: DataPayload(nodes: [], truncated: false)) - .stampingCurrentMainThreadBusy(true) - - XCTAssertEqual(stamped.ok, true) - XCTAssertEqual(stamped.data?.runnerMainThreadBusy, true) - } - - func testStampingCurrentMainThreadBusySkipsErrorResponses() { - let response = Response(ok: false, error: ErrorPayload(code: "RUNNER_BUSY", message: "busy")) - let stamped = response.stampingCurrentMainThreadBusy(true) - - XCTAssertEqual(stamped.ok, false) - XCTAssertNil(stamped.data) - XCTAssertEqual(stamped.error?.code, "RUNNER_BUSY") - } - - func testMainThreadBusyStateReportsOccupancy() { - XCTAssertFalse(MainThreadBusyState.idle.reportsMainThreadBusy) - XCTAssertTrue(MainThreadBusyState.busy(abandonedForSeconds: 5).reportsMainThreadBusy) - XCTAssertTrue(MainThreadBusyState.wedged(abandonedForSeconds: 200).reportsMainThreadBusy) - XCTAssertEqual( - Response(ok: true).stampingCurrentMainThreadBusy(false).data?.runnerMainThreadBusy, false) - } - - func testCommandFailedResponseTagsMainThreadTimeoutWithTypedCode() { - let timeout = NSError( - domain: RunnerErrorDomain.general, - code: RunnerErrorCode.mainThreadExecutionTimedOut, - userInfo: [NSLocalizedDescriptionKey: "main thread execution timed out"] - ) - - let response = commandFailedResponse(from: timeout) - - XCTAssertEqual(response.ok, false) - XCTAssertEqual(response.error?.code, RunnerWireErrorCode.mainThreadTimeout) - } - - func testCommandFailedResponseKeepsGenericCodeForOtherErrors() { - let other = NSError(domain: "SomeOtherDomain", code: 99, userInfo: nil) - - let response = commandFailedResponse(from: other) - - XCTAssertEqual(response.error?.code, "COMMAND_FAILED") - } - - func testJournalStoredResponseStaysUnstamped() throws { - let journal = RunnerCommandJournal() - let recordStart = runnerJournalCommand("recordStart", id: "record-start-anchor") - - journal.accept(command: recordStart) - journal.finish( - command: recordStart, - response: Response(ok: true, data: DataPayload(message: "recording started")) - ) - - let status = journal.status(normalizedCommandId: "record-start-anchor") - let responseJson = try XCTUnwrap(status.lifecycleResponseJson) - XCTAssertFalse(responseJson.contains("currentUptimeMs")) - } - - func testCommandJournalRetentionPolicy() throws { - let journal = RunnerCommandJournal() - - let uptime = runnerJournalCommand("uptime", id: "small-scalar") - journal.accept(command: uptime) - journal.finish( - command: uptime, - response: Response(ok: true, data: DataPayload(currentUptimeMs: 12.5)) - ) - - let scalarStatus = journal.status(normalizedCommandId: "small-scalar") - XCTAssertEqual(scalarStatus.lifecycleState, RunnerCommandLifecycleState.completed.rawValue) - XCTAssertEqual(scalarStatus.lifecycleResponseOk, true) - XCTAssertNotNil(scalarStatus.lifecycleResponseJson) - let scalarResponse = try decodeRunnerJournalResponse(scalarStatus.lifecycleResponseJson) - XCTAssertEqual(scalarResponse.data?.currentUptimeMs, 12.5) - - let querySelector = runnerJournalCommand("querySelector", id: "small-object") - journal.accept(command: querySelector) - journal.finish( - command: querySelector, - response: Response(ok: true, data: DataPayload(found: true, nodes: [runnerJournalNode()])) - ) - - let objectStatus = journal.status(normalizedCommandId: "small-object") - XCTAssertNotNil(objectStatus.lifecycleResponseJson) - let objectResponse = try decodeRunnerJournalResponse(objectStatus.lifecycleResponseJson) - XCTAssertEqual(objectResponse.data?.found, true) - XCTAssertEqual(objectResponse.data?.nodes?.count, 1) - - let snapshot = runnerJournalCommand("snapshot", id: "snapshot-tree") - journal.accept(command: snapshot) - journal.finish( - command: snapshot, - response: Response(ok: true, data: DataPayload(nodes: [runnerJournalNode()], truncated: false)) - ) - - let snapshotStatus = journal.status(normalizedCommandId: "snapshot-tree") - XCTAssertEqual(snapshotStatus.lifecycleState, RunnerCommandLifecycleState.completed.rawValue) - XCTAssertEqual(snapshotStatus.lifecycleResponseOk, true) - XCTAssertNil(snapshotStatus.lifecycleResponseJson) - - let screenshot = runnerJournalCommand("screenshot", id: "screenshot-artifact") - journal.accept(command: screenshot) - journal.finish( - command: screenshot, - response: Response(ok: true, data: DataPayload(message: "tmp/screenshot-1.png")) - ) - - let screenshotStatus = journal.status(normalizedCommandId: "screenshot-artifact") - XCTAssertEqual(screenshotStatus.lifecycleState, RunnerCommandLifecycleState.completed.rawValue) - XCTAssertEqual(screenshotStatus.lifecycleResponseOk, true) - XCTAssertNil(screenshotStatus.lifecycleResponseJson) - - let scroll = runnerJournalCommand("scroll", id: "scroll-drag") - journal.accept(command: scroll) - journal.finish( - command: scroll, - response: Response( - ok: true, - data: DataPayload( - message: "scrolled", - gestureStartUptimeMs: 1, - gestureEndUptimeMs: 2, - x: 155, - y: 420, - x2: 155, - y2: 301, - referenceWidth: 300, - referenceHeight: 600 - ) - ) - ) - - let scrollStatus = journal.status(normalizedCommandId: "scroll-drag") - XCTAssertEqual(scrollStatus.lifecycleState, RunnerCommandLifecycleState.completed.rawValue) - XCTAssertEqual(scrollStatus.lifecycleResponseOk, true) - XCTAssertNotNil(scrollStatus.lifecycleResponseJson) - let scrollResponse = try decodeRunnerJournalResponse(scrollStatus.lifecycleResponseJson) - XCTAssertEqual(scrollResponse.data?.x, 155) - XCTAssertEqual(scrollResponse.data?.y, 420) - XCTAssertEqual(scrollResponse.data?.x2, 155) - XCTAssertEqual(scrollResponse.data?.y2, 301) - XCTAssertEqual(scrollResponse.data?.referenceWidth, 300) - XCTAssertEqual(scrollResponse.data?.referenceHeight, 600) - - let largeRead = runnerJournalCommand("readText", id: "large-read") - journal.accept(command: largeRead) - journal.finish( - command: largeRead, - response: Response(ok: true, data: DataPayload(text: String(repeating: "x", count: 17 * 1024))) - ) - - let largeReadStatus = journal.status(normalizedCommandId: "large-read") - XCTAssertEqual(largeReadStatus.lifecycleState, RunnerCommandLifecycleState.completed.rawValue) - XCTAssertEqual(largeReadStatus.lifecycleResponseOk, true) - XCTAssertNil(largeReadStatus.lifecycleResponseJson) - } - - func testCommandJournalKeepsErrorMetadataWhenResponseJsonIsDropped() { - let journal = RunnerCommandJournal() - let snapshot = runnerJournalCommand("snapshot", id: "snapshot-error") - let hint = "Try a smaller read such as snapshot -s -d 8." - - journal.accept(command: snapshot) - journal.finish( - command: snapshot, - response: Response( - ok: false, - error: ErrorPayload( - code: "IOS_AX_SNAPSHOT_FAILED", - message: "iOS XCTest snapshot failed while serializing the accessibility tree.", - hint: hint - ) - ) - ) - - let status = journal.status(normalizedCommandId: "snapshot-error") - XCTAssertEqual(status.lifecycleState, RunnerCommandLifecycleState.failed.rawValue) - XCTAssertEqual(status.lifecycleResponseOk, false) - XCTAssertNil(status.lifecycleResponseJson) - XCTAssertEqual(status.lifecycleErrorCode, "IOS_AX_SNAPSHOT_FAILED") - XCTAssertEqual( - status.lifecycleErrorMessage, - "iOS XCTest snapshot failed while serializing the accessibility tree." - ) - XCTAssertEqual(status.lifecycleErrorHint, hint) - } - - func testCommandJournalRetainsCompletedSequenceResults() throws { - let journal = RunnerCommandJournal() - let sequence = runnerJournalCommand("sequence", id: "sequence-completed") - let results = (0..<20).map { _ in - SequenceStepResult( - ok: true, - kind: "tap", - errorCode: nil, - errorMessage: nil, - gestureStartUptimeMs: 100, - gestureEndUptimeMs: 120 - ) - } - - journal.accept(command: sequence) - journal.finish( - command: sequence, - response: Response( - ok: true, - data: DataPayload( - message: "sequence", - completedSteps: 20, - failedStepIndex: nil, - sequenceResults: results - ) - ) - ) - - let status = journal.status(normalizedCommandId: "sequence-completed") - XCTAssertEqual(status.lifecycleState, RunnerCommandLifecycleState.completed.rawValue) - XCTAssertEqual(status.lifecycleResponseOk, true) - let json = try XCTUnwrap(status.lifecycleResponseJson) - // Worst-case 20-step response must stay under the 16KB journal retention cap. - XCTAssertLessThan(json.utf8.count, 16 * 1024) - let decoded = try decodeRunnerJournalResponse(status.lifecycleResponseJson) - XCTAssertEqual(decoded.data?.completedSteps, 20) - XCTAssertEqual(decoded.data?.sequenceResults?.count, 20) - } - - func testCommandJournalRetainsFailedSequenceResults() throws { - let journal = RunnerCommandJournal() - let sequence = runnerJournalCommand("sequence", id: "sequence-failed") - let longError = String(repeating: "z", count: 200) - let results: [SequenceStepResult] = [ - SequenceStepResult(ok: true, kind: "tap", errorCode: nil, errorMessage: nil, - gestureStartUptimeMs: 100, gestureEndUptimeMs: 120), - SequenceStepResult(ok: true, kind: "tap", errorCode: nil, errorMessage: nil, - gestureStartUptimeMs: 130, gestureEndUptimeMs: 150), - SequenceStepResult(ok: false, kind: "longPress", errorCode: "UNSUPPORTED_OPERATION", - errorMessage: longError, gestureStartUptimeMs: 160, gestureEndUptimeMs: 180), - ] - - journal.accept(command: sequence) - journal.finish( - command: sequence, - response: Response( - ok: true, - data: DataPayload( - message: "sequence", - completedSteps: 2, - failedStepIndex: 2, - sequenceResults: results - ) - ) - ) - - let status = journal.status(normalizedCommandId: "sequence-failed") - XCTAssertEqual(status.lifecycleState, RunnerCommandLifecycleState.completed.rawValue) - let decoded = try decodeRunnerJournalResponse(status.lifecycleResponseJson) - XCTAssertEqual(decoded.data?.completedSteps, 2) - XCTAssertEqual(decoded.data?.failedStepIndex, 2) - XCTAssertEqual(decoded.data?.sequenceResults?.count, 3) - XCTAssertEqual(decoded.data?.sequenceResults?[2].ok, false) - XCTAssertEqual(decoded.data?.sequenceResults?[2].errorCode, "UNSUPPORTED_OPERATION") - } - - private func runnerJournalCommand(_ command: String, id: String) -> Command { - let json = #"{"command":"\#(command)","commandId":"\#(id)"}"# - return try! JSONDecoder().decode(Command.self, from: Data(json.utf8)) - } - - private func runnerJournalNode() -> PresentedNode { - SnapshotPresentation.singleElementRead( - RawAXNode( - index: 0, - type: "button", - label: "Continue", - identifier: "continue", - value: nil, - rect: SnapshotRect(x: 10, y: 20, width: 100, height: 44), - enabled: true, - focused: nil, - selected: nil, - hittable: true, - depth: 0, - parentIndex: nil, - hiddenContentAbove: nil, - hiddenContentBelow: nil - ) - ) - } - - private func decodeRunnerJournalResponse(_ responseJson: String?) throws -> Response { - let responseJson = try XCTUnwrap(responseJson) - return try JSONDecoder().decode(Response.self, from: Data(responseJson.utf8)) - } -} -#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Environment.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Environment.swift index 610c3384c6..0ae4f1f1c9 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Environment.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Environment.swift @@ -15,16 +15,4 @@ enum RunnerEnv { } return 0 } - - static func isTruthy(_ name: String) -> Bool { - guard let raw = ProcessInfo.processInfo.environment[name] else { - return false - } - switch raw.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() { - case "1", "true", "yes", "on": - return true - default: - return false - } - } } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+GestureExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+GestureExecution.swift new file mode 100644 index 0000000000..7959acbf94 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+GestureExecution.swift @@ -0,0 +1,160 @@ +import XCTest +import AgentDeviceSnapshotPresentation + +extension RunnerTests { + func currentUptimeMs() -> Double { + ProcessInfo.processInfo.systemUptime * 1000 + } + + func measureGesture(_ action: () -> Void) -> (gestureStartUptimeMs: Double, gestureEndUptimeMs: Double) { + let gestureStartUptimeMs = currentUptimeMs() + action() + return (gestureStartUptimeMs, currentUptimeMs()) + } + + func synthesizedSwipeFallbackHoldDuration(durationMs: Double) -> TimeInterval { + min(max((durationMs / 5.0) / 1000.0, 0.016), 0.120) + } + + func coordinateDragHoldDuration() -> TimeInterval { + 0.050 + } + + func unsupportedResponse(for outcome: RunnerInteractionOutcome) -> Response? { + switch outcome { + case .performed: + return nil + case .unsupported(let message, let hint): + return unsupportedResponse(message: message, hint: hint) + } + } + + func unsupportedResponse(message: String, hint: String?) -> Response { + Response( + ok: false, + error: ErrorPayload(code: "UNSUPPORTED_OPERATION", message: message, hint: hint) + ) + } + + /// Optional visualization frame returned with a gesture response. + enum GestureFrame { + case none + case touch(TouchVisualizationFrame?) + case drag(DragVisualizationFrame) + } + + struct GestureFallback { + let strategy: String + let message: String + let hint: String? + } + + + /// Runs a gesture action with uniform timing capture. Touch gestures pass `idleTimeout: true` + /// (the default) to run inside the scroll idle-timeout + quiescence-skip wrapper; synthesis + /// pointer-plan gestures pass `false` because RunnerSynthesizedGesture governs their + /// own timing. Returns the captured timing and the action's outcome. + /// + /// NOTE: a new SYNTHESIS gesture must pass `idleTimeout: false` — the default `true` would wrap + /// it in the scroll idle-timeout/quiescence-skip path and change its runtime behavior. + func performGesture( + _ app: XCUIApplication, + idleTimeout: Bool = true, + _ action: () -> RunnerInteractionOutcome + ) -> (timing: (gestureStartUptimeMs: Double, gestureEndUptimeMs: Double), outcome: RunnerInteractionOutcome) { + var outcome = RunnerInteractionOutcome.performed + let timing = measureGesture { + if idleTimeout { + withBoundedInteractionIdleTimeoutIfSupported(app, waits: .bothSkipped) { + outcome = action() + } + } else { + outcome = action() + } + } + return (timing, outcome) + } + + /// Single factory for the success payload every gesture returns (message + gesture timing + + /// an optional touch/drag visualization frame), so the field shape lives in one place. + func gestureResponse( + message: String, + timing: (gestureStartUptimeMs: Double, gestureEndUptimeMs: Double), + frame: GestureFrame = .none, + fallback: GestureFallback? = nil, + maestroNonHittableCoordinateFallbackUsed: Bool? = nil + ) -> Response { + let data: DataPayload + switch frame { + case .none: + data = DataPayload( + message: message, + gestureStartUptimeMs: timing.gestureStartUptimeMs, + gestureEndUptimeMs: timing.gestureEndUptimeMs, + gestureFallback: fallback?.strategy, + gestureFallbackMessage: fallback?.message, + gestureFallbackHint: fallback?.hint, + maestroNonHittableCoordinateFallbackUsed: maestroNonHittableCoordinateFallbackUsed + ) + case .touch(let f): + data = DataPayload( + message: message, + gestureStartUptimeMs: timing.gestureStartUptimeMs, + gestureEndUptimeMs: timing.gestureEndUptimeMs, + x: f?.x, + y: f?.y, + referenceWidth: f?.referenceWidth, + referenceHeight: f?.referenceHeight, + gestureFallback: fallback?.strategy, + gestureFallbackMessage: fallback?.message, + gestureFallbackHint: fallback?.hint, + maestroNonHittableCoordinateFallbackUsed: maestroNonHittableCoordinateFallbackUsed + ) + case .drag(let f): + data = DataPayload( + message: message, + gestureStartUptimeMs: timing.gestureStartUptimeMs, + gestureEndUptimeMs: timing.gestureEndUptimeMs, + x: f.x, + y: f.y, + x2: f.x2, + y2: f.y2, + referenceWidth: f.referenceWidth, + referenceHeight: f.referenceHeight, + gestureFallback: fallback?.strategy, + gestureFallbackMessage: fallback?.message, + gestureFallbackHint: fallback?.hint + ) + } + return Response(ok: true, data: data) + } + + /// Gesture plans already return canonical centroid endpoints from the portable runtime. + /// Keep runner timing/fallback diagnostics, but do not leak the coordinate-drag adapter's + /// visualization frame into only the fast-fling response shape. + func canonicalPlannedGestureResponse(_ response: Response) -> Response { + guard response.ok, let data = response.data else { return response } + return Response( + ok: true, + data: DataPayload( + message: data.message, + gestureStartUptimeMs: data.gestureStartUptimeMs, + gestureEndUptimeMs: data.gestureEndUptimeMs, + gestureFallback: data.gestureFallback, + gestureFallbackMessage: data.gestureFallbackMessage, + gestureFallbackHint: data.gestureFallbackHint + ) + ) + } + + func plannedGestureResponse( + plan: RunnerGesturePlan, + timing: (gestureStartUptimeMs: Double, gestureEndUptimeMs: Double), + outcome: RunnerInteractionOutcome + ) -> Response { + if let response = unsupportedResponse(for: outcome) { + return response + } + return gestureResponse(message: plan.intent, timing: timing) + } +} diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index b4eb25b33d..002459b19c 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -18,7 +18,6 @@ extension RunnerTests { } enum SynthesizedDragProfile: Equatable { - case continuous case controlledScroll case fastSwipe } @@ -76,7 +75,7 @@ extension RunnerTests { private func performCoordinateBackGesture(app: XCUIApplication) { #if !os(tvOS) - let target = app.windows.firstMatch.exists ? app.windows.firstMatch : app + let target = resolveRunnerWindow(app: app).window ?? app let start = target.coordinate(withNormalizedOffset: CGVector(dx: 0.05, dy: 0.5)) let end = target.coordinate(withNormalizedOffset: CGVector(dx: 0.8, dy: 0.5)) start.press(forDuration: 0.05, thenDragTo: end) @@ -106,7 +105,7 @@ extension RunnerTests { private func performCoordinateAppSwitcherGesture(app: XCUIApplication) { #if !os(tvOS) - let target = app.windows.firstMatch.exists ? app.windows.firstMatch : app + let target = resolveRunnerWindow(app: app).window ?? app let start = target.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.99)) let end = target.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.7)) start.press(forDuration: 0.6, thenDragTo: end) @@ -219,15 +218,35 @@ extension RunnerTests { // transformed subtrees, so a closed drawer at negative x inflates it and // out-of-window coordinates still pass containment. Falls back to app.frame // when no window frame is readable. - func onScreenWindowFrame(app: XCUIApplication) -> CGRect { - let window = app.windows.element(boundBy: 0) - if window.exists { - let frame = window.frame - if !frame.isEmpty { - return frame + /// The app window interactions and captures are booked against: the first window the app + /// reports with a non-empty frame, and no window at all when none qualifies. The viewport, + /// the interaction anchor, and the synthesized reference frame all resolve through here, so a + /// fold, a sheet, or a rotation moves every consumer to the same window in the same pass. + /// Synthesized gestures route their display ID through the returned window so the record's + /// display can never name a different window than the one the reference frame was measured on. + func resolveRunnerWindow(app: XCUIApplication) -> (window: XCUIElement?, frame: CGRect) { + let windows = app.windows.allElementsBoundByIndex + var frames: [CGRect?] = [] + for window in windows { + frames.append(window.exists ? window.frame : nil) + } + if let index = Self.firstUsableWindowIndex(frames: frames) { + return (windows[index], frames[index] ?? .zero) + } + return (nil, app.frame) + } + + static func firstUsableWindowIndex(frames: [CGRect?]) -> Int? { + for (index, frame) in frames.enumerated() { + if let frame, !frame.isEmpty { + return index } } - return app.frame + return nil + } + + func onScreenWindowFrame(app: XCUIApplication) -> CGRect { + resolveRunnerWindow(app: app).frame } func queryElement(app: XCUIApplication, selectorKey: String, selectorValue: String) -> Response { @@ -505,580 +524,6 @@ extension RunnerTests { return performCoordinateDrag(app: app, x: x, y: y, x2: x2, y2: y2, holdDuration: holdDuration) } - func synthesizedDragAt( - app: XCUIApplication, - x: Double, - y: Double, - x2: Double, - y2: Double, - durationMs: Double, - profile: SynthesizedDragProfile = .continuous, - context: SynthesizedCoordinateContext? = nil - ) -> RunnerInteractionOutcome { -#if os(iOS) - guard x.isFinite, y.isFinite, x2.isFinite, y2.isFinite else { - return .unsupported( - message: "synthesized coordinate drag requires finite coordinates", - hint: "Retry with finite x, y, x2, and y2 values." - ) - } - let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)) - guard let context = context ?? synthesizedCoordinateContext( - app: app, - policy: synthesizedGesturePolicy(.synthesizedDrag) - ) else { - return .unsupported( - message: "synthesized coordinate drag could not resolve a finite screen frame", - hint: "Retry after the app is foregrounded, or use a plain screenshot to choose coordinates." - ) - } - let frame = context.referenceFrame - let start = CoordinateSpaceRotation.native( - point: CGPoint(x: x, y: y), - in: frame, - interfaceOrientation: orientation - ) - let end = CoordinateSpaceRotation.native( - point: CGPoint(x: x2, y: y2), - in: frame, - interfaceOrientation: orientation - ) - let message = switch profile { - case .continuous: - RunnerSynthesizedGesture.synthesizeContinuousDrag( - withApplication: app, - x: Double(start.x), - y: Double(start.y), - x2: Double(end.x), - y2: Double(end.y), - durationMs: durationMs - ) - case .controlledScroll: - RunnerSynthesizedGesture.synthesizeControlledScroll( - withApplication: app, - x: Double(start.x), - y: Double(start.y), - x2: Double(end.x), - y2: Double(end.y), - durationMs: durationMs - ) - case .fastSwipe: - RunnerSynthesizedGesture.synthesizeSwipe( - withApplication: app, - x: Double(start.x), - y: Double(start.y), - x2: Double(end.x), - y2: Double(end.y), - durationMs: durationMs - ) - } - if let message { - return .unsupported( - message: message, - hint: "Private XCTest event synthesis is required for AX-free coordinate drag on iOS; update Xcode if this persists." - ) - } - return .performed -#elseif os(tvOS) - return .unsupported( - message: "coordinate drag is not supported on tvOS", - hint: "tvOS has no coordinate input; use remote-driven swipe/scroll to move focus instead." - ) -#else - return .unsupported( - message: "coordinate drag is not supported on macOS", - hint: "macOS automation has no touchscreen; use mouse-driven interactions instead." - ) -#endif - } - - func synthesizedTapAt( - app: XCUIApplication, - x: Double, - y: Double, - context: SynthesizedCoordinateContext? = nil - ) -> RunnerInteractionOutcome { -#if os(iOS) - guard x.isFinite, y.isFinite else { - return .unsupported( - message: "synthesized coordinate tap requires finite coordinates", - hint: "Retry with finite x and y values." - ) - } - let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)) - guard let context = context ?? synthesizedCoordinateContext( - app: app, - policy: synthesizedGesturePolicy(.coordinateTap) - ) else { - return .unsupported( - message: "synthesized coordinate tap could not resolve a finite screen frame", - hint: "Retry after the app is foregrounded, or use a plain screenshot to choose coordinates." - ) - } - let frame = context.referenceFrame - let point = CoordinateSpaceRotation.native( - point: CGPoint(x: x, y: y), - in: frame, - interfaceOrientation: orientation - ) - if let message = RunnerSynthesizedGesture.synthesizeTap( - withApplication: app, - x: Double(point.x), - y: Double(point.y) - ) { - return .unsupported( - message: message, - hint: "Falling back to XCTest coordinate tap may be slower and can still need a healthy accessibility tree." - ) - } - return .performed -#elseif os(tvOS) - return .unsupported( - message: "coordinate tap is not supported on tvOS; move focus with swipe or scroll, then select the focused element", - hint: "tvOS has no coordinate input; move focus with swipe/scroll to the target, then select it." - ) -#else - return .unsupported( - message: "synthesized coordinate tap is not supported on macOS", - hint: "macOS automation has no touchscreen; use mouse-driven interactions instead." - ) -#endif - } - - func keyboardAvoidingDragPoints( - app: XCUIApplication, - x: Double, - y: Double, - x2: Double, - y2: Double - ) -> DragPoints { - let original = DragPoints(x: x, y: y, x2: x2, y2: y2) -#if os(iOS) - guard let keyboardFrame = visibleKeyboardFrame(app: app) else { - return original - } - let minX = min(x, x2) - let minY = min(y, y2) - let gestureBounds = CGRect( - x: CGFloat(minX), - y: CGFloat(minY), - width: CGFloat(max(abs(x2 - x), 1)), - height: CGFloat(max(abs(y2 - y), 1)) - ) - guard gestureBounds.intersects(keyboardFrame) else { - return original - } - - let window = app.windows.firstMatch - let appFrame = window.exists && !window.frame.isEmpty ? window.frame : app.frame - guard !appFrame.isEmpty else { - return original - } - - let padding: Double = 12 - let targetMaxY = Double(keyboardFrame.minY) - padding - let currentMaxY = max(y, y2) - let shift = currentMaxY - targetMaxY - guard shift > 0 else { - return original - } - - let adjustedY = y - shift - let adjustedY2 = y2 - shift - guard min(adjustedY, adjustedY2) >= Double(appFrame.minY) + padding else { - return original - } - - NSLog( - "AGENT_DEVICE_RUNNER_KEYBOARD_AVOIDING_DRAG from=(%.1f,%.1f)->(%.1f,%.1f) adjusted=(%.1f,%.1f)->(%.1f,%.1f) keyboardMinY=%.1f", - x, - y, - x2, - y2, - x, - adjustedY, - x2, - adjustedY2, - Double(keyboardFrame.minY) - ) - return DragPoints(x: x, y: adjustedY, x2: x2, y2: adjustedY2) -#else - return original -#endif - } - - func resolvedTouchVisualizationFrame(app: XCUIApplication, x: Double, y: Double) -> TouchVisualizationFrame { - let appFrame = app.frame - let referenceFrame = resolvedTouchReferenceFrame(app: app, appFrame: appFrame) - let originX = appFrame.isEmpty ? referenceFrame.minX : appFrame.minX - let originY = appFrame.isEmpty ? referenceFrame.minY : appFrame.minY - return TouchVisualizationFrame( - x: originX + x, - y: originY + y, - referenceWidth: referenceFrame.width, - referenceHeight: referenceFrame.height - ) - } - - func resolvedDragVisualizationFrame( - app: XCUIApplication, - x: Double, - y: Double, - x2: Double, - y2: Double - ) -> DragVisualizationFrame { - let start = resolvedTouchVisualizationFrame(app: app, x: x, y: y) - let end = resolvedTouchVisualizationFrame(app: app, x: x2, y: y2) - return DragVisualizationFrame( - x: start.x, - y: start.y, - x2: end.x, - y2: end.y, - referenceWidth: start.referenceWidth, - referenceHeight: start.referenceHeight - ) - } - - func resolvedTouchReferenceFrame(app: XCUIApplication, appFrame: CGRect) -> CGRect { - let window = app.windows.firstMatch - if window.exists { - let windowFrame = window.frame - if !windowFrame.isEmpty { - return frameAvoidingKeyboard(app: app, frame: windowFrame) - } - } - if !appFrame.isEmpty { - return frameAvoidingKeyboard(app: app, frame: appFrame) - } - return CGRect(x: 0, y: 0, width: 0, height: 0) - } - - private func frameAvoidingKeyboard(app: XCUIApplication, frame: CGRect) -> CGRect { -#if os(iOS) - guard let keyboardFrame = visibleKeyboardFrame(app: app), !frame.isEmpty else { - return frame - } - let intersection = frame.intersection(keyboardFrame) - guard !intersection.isNull && intersection.height > 0 else { - return frame - } - let keyboardCoverage = intersection.width / max(frame.width, 1) - guard keyboardCoverage >= 0.5 else { - return frame - } - let safeHeight = keyboardFrame.minY - frame.minY - guard safeHeight >= frame.height * 0.25 else { - return frame - } - return CGRect(x: frame.minX, y: frame.minY, width: frame.width, height: safeHeight) -#else - return frame -#endif - } - - func axFreeSynthesizedDragPlan( - app: XCUIApplication, - x: Double, - y: Double, - x2: Double, - y2: Double, - context: SynthesizedCoordinateContext? = nil - ) -> SynthesizedDragPlan? { -#if os(iOS) - let context = context ?? synthesizedCoordinateContext( - app: app, - policy: synthesizedGesturePolicy(.synthesizedDrag) - ) - guard x.isFinite, y.isFinite, x2.isFinite, y2.isFinite, - let context - else { - return nil - } - let points = keyboardAvoidingSynthesizedDragPoints( - app: app, - x: x, - y: y, - x2: x2, - y2: y2, - context: context - ) - return SynthesizedDragPlan( - points: points, - context: context - ) -#else - return nil -#endif - } - - func axFreeDragVisualizationFrame( - x: Double, - y: Double, - x2: Double, - y2: Double, - referenceFrame: CGRect - ) -> DragVisualizationFrame { - return DragVisualizationFrame( - x: x, - y: y, - x2: x2, - y2: y2, - referenceWidth: Double(referenceFrame.width), - referenceHeight: Double(referenceFrame.height) - ) - } - - func synthesizedCoordinateContext( - app: XCUIApplication, - policy: SynthesizedGesturePolicy - ) -> SynthesizedCoordinateContext? { -#if os(iOS) - let health = runnerAccessibilityHealth - let orientation = Int( - RunnerSynthesizedGesture.interfaceOrientation(forApplication: app) - ) - guard let referenceFrame = orientedSynthesizedScreenshotReferenceFrame( - screenshotSize: XCUIScreen.main.screenshot().image.size, - interfaceOrientation: orientation - ) else { - return nil - } - return SynthesizedCoordinateContext( - referenceFrame: referenceFrame, - keyboardPolicy: policy.keyboardPolicy, - fallbackPolicy: policy.fallbackPolicy, - accessibilityHealth: health - ) -#else - return nil -#endif - } - - func orientedSynthesizedScreenshotReferenceFrame( - screenshotSize: CGSize, - interfaceOrientation: Int - ) -> CGRect? { - // Physical iOS screenshots can retain portrait dimensions after the interface rotates, - // while accessibility frames remain in the logical landscape coordinate space. - guard screenshotSize.width.isFinite, screenshotSize.height.isFinite, - screenshotSize.width > 0, - screenshotSize.height > 0 - else { - return nil - } - let isLandscape = interfaceOrientation == RunnerInterfaceOrientation.landscapeLeft - || interfaceOrientation == RunnerInterfaceOrientation.landscapeRight - let isPortrait = interfaceOrientation == RunnerInterfaceOrientation.portrait - || interfaceOrientation == RunnerInterfaceOrientation.portraitUpsideDown - let width: CGFloat - let height: CGFloat - if isLandscape { - width = max(screenshotSize.width, screenshotSize.height) - height = min(screenshotSize.width, screenshotSize.height) - } else if isPortrait { - width = min(screenshotSize.width, screenshotSize.height) - height = max(screenshotSize.width, screenshotSize.height) - } else { - width = screenshotSize.width - height = screenshotSize.height - } - return CGRect(x: 0, y: 0, width: width, height: height) - } - - func keyboardAvoidingSynthesizedDragPoints( - app: XCUIApplication, - x: Double, - y: Double, - x2: Double, - y2: Double, - context: SynthesizedCoordinateContext - ) -> DragPoints { -#if os(iOS) - guard context.allowsKeyboardProbe else { - return DragPoints(x: x, y: y, x2: x2, y2: y2) - } - return keyboardAvoidingDragPoints(app: app, x: x, y: y, x2: x2, y2: y2) -#else - return DragPoints(x: x, y: y, x2: x2, y2: y2) -#endif - } - - func swipe(app: XCUIApplication, direction: String) -> DragVisualizationFrame? { - if performTvRemoteSwipeIfAvailable(direction: direction) { - let frame = resolvedTouchReferenceFrame(app: app, appFrame: app.frame) - let midX = frame.midX - let midY = frame.midY - return DragVisualizationFrame( - x: midX, - y: midY, - x2: midX, - y2: midY, - referenceWidth: frame.width, - referenceHeight: frame.height - ) - } - return nil - } - - private func performTvRemoteSwipeIfAvailable(direction: String) -> Bool { - switch direction { - case "up": - return pressTvRemote(.up) - case "down": - return pressTvRemote(.down) - case "left": - return pressTvRemote(.left) - case "right": - return pressTvRemote(.right) - default: - return false - } - } - - func plannedGestureValidationError(_ plan: RunnerGesturePlan) -> String? { - guard plan.topology == "single" || plan.topology == "two" else { - return "planned gesture topology must be single or two" - } - let supportedIntent = plan.topology == "single" - ? plan.intent == "fling" || plan.intent == "pan" - : plan.intent == "pan" || plan.intent == "pinch" || plan.intent == "rotate" - || plan.intent == "transform" - guard supportedIntent else { return "planned gesture has unsupported intent for its topology" } - if plan.topology == "single" { - guard plan.executionProfile == "endpoint-hold" || plan.executionProfile == "timed-pan" else { - return "single-pointer gesture requires a supported execution profile" - } - } else if plan.executionProfile != nil { - return "multi-touch gesture cannot define a single-pointer execution profile" - } - guard plan.durationMs.isFinite, plan.durationMs >= 16, plan.durationMs <= 10_000 else { - return "planned gesture durationMs must be between 16 and 10000" - } - let viewport = plan.viewport - guard viewport.x.isFinite, viewport.y.isFinite, viewport.width.isFinite, - viewport.height.isFinite, viewport.width > 0, viewport.height > 0 - else { - return "planned gesture viewport must be finite and positive" - } - let expectedPointerCount = plan.topology == "single" ? 1 : 2 - guard plan.pointers.count == expectedPointerCount else { - return "planned gesture pointer count does not match topology" - } - for (index, pointer) in plan.pointers.enumerated() where pointer.pointerId != index { - return "planned gesture requires ordered pointer ids" - } - let firstSamples = plan.pointers[0].samples - guard firstSamples.count >= 2 else { return "planned pointer paths require at least two samples" } - for pointer in plan.pointers { - guard pointer.samples.count == firstSamples.count else { - return "planned pointer paths require matching samples" - } - var previousOffset = -1.0 - for (index, sample) in pointer.samples.enumerated() { - guard sample.offsetMs.isFinite, - sample.offsetMs == firstSamples[index].offsetMs, - sample.offsetMs > previousOffset - else { - return "planned pointer sample offsets must match and strictly increase" - } - let point = sample.point - guard point.x.isFinite, point.y.isFinite, - point.x >= viewport.x, - point.x <= viewport.x + viewport.width, - point.y >= viewport.y, - point.y <= viewport.y + viewport.height - else { - return "planned pointer sample lies outside the viewport" - } - previousOffset = sample.offsetMs - } - guard pointer.samples.first?.offsetMs == 0, - pointer.samples.last?.offsetMs == plan.durationMs - else { - return "planned pointer paths must start at 0 and end at durationMs" - } - } - if plan.topology == "two" { - guard let firstStart = firstSamples.first?.point, - let secondStart = plan.pointers[1].samples.first?.point, - hypot(firstStart.x - secondStart.x, firstStart.y - secondStart.y) > 0 - else { - return "planned pointer paths require a positive initial span" - } - } - return nil - } - - func plannedGestureExecution(for plan: RunnerGesturePlan) -> PlannedGestureExecution { - plan.topology == "single" && plan.executionProfile == "endpoint-hold" - ? .fastSwipe - : .sampled - } - - func sampledPlannedGesture( - app: XCUIApplication, - plan: RunnerGesturePlan - ) -> RunnerInteractionOutcome { -#if os(iOS) - let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)) - // The portable planner and validation use this exact viewport. Using app.frame here can - // diverge when XCTest unions transformed/off-screen descendants into the application frame. - let frame = CGRect( - x: plan.viewport.x, - y: plan.viewport.y, - width: plan.viewport.width, - height: plan.viewport.height - ) - let pointerSamples: [[[String: NSNumber]]] = plan.pointers.map { pointer in - pointer.samples.map { sample in - let point = CoordinateSpaceRotation.native( - point: CGPoint(x: sample.point.x, y: sample.point.y), - in: frame, - interfaceOrientation: orientation - ) - return [ - "x": NSNumber(value: Double(point.x)), - "y": NSNumber(value: Double(point.y)), - "offsetMs": NSNumber(value: sample.offsetMs), - ] - } - } - if let message = RunnerSynthesizedGesture.synthesizeGesture( - withApplication: app, - pointerSamples: pointerSamples - ) { - return .unsupported( - message: message, - hint: "This gesture uses private XCTest event-synthesis APIs; rebuild the runner with a supported Xcode if this persists." - ) - } - return .performed -#elseif os(tvOS) - return .unsupported( - message: "two-finger gestures are not supported on tvOS", - hint: "tvOS has no touch input; use remote-driven navigation." - ) -#elseif os(visionOS) - return .unsupported( - message: "two-finger touch gestures are not supported on visionOS", - hint: "The current XCTest synthesizer supports iOS and iPadOS touch simulators only." - ) -#else - return .unsupported( - message: "two-finger gestures are not supported on macOS", - hint: "macOS automation has no multi-touch input; run on an iOS simulator." - ) -#endif - } - - private func interactionRoot(app: XCUIApplication) -> XCUIElement { - let windows = app.windows.allElementsBoundByIndex - if let window = windows.first(where: { $0.exists && !$0.frame.isEmpty }) { - return window - } - return app - } - private func performCoordinateTap(app: XCUIApplication, x: Double, y: Double) -> RunnerInteractionOutcome { #if os(tvOS) return .unsupported( @@ -1139,186 +584,22 @@ extension RunnerTests { #if !os(tvOS) private func interactionCoordinate(app: XCUIApplication, x: Double, y: Double) -> XCUICoordinate { #if os(iOS) + // Coordinate taps, double taps, long presses, and drags anchor at the app origin, not the + // resolved window: on iOS the snapshot hands back app-space points, and the first qualifying + // window is not always the one under the finger — a SpringBoard `alert accept` can live in an + // alert window while the wallpaper or status-bar window qualifies first, so a window-relative + // anchor misses the button. Reference frames and the synthesized display ID still resolve + // through `resolveRunnerWindow`, so a foldable tap keeps its panel and stays on its display. let origin = app.coordinate(withNormalizedOffset: CGVector(dx: 0, dy: 0)) return origin.withOffset(CGVector(dx: x, dy: y)) #else - let root = interactionRoot(app: app) + let resolved = resolveRunnerWindow(app: app) + let root = resolved.window ?? app let origin = root.coordinate(withNormalizedOffset: CGVector(dx: 0, dy: 0)) - let rootFrame = root.frame - let offsetX = x - Double(rootFrame.origin.x) - let offsetY = y - Double(rootFrame.origin.y) + let offsetX = x - Double(resolved.frame.origin.x) + let offsetY = y - Double(resolved.frame.origin.y) return origin.withOffset(CGVector(dx: offsetX, dy: offsetY)) #endif } #endif - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS - func testSynthesizedScreenshotReferenceFrameUsesScreenshotSize() throws { - let resolved = try XCTUnwrap( - orientedSynthesizedScreenshotReferenceFrame( - screenshotSize: CGSize(width: 430, height: 932), - interfaceOrientation: RunnerInterfaceOrientation.portrait - ) - ) - - XCTAssertEqual(resolved, CGRect(x: 0, y: 0, width: 430, height: 932)) - } - - func testOrientedSynthesizedScreenshotReferenceFrameUsesLandscapeLogicalDimensions() { - let portraitCapture = CGSize(width: 430, height: 932) - let landscapeCapture = CGSize(width: 932, height: 430) - - for orientation in [ - RunnerInterfaceOrientation.landscapeLeft, - RunnerInterfaceOrientation.landscapeRight, - ] { - XCTAssertEqual( - orientedSynthesizedScreenshotReferenceFrame( - screenshotSize: portraitCapture, - interfaceOrientation: orientation - ), - CGRect(x: 0, y: 0, width: 932, height: 430) - ) - XCTAssertEqual( - orientedSynthesizedScreenshotReferenceFrame( - screenshotSize: landscapeCapture, - interfaceOrientation: orientation - ), - CGRect(x: 0, y: 0, width: 932, height: 430) - ) - } - - for orientation in [ - RunnerInterfaceOrientation.portrait, - RunnerInterfaceOrientation.portraitUpsideDown, - ] { - XCTAssertEqual( - orientedSynthesizedScreenshotReferenceFrame( - screenshotSize: portraitCapture, - interfaceOrientation: orientation - ), - CGRect(x: 0, y: 0, width: 430, height: 932) - ) - XCTAssertEqual( - orientedSynthesizedScreenshotReferenceFrame( - screenshotSize: landscapeCapture, - interfaceOrientation: orientation - ), - CGRect(x: 0, y: 0, width: 430, height: 932) - ) - } - - XCTAssertEqual( - orientedSynthesizedScreenshotReferenceFrame( - screenshotSize: landscapeCapture, - interfaceOrientation: RunnerInterfaceOrientation.unknown - ), - CGRect(x: 0, y: 0, width: 932, height: 430) - ) - } - - func testSynthesizedScreenshotReferenceFrameRejectsInvalidSize() { - XCTAssertNil( - orientedSynthesizedScreenshotReferenceFrame( - screenshotSize: CGSize(width: CGFloat.infinity, height: 932), - interfaceOrientation: RunnerInterfaceOrientation.portrait - ) - ) - } - - func testPlannedMultiTouchGestureAcceptsMatchingInBoundsTrajectories() throws { - let plan = try JSONDecoder().decode( - RunnerGesturePlan.self, - from: Data( - #"{"topology":"two","intent":"pan","durationMs":32,"viewport":{"x":0,"y":0,"width":200,"height":300},"pointers":[{"pointerId":0,"samples":[{"offsetMs":0,"point":{"x":80,"y":80}},{"offsetMs":16,"point":{"x":90,"y":85}},{"offsetMs":32,"point":{"x":100,"y":90}}]},{"pointerId":1,"samples":[{"offsetMs":0,"point":{"x":80,"y":120}},{"offsetMs":16,"point":{"x":90,"y":125}},{"offsetMs":32,"point":{"x":100,"y":130}}]}]}"#.utf8 - ) - ) - - XCTAssertNil(plannedGestureValidationError(plan)) - XCTAssertEqual(plannedGestureExecution(for: plan), .sampled) - } - - func testPlannedMultiTouchGestureRejectsMismatchedOffsets() throws { - let plan = try JSONDecoder().decode( - RunnerGesturePlan.self, - from: Data( - #"{"topology":"two","intent":"transform","durationMs":32,"viewport":{"x":0,"y":0,"width":200,"height":300},"pointers":[{"pointerId":0,"samples":[{"offsetMs":0,"point":{"x":80,"y":80}},{"offsetMs":32,"point":{"x":100,"y":90}}]},{"pointerId":1,"samples":[{"offsetMs":0,"point":{"x":80,"y":120}},{"offsetMs":31,"point":{"x":100,"y":130}}]}]}"#.utf8 - ) - ) - - XCTAssertEqual( - plannedGestureValidationError(plan), - "planned pointer sample offsets must match and strictly increase" - ) - } - - func testSinglePointerFlingUsesFastSwipeExecution() throws { - let plan = try JSONDecoder().decode( - RunnerGesturePlan.self, - from: Data( - #"{"topology":"single","intent":"fling","executionProfile":"endpoint-hold","durationMs":100,"viewport":{"x":0,"y":0,"width":200,"height":300},"pointers":[{"pointerId":0,"samples":[{"offsetMs":0,"point":{"x":160,"y":150}},{"offsetMs":100,"point":{"x":40,"y":150}}]}]}"#.utf8 - ) - ) - - XCTAssertEqual(plannedGestureExecution(for: plan), .fastSwipe) - } - - func testSinglePointerTimedPanUsesSampledExecution() throws { - let plan = try JSONDecoder().decode( - RunnerGesturePlan.self, - from: Data( - #"{"topology":"single","intent":"pan","executionProfile":"timed-pan","durationMs":500,"viewport":{"x":0,"y":0,"width":200,"height":300},"pointers":[{"pointerId":0,"samples":[{"offsetMs":0,"point":{"x":160,"y":150}},{"offsetMs":250,"point":{"x":100,"y":150}},{"offsetMs":500,"point":{"x":40,"y":150}}]}]}"#.utf8 - ) - ) - - XCTAssertEqual(plannedGestureExecution(for: plan), .sampled) - } - - func testSinglePointerEndpointHoldUsesFastSwipeExecution() throws { - let plan = try JSONDecoder().decode( - RunnerGesturePlan.self, - from: Data( - #"{"topology":"single","intent":"pan","executionProfile":"endpoint-hold","durationMs":500,"viewport":{"x":0,"y":0,"width":200,"height":300},"pointers":[{"pointerId":0,"samples":[{"offsetMs":0,"point":{"x":160,"y":150}},{"offsetMs":500,"point":{"x":40,"y":150}}]}]}"#.utf8 - ) - ) - - XCTAssertNil(plannedGestureValidationError(plan)) - XCTAssertEqual(plannedGestureExecution(for: plan), .fastSwipe) - } - - func testSinglePointerGestureRejectsMissingExecutionProfile() throws { - let plan = try JSONDecoder().decode( - RunnerGesturePlan.self, - from: Data( - #"{"topology":"single","intent":"pan","durationMs":500,"viewport":{"x":0,"y":0,"width":200,"height":300},"pointers":[{"pointerId":0,"samples":[{"offsetMs":0,"point":{"x":160,"y":150}},{"offsetMs":500,"point":{"x":40,"y":150}}]}]}"#.utf8 - ) - ) - - XCTAssertEqual( - plannedGestureValidationError(plan), - "single-pointer gesture requires a supported execution profile" - ) - } - - func testDesktopScrollWheelDeltasMapDirections() { - XCTAssertEqual(desktopScrollWheelDeltas(direction: .up, pixels: 120).vertical, 120) - XCTAssertEqual(desktopScrollWheelDeltas(direction: .down, pixels: 120).vertical, -120) - XCTAssertEqual(desktopScrollWheelDeltas(direction: .left, pixels: 120).horizontal, 120) - XCTAssertEqual(desktopScrollWheelDeltas(direction: .right, pixels: 120).horizontal, -120) - } - - func testDesktopScrollWheelDeltaEventsHonorDurationAndPreservePixels() { - let events = desktopScrollWheelDeltaEvents(direction: .down, pixels: 200, durationMs: 50) - XCTAssertEqual(events.count, 4) - XCTAssertEqual(events.map(\.vertical).reduce(0, +), -200) - XCTAssertEqual(events.map(\.horizontal).reduce(0, +), 0) - XCTAssertEqual(desktopScrollEventIntervalSeconds(durationMs: 50, eventCount: events.count), 0.05 / 3.0) - } - - func testDesktopScrollWheelDeltaEventsKeepInstantScrollSingleEvent() { - let events = desktopScrollWheelDeltaEvents(direction: .down, pixels: 200, durationMs: 0) - XCTAssertEqual(events.count, 1) - XCTAssertEqual(events.first?.vertical, -200) - } -#endif } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Keyboard.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Keyboard.swift index de384d2ee6..d743ec2a7c 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Keyboard.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Keyboard.swift @@ -212,6 +212,7 @@ extension RunnerTests { if tapKeyboardDismissControl(app: app) { _ = keyboard.waitForNonExistence(timeout: KeyboardDismissObservationTiming.timeout) waitForScreenshotStability( + app: app, timeout: KeyboardDismissObservationTiming.settleTimeout, sampleInterval: KeyboardDismissObservationTiming.settleSampleInterval, requiredConsecutiveMatches: KeyboardDismissObservationTiming.settleRequiredConsecutiveMatches @@ -240,23 +241,38 @@ extension RunnerTests { // so far, so it is unit-testable without a real screenshot pipeline; this loop // is the thin, untestable I/O shell around it. private func waitForScreenshotStability( + app: XCUIApplication, timeout: TimeInterval, sampleInterval: TimeInterval, requiredConsecutiveMatches: Int ) { let deadline = Date().addingTimeInterval(timeout) - var samples: [Data?] = [screenshotFingerprintForStabilityCheck()] + var samples: [Data?] = [screenshotFingerprintForStabilityCheck(app: app)] while Date() < deadline { sleepFor(sampleInterval) - samples.append(screenshotFingerprintForStabilityCheck()) + samples.append(screenshotFingerprintForStabilityCheck(app: app)) if runnerScreenshotStabilitySettled(samples, requiredConsecutiveMatches: requiredConsecutiveMatches) { + logScreenshotStability(samples: samples, settled: true) return } } + logScreenshotStability(samples: samples, settled: false) + } + + /// Says what the settle actually looked at: samples that are all `nil` means the capture refused to + /// produce a frame, which is a different fact from a screen that never held still, and the two read + /// identically from the outside (#2728). + private func logScreenshotStability(samples: [Data?], settled: Bool) { + NSLog( + "AGENT_DEVICE_RUNNER_KEYBOARD_STABILITY samples=%ld captured=%ld settled=%@", + samples.count, + samples.compactMap { $0 }.count, + settled ? "yes" : "no" + ) } - private func screenshotFingerprintForStabilityCheck() -> Data? { - guard let image = captureRunnerFrame() else { return nil } + private func screenshotFingerprintForStabilityCheck(app: XCUIApplication) -> Data? { + guard let image = captureRunnerFrame(app: app) else { return nil } return runnerPngData(for: image) } @@ -431,145 +447,3 @@ func runnerScreenshotStabilitySettled( guard let first = window.first, let firstData = first else { return false } return window.allSatisfy { $0 == firstData } } - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS -extension RunnerTests { - func testKeyboardBandFactReadFailureIsStatedAsUnmeasurableNotAbsence() { - // Absence would let a consumer claim the screen is clear of a keyboard on the strength of a read - // that never answered. - let fact = runnerKeyboardBandFact( - readSucceeded: false, - exists: false, - frame: CGRect(x: 0, y: 198, width: 402, height: 204) - ) - XCTAssertEqual(fact, .unmeasurable(RunnerKeyboardBandReason.queryFailed)) - XCTAssertEqual(fact.payload.kind, "unmeasurable") - XCTAssertEqual(fact.payload.reason, RunnerKeyboardBandReason.queryFailed) - XCTAssertNil(fact.payload.frame) - } - - func testKeyboardBandFactPublishesAbsenceWhenTheQueryFindsNoKeyboard() { - let fact = runnerKeyboardBandFact(readSucceeded: true, exists: false, frame: .zero) - XCTAssertEqual(fact, .absent) - XCTAssertEqual(fact.payload.kind, "absent") - XCTAssertNil(fact.payload.frame) - XCTAssertNil(fact.payload.reason) - } - - func testKeyboardBandFactPublishesTheMeasuredBandInAppOrientationSpace() { - // The landscape band measured on iPhone 17 Pro (iOS 26.2) after #2653: full width across the - // bottom of a 402 pt-tall app, which is what the tree reported as a strip down the left edge. - let frame = CGRect(x: 0, y: 198, width: 874, height: 204) - let fact = runnerKeyboardBandFact(readSucceeded: true, exists: true, frame: frame) - XCTAssertEqual(fact, .visible(frame)) - let payload = fact.payload - XCTAssertEqual(payload.kind, "visible") - XCTAssertEqual(payload.frame, SnapshotRect(x: 0, y: 198, width: 874, height: 204)) - XCTAssertNil(payload.reason) - } - - func testKeyboardBandFactRefusesUnusableGeometryInsteadOfClaimingAbsence() { - let nan = CGFloat(Double.nan) - let infinite = CGFloat.infinity - let unusable: [CGRect] = [ - .zero, - CGRect(x: 0, y: 198, width: 0, height: 204), - CGRect(x: 0, y: 198, width: 874, height: -1), - CGRect(x: 0, y: 198, width: -874, height: 204), - CGRect(x: 0, y: nan, width: 874, height: 204), - CGRect(x: 0, y: 198, width: nan, height: 204), - CGRect(x: infinite, y: 198, width: 874, height: 204) - ] - for frame in unusable { - let fact = runnerKeyboardBandFact(readSucceeded: true, exists: true, frame: frame) - XCTAssertEqual( - fact, - .unmeasurable(RunnerKeyboardBandReason.unusableFrame), - "expected \(frame) to be refused as a band" - ) - } - } - - func testKeyboardBandFactPayloadRoundTripsThroughTheWireShape() throws { - let cases: [RunnerKeyboardBandFact] = [ - .visible(CGRect(x: 0, y: 583, width: 402, height: 291)), - .absent, - .unmeasurable(RunnerKeyboardBandReason.queryTimeout) - ] - for fact in cases { - let data = try JSONEncoder().encode(fact.payload) - // `encodeIfPresent` for the two optional fields: a fact carries its own evidence and nothing - // else, so the daemon never has to distinguish a null from an absent key. - let object = try XCTUnwrap(JSONSerialization.jsonObject(with: data) as? [String: Any]) - switch fact { - case .visible: - XCTAssertNil(object["reason"]) - XCTAssertNotNil(object["frame"]) - case .absent: - XCTAssertNil(object["frame"]) - XCTAssertNil(object["reason"]) - case .unmeasurable: - XCTAssertNil(object["frame"]) - XCTAssertNotNil(object["reason"]) - } - XCTAssertEqual(try JSONDecoder().decode(KeyboardBandFactPayload.self, from: data), fact.payload) - } - } - - func testRunnerScreenshotStabilitySettledNeedsEnoughSamples() { - XCTAssertFalse(runnerScreenshotStabilitySettled([], requiredConsecutiveMatches: 3)) - XCTAssertFalse(runnerScreenshotStabilitySettled([Data([1])], requiredConsecutiveMatches: 3)) - let frame = Data([1, 2, 3]) - XCTAssertFalse( - runnerScreenshotStabilitySettled([frame, frame], requiredConsecutiveMatches: 3) - ) - } - - func testRunnerScreenshotStabilitySettledTrueWhenWindowMatches() { - let frame = Data([1, 2, 3]) - XCTAssertTrue( - runnerScreenshotStabilitySettled([Data([9]), frame, frame, frame], requiredConsecutiveMatches: 3) - ) - } - - func testRunnerScreenshotStabilitySettledFalseOnMidWindowMismatch() { - // A momentary pause (two matching samples) followed by resumed movement - // must not read as settled: the 3-sample window still spans the mismatch. - let frame = Data([1, 2, 3]) - let moved = Data([4, 5, 6]) - XCTAssertFalse( - runnerScreenshotStabilitySettled([frame, frame, moved], requiredConsecutiveMatches: 3) - ) - } - - func testRunnerScreenshotStabilitySettledFalseOnFailedCapture() { - // A nil sample (failed screenshot) never counts as a match, even against - // other nils — an unverifiable run must not look "stable". - XCTAssertFalse(runnerScreenshotStabilitySettled([nil, nil, nil], requiredConsecutiveMatches: 3)) - let frame = Data([1, 2, 3]) - XCTAssertFalse( - runnerScreenshotStabilitySettled([frame, frame, nil], requiredConsecutiveMatches: 3) - ) - } - - func testRunnerScreenshotStabilitySettledOnlyLooksAtTheTrailingWindow() { - // An older mismatch before the trailing window must not block settlement - // once the required run of most-recent samples agrees. - let frame = Data([9]) - XCTAssertTrue( - runnerScreenshotStabilitySettled( - [Data([1]), Data([2]), frame, frame, frame], - requiredConsecutiveMatches: 3 - ) - ) - } - - func testRunnerScreenshotStabilitySettledRejectsDegenerateRequirement() { - // Fewer than 2 required matches would make any single sample "settled" — - // guard against a misconfigured caller rather than silently no-op the wait. - XCTAssertFalse( - runnerScreenshotStabilitySettled([Data([1])], requiredConsecutiveMatches: 1) - ) - } -} -#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift index 34fe74d579..304a5b1829 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift @@ -42,18 +42,60 @@ enum RunnerInteractionIdleWaits { extension RunnerTests { // MARK: - Recording - func captureRunnerFrame() -> RunnerImage? { - var image: RunnerImage? + /// One frame for a caller that tolerates a dropped one — keyboard settling, which skips a sample it + /// cannot take and keeps polling. A frame that must exist goes through `captureRunnerFrameResult`, + /// which says why it refused. + /// + /// On iOS the frame comes from the display owning a window, because a foldable's + /// `XCUIScreen.main` can be the dark outer panel while the app runs on the inner one — a stream of + /// identical black frames would then read as a settled screen (#2728). An observation with no + /// session window falls to the system surface's window, which is what the home screen is. macOS + /// keeps the host display it always recorded. + func captureRunnerFrame(app: XCUIApplication) -> RunnerImage? { + switch captureRunnerFrameResult(app: app) { + case .success(let captured): + return captured.image + case .failure: + return nil + } + } + + /// The same frame as `captureRunnerFrame`, but carrying the reason it refused, so a required first + /// frame — a recording's bootstrap, which sizes the whole writer from it — fails closed with a + /// typed code rather than a message. The ongoing pump reads the same result and ignores a refusal + /// the way it ignored the `nil` it used to get; only a frame that must exist owes a reason (#2728). + func captureRunnerFrameResult( + app: XCUIApplication + ) -> Result { +#if os(iOS) + return captureObservedScreen(app: app) +#else + var outcome: Result = .failure( + .unrenderableImage + ) let capture = { - let screenshot = XCUIScreen.main.screenshot() - image = screenshot.image + let image = XCUIScreen.main.screenshot().image + if let cgImage = runnerCGImage(from: image) { + // The host display has no resolved-panel facts to report; the recorder reads only the image + // and its pixel size, so these two are inert placeholders, not measurements the host scales by. + outcome = .success( + CapturedAppScreen( + image: image, + displayID: 0, + pixelWidth: cgImage.width, + pixelHeight: cgImage.height, + pixelsPerPoint: 1 + ) + ) + } } if Thread.isMainThread { capture() } else { DispatchQueue.main.sync(execute: capture) } - return image + return outcome +#endif } func screenshotRoot(app: XCUIApplication) -> XCUIElement { @@ -66,6 +108,54 @@ extension RunnerTests { return app } + /// Answers a `screenshot` command with one encoded image: inline when the caller asked for bytes, + /// otherwise as a path the host reads out of the runner's own container. `metadata` carries the + /// display the image came from whenever the capture resolved one, so the host never has to guess + /// the density of a panel it did not measure (#2728). + func screenshotResponse( + pngData: Data, + inlineScreenshot: Bool, + metadata: ScreenshotMetadataPayload? = nil + ) -> Response { + if inlineScreenshot { + return Response( + ok: true, + data: DataPayload(imageBase64: pngData.base64EncodedString(), screenshotMetadata: metadata) + ) + } + let fileName = "screenshot-\(Int(Date().timeIntervalSince1970 * 1000)).png" + let filePath = (NSTemporaryDirectory() as NSString).appendingPathComponent(fileName) + do { + try pngData.write(to: URL(fileURLWithPath: filePath)) + } catch { + return Response( + ok: false, + error: ErrorPayload(message: "Failed to write screenshot: \(error.localizedDescription)") + ) + } +#if os(macOS) + return Response(ok: true, data: DataPayload(message: filePath, screenshotMetadata: metadata)) +#else + // Return path relative to app container root (tmp/ maps to NSTemporaryDirectory) + return Response( + ok: true, + data: DataPayload(message: "tmp/\(fileName)", screenshotMetadata: metadata) + ) +#endif + } + + /// Encodes a captured image as PNG and answers with it, or with the failure to encode it. + func screenshotResponse( + image: RunnerImage, + inlineScreenshot: Bool, + metadata: ScreenshotMetadataPayload? = nil + ) -> Response { + guard let pngData = runnerPngData(for: image) else { + return Response(ok: false, error: ErrorPayload(message: "Failed to encode screenshot as PNG")) + } + return screenshotResponse(pngData: pngData, inlineScreenshot: inlineScreenshot, metadata: metadata) + } + func stopRecordingIfNeeded() { guard let recorder = activeRecording else { return } do { @@ -104,8 +194,17 @@ extension RunnerTests { currentApp = app currentBundleId = nil currentAppProcessIdentifier = nil + resetTargetBoundState() + } + + /// State that belongs to the currently bound target and must not outlive it: the text-entry tap + /// witness, the fresh-process snapshot warmup exemption, and the last-written per-command log + /// markers. Every site that binds, rebinds, or drops the target runs this. + func resetTargetBoundState() { clearRememberedTextEntryTap() - snapshotXCTestPenaltyWarmupExemptionPending = false + snapshotXCTestPenaltyWarmupExemption.isPending = false + lastLoggedFastAppGuardLine = nil + lastLoggedGesturePolicyLines.removeAll() } func invalidateCachedTarget(reason: String) { @@ -115,8 +214,7 @@ extension RunnerTests { currentApp = nil currentBundleId = nil currentAppProcessIdentifier = nil - clearRememberedTextEntryTap() - snapshotXCTestPenaltyWarmupExemptionPending = false + resetTargetBoundState() } func resetTargetAfterExternalRelaunch() -> Response { @@ -146,10 +244,10 @@ extension RunnerTests { ) currentApp = candidate currentAppProcessIdentifier = observedProcessIdentifier - clearRememberedTextEntryTap() + resetTargetBoundState() clearSnapshotXCTestChannelPenalty(reason: "target_process_changed") clearPrivateAXAcceptedDepth(reason: "target_process_changed") - snapshotXCTestPenaltyWarmupExemptionPending = true + snapshotXCTestPenaltyWarmupExemption.isPending = true beginFirstInteractionStabilization() } @@ -184,22 +282,26 @@ extension RunnerTests { func canUseFastForegroundAppGuard( activeApp: XCUIApplication, - requestedBundleId: String?, - command: CommandType + requestedBundleId: String? ) -> Bool { guard let requestedBundleId, currentBundleId == requestedBundleId, currentApp != nil else { return false } guard activeApp.state == .runningForeground else { return false } - NSLog( - "AGENT_DEVICE_RUNNER_FAST_APP_GUARD command=%@ bundle=%@ state=%d", - String(describing: command), - requestedBundleId, - activeApp.state.rawValue - ) + writeFastAppGuardMarker(bundleId: requestedBundleId, state: activeApp.state) return true } + func writeFastAppGuardMarker(bundleId: String, state: XCUIApplication.State) { + // The command is on the adjacent COMMAND_ACCEPTED line; repeating it here would make a deduped + // marker read as if only that command ever passed the guard. + let line = "AGENT_DEVICE_RUNNER_FAST_APP_GUARD bundle=\(bundleId) state=\(state.rawValue)" + if lastLoggedFastAppGuardLine != line { + lastLoggedFastAppGuardLine = line + runnerMarkerWriter(line) + } + } + /// The pid of the one other application holding an active accessibility session, or nil unless /// exactly one exists. What this proves is that liveness claim and nothing more: the private AX /// client exposes no ordering of `activeApplications`, so this is NOT a foreground owner — it is @@ -214,6 +316,31 @@ extension RunnerTests { return foreign.count == 1 ? foreign.first : nil } + /// `activate()` on a not-running app is a bare launch, which would drop the URL of a launch + /// SpringBoard still holds behind its "Open in …?" confirmation; see `APP_NOT_RUNNING_RUNNER_CODE`. + func notRunningReadResponse(command: Command, bundleId: String) -> Response? { +#if os(iOS) + guard isReadOnlyCommand(command), + XCUIApplication(bundleIdentifier: bundleId).state == .notRunning + else { return nil } + NSLog( + "AGENT_DEVICE_RUNNER_READ_TARGET_NOT_RUNNING bundle=%@ command=%@", + bundleId, + command.command.rawValue + ) + return Response( + ok: false, + error: ErrorPayload( + code: RunnerWireErrorCode.appNotRunning, + message: "app '\(bundleId)' is not running", + hint: "Reads do not launch the app. Relaunch it with open; if a system prompt such as a deep-link confirmation holds its launch, answer it with alert accept." + ) + ) +#else + return nil +#endif + } + func activateTarget(bundleId: String, reason: String) -> XCUIApplication { let target = XCUIApplication(bundleIdentifier: bundleId) let initialState = target.state @@ -250,8 +377,7 @@ extension RunnerTests { currentApp = target currentBundleId = bundleId currentAppProcessIdentifier = Self.processIdentifier(of: target) - clearRememberedTextEntryTap() - snapshotXCTestPenaltyWarmupExemptionPending = false + resetTargetBoundState() beginFirstInteractionStabilization() return target } @@ -325,25 +451,13 @@ extension RunnerTests { } func shouldRetryCommand(_ command: Command) -> Bool { - if RunnerEnv.isTruthy("AGENT_DEVICE_RUNNER_DISABLE_READONLY_RETRY") { - return false - } - return isReadOnlyCommand(command) + isReadOnlyCommand(command) } func shouldRetryException(_ command: Command, message: String) -> Bool { guard shouldRetryCommand(command) else { return false } - let normalized = message.lowercased() - if normalized.contains("kaxerrorservernotfound") { - return true - } - if normalized.contains("main thread execution timed out") { - return true - } - if normalized.contains("timed out") && command.command == .snapshot { - return true - } - return false + // XCTest raises this AX error as an ObjC exception whose reason is the only handle on it. + return message.lowercased().contains("kaxerrorservernotfound") } // MARK: - Command Classification @@ -362,8 +476,7 @@ extension RunnerTests { func shouldRetryResponse(_ response: Response) -> Bool { guard response.ok == false else { return false } - guard let message = response.error?.message.lowercased() else { return false } - return message.contains("is not available") + return response.error?.retryableFailure != nil } func isInteractionCommand(_ command: CommandType) -> Bool { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+MainThreadWork.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+MainThreadWork.swift index c0dafe8f05..267f2b0b61 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+MainThreadWork.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+MainThreadWork.swift @@ -12,7 +12,9 @@ import XCTest extension RunnerTests { /// Tracks one main-queue dispatch so the watchdog and the dispatched block can agree, under /// `mainThreadWorkLock`, on exactly one of: finished in time, or abandoned. - private final class MainThreadWorkState { + private final class MainThreadWorkState { + let completed = DispatchSemaphore(value: 0) + var result: Result? var finished = false var abandoned = false } @@ -62,24 +64,71 @@ extension RunnerTests { if Thread.isMainThread { return try work() } - var result: Result? - let semaphore = DispatchSemaphore(value: 0) - let workState = MainThreadWorkState() + mainThreadWorkLock.lock() + let state = enqueueMainThreadWorkLocked(operation, work) + mainThreadWorkLock.unlock() + return try awaitMainThreadWork( + state, + operation: operation, + timeout: timeout, + timeoutError: timeoutError, + onAbandoned: onAbandoned + ) + } + + /// Runs optional `work` like `runMainThreadWork`, but only while no other dispatched main-thread + /// work is in flight; otherwise it returns `nil` without dispatching. The check and the enqueue + /// happen under one hold of `mainThreadWorkLock`, the same lock every dispatch enqueues under, so + /// admitted work never waits in the main queue behind a command's hop. The in-flight count covers + /// abandoned work too: a block stays counted until it returns, and it marks itself abandoned in + /// the same window, so occupancy that outlived its slice is already declined here. + func runMainThreadWorkIfIdle( + _ operation: String, + timeout: TimeInterval, + timeoutError: @escaping () -> Error, + _ work: @escaping () throws -> T + ) throws -> T? { + if Thread.isMainThread { + return nil + } + mainThreadWorkLock.lock() + guard mainThreadWorkInFlightCount == 0 else { + mainThreadWorkLock.unlock() + return nil + } + let state = enqueueMainThreadWorkLocked(operation, work) + mainThreadWorkLock.unlock() + return try awaitMainThreadWork( + state, + operation: operation, + timeout: timeout, + timeoutError: timeoutError, + onAbandoned: nil + ) + } + + private func enqueueMainThreadWorkLocked( + _ operation: String, + _ work: @escaping () throws -> T + ) -> MainThreadWorkState { + let state = MainThreadWorkState() + mainThreadWorkInFlightCount += 1 DispatchQueue.main.async { do { - result = .success(try work()) + state.result = .success(try work()) } catch { - result = .failure(error) + state.result = .failure(error) } self.mainThreadWorkLock.lock() - let abandoned = workState.abandoned + self.mainThreadWorkInFlightCount -= 1 + let abandoned = state.abandoned if abandoned { self.abandonedMainThreadWorkCount -= 1 if self.abandonedMainThreadWorkCount == 0 { self.abandonedMainThreadWorkSince = nil } } else { - workState.finished = true + state.finished = true } let allDrained = abandoned && self.abandonedMainThreadWorkCount == 0 self.mainThreadWorkLock.unlock() @@ -89,14 +138,29 @@ extension RunnerTests { NSLog("AGENT_DEVICE_RUNNER_ABANDONED_WORK_DRAINED") } } - semaphore.signal() + state.completed.signal() } - let waitResult = semaphore.wait(timeout: .now() + timeout) + return state + } + + private func awaitMainThreadWork( + _ state: MainThreadWorkState, + operation: String, + timeout: TimeInterval, + timeoutError: @escaping () -> Error, + onAbandoned: (() -> Void)? + ) throws -> T { + let waitResult = state.completed.wait(timeout: .now() + timeout) if waitResult == .timedOut { + #if AGENT_DEVICE_RUNNER_UNIT_TESTS + mainThreadWorkTimedOutForTesting?() + #endif + // Work that finished before the lock was taken already stored its result: it is answered + // like work that finished in time, so an action that happened is never reported as a timeout. mainThreadWorkLock.lock() - let abandoned = !workState.finished + let abandoned = !state.finished if abandoned { - workState.abandoned = true + state.abandoned = true abandonedMainThreadWorkCount += 1 if abandonedMainThreadWorkSince == nil { abandonedMainThreadWorkSince = Date() @@ -110,10 +174,10 @@ extension RunnerTests { timeout ) onAbandoned?() + throw timeoutError() } - throw timeoutError() } - switch result { + switch state.result { case .success(let value): return value case .failure(let error): @@ -135,95 +199,3 @@ extension RunnerTests { ) } } - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS -extension RunnerTests { - func testRunMainThreadWorkExecutesOffMainCallerOnMainThread() { - final class ResultBox { - var observedMainThread: Bool? - var error: Error? - } - let box = ResultBox() - let finished = expectation(description: "off-main caller finished") - - DispatchQueue(label: "agent-device.runner.tests.off-main").async { - do { - box.observedMainThread = try self.runMainThreadWork( - "command_execution", - timeout: 1, - timeoutError: self.mainThreadExecutionTimeoutError - ) { - Thread.isMainThread - } - } catch { - box.error = error - } - finished.fulfill() - } - - wait(for: [finished], timeout: 2) - XCTAssertNil(box.error) - XCTAssertEqual(box.observedMainThread, true) - } - - func testRunMainThreadWorkTimeoutMarksAbandonedUntilDrained() { - final class ResultBox { - var error: Error? - var abandonedCount: Int? - var abandonedSinceSet: Bool? - var busyWhileAbandoned = false - } - let box = ResultBox() - let releaseWork = DispatchSemaphore(value: 0) - let observedAbandoned = DispatchSemaphore(value: 0) - let timedOut = expectation(description: "off-main caller timed out") - - DispatchQueue(label: "agent-device.runner.tests.timeout").async { - do { - _ = try self.runMainThreadWork( - "command_execution", - timeout: 0, - timeoutError: self.mainThreadExecutionTimeoutError - ) { - _ = releaseWork.wait(timeout: .now() + 2) - return true - } - } catch { - box.error = error - } - self.mainThreadWorkLock.lock() - box.abandonedCount = self.abandonedMainThreadWorkCount - box.abandonedSinceSet = self.abandonedMainThreadWorkSince != nil - self.mainThreadWorkLock.unlock() - if case .busy = self.currentMainThreadBusyState() { - box.busyWhileAbandoned = true - } - observedAbandoned.signal() - timedOut.fulfill() - } - DispatchQueue(label: "agent-device.runner.tests.release-timeout").async { - _ = observedAbandoned.wait(timeout: .now() + 2) - releaseWork.signal() - } - - wait(for: [timedOut], timeout: 3) - let drainDeadline = Date().addingTimeInterval(2) - while hasAbandonedMainThreadWork(), Date() < drainDeadline { - sleepFor(0.005) - } - - XCTAssertEqual((box.error as NSError?)?.code, RunnerErrorCode.mainThreadExecutionTimedOut) - XCTAssertEqual(box.abandonedCount, 1) - XCTAssertEqual(box.abandonedSinceSet, true) - XCTAssertTrue(box.busyWhileAbandoned) - XCTAssertFalse(hasAbandonedMainThreadWork(), "drained work must release the main thread") - mainThreadWorkLock.lock() - let sinceCleared = abandonedMainThreadWorkSince == nil - mainThreadWorkLock.unlock() - XCTAssertTrue(sinceCleared) - guard case .idle = currentMainThreadBusyState() else { - return XCTFail("expected the runner idle once the abandoned work drained") - } - } -} -#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift index 4e5c1974a7..d33303a79f 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift @@ -2,7 +2,7 @@ import AgentDeviceSnapshotPresentation // MARK: - Wire Models -enum CommandType: String, Codable { +enum CommandType: String, Codable, CaseIterable { case tap case mouseClick case longPress @@ -17,7 +17,6 @@ enum CommandType: String, Codable { case readText case snapshot case screenshot - case back case backInApp case backSystem case home @@ -78,7 +77,7 @@ extension CommandType { // classification as .drag. .desktopScroll is the macOS frame-resolve + wheel event sibling. // .sequence is the fused multi-step gesture batch. case .tap, .longPress, .drag, .remotePress, .type, .swipe, .scroll, .desktopScroll, - .back, .backInApp, .backSystem, .rotate, .appSwitcher, + .backInApp, .backSystem, .rotate, .appSwitcher, .keyboardDismiss, .keyboardReturn, .sequence, .gesture: return CommandTraits(isInteraction: true, readOnly: .never, isLifecycle: false) @@ -260,6 +259,16 @@ extension Response { } } +/// The display one runner screenshot came from, as the capture measured it rather than as the host +/// could guess. A runner capture can come from a different panel than the one the host resolved, so +/// density normalization has to read the scale of the image that was actually taken (#2728). +struct ScreenshotMetadataPayload: Codable { + let displayID: UInt + let pixelWidth: Int + let pixelHeight: Int + let pixelsPerPoint: Double +} + /// Foreground repair the runner performed while serving one command (#2682). `priorState` is the /// bound app's `XCApplicationState` raw value read BEFORE `XCUIApplication.activate()` ran, so the /// fact describes what was repaired rather than what the repair produced. `otherActiveApplicationPid` @@ -330,6 +339,9 @@ struct DataPayload: Codable { var failedStepIndex: Int? var sequenceResults: [SequenceStepResult]? var targetActivation: TargetActivationFactPayload? + /// Present on a screenshot the runner captured from a display it resolved, alongside the + /// `message` path or `imageBase64` payload that carries the image itself (#2728). + var screenshotMetadata: ScreenshotMetadataPayload? } /// `kind` mirrors the TS `SnapshotKeyboardBandFact`: "visible" carries `frame`, "unmeasurable" @@ -373,8 +385,30 @@ struct SnapshotQualityPayload: Codable { } } +/// A runner failure the read-only retry may recover from by re-resolving the target. +enum RetryableResponseFailure: Equatable { + case targetAppUnavailable +} + struct ErrorPayload: Codable { var code: String? let message: String var hint: String? + /// Runner-internal: read by `shouldRetryResponse` and never encoded, so the host's decoding of + /// the error is unchanged. + var retryableFailure: RetryableResponseFailure? = nil + + private enum CodingKeys: String, CodingKey { + case code + case message + case hint + } + + static func targetAppUnavailable(bundleId: String?) -> ErrorPayload { + let subject = bundleId.map { "app '\($0)'" } ?? "runner app" + return ErrorPayload( + message: "\(subject) is not available", + retryableFailure: .targetAppUnavailable + ) + } } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift index 7e4fcc3952..3ab477ff74 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift @@ -1,30 +1,70 @@ +import AgentDeviceSnapshotPresentation import XCTest extension RunnerTests { static let navigationBackKeywords = ["back", "close", "cancel"] static let navigationFallbackVerificationDelay: TimeInterval = 0.25 - func tapInAppBackControl(app: XCUIApplication) -> Bool { + /// What one in-app `back` attempt concluded. `.unverified` carries the typed capture failure the + /// display refused with, so an optional visual check that could not run reports an unknown instead + /// of the false "no back control exists" that a `nil` sample would otherwise become (#2728). + enum InAppBackOutcome { + case performed + case unavailable + case unverified(ErrorPayload) + } + + /// The three answers a before/after visual comparison can give. `unobserved` is not evidence of + /// "no change": a display that refuses to be sampled proves nothing either way (#2728). + enum NavigationVisualObservation: Equatable { + case changed + case unchanged + case unobserved + + var logToken: String { + switch self { + case .changed: return "yes" + case .unchanged: return "no" + case .unobserved: return "unknown" + } + } + } + + /// One navigation-fallback capture: the encoded frame when the display answered, and the typed + /// reason it refused otherwise. Only iOS produces a refusal; other platforms capture nothing here. + struct NavigationVisualSample { + let data: Data? + let refusalCode: String? + let refusalHint: String? + + init(data: Data?, refusalCode: String? = nil, refusalHint: String? = nil) { + self.data = data + self.refusalCode = refusalCode + self.refusalHint = refusalHint + } + } + + func tapInAppBackControl(app: XCUIApplication) -> InAppBackOutcome { #if os(macOS) if let back = macOSNavigationBackElement(app: app) { tapElementCenter(app: app, element: back) - return true + return .performed } - return false + return .unavailable #elseif os(tvOS) _ = pressTvRemote(.menu) - return true + return .performed #else let buttons = app.navigationBars.buttons.allElementsBoundByIndex if let back = buttons.first(where: { $0.isHittable }) { back.tap() - return true + return .performed } if isSnapshotXCTestChannelPenalized(bundleId: currentBundleId) { NSLog("AGENT_DEVICE_RUNNER_IN_APP_BACK_SKIPPED_XCTEST_ENUMERATION bundle=%@", currentBundleId ?? "") } else if let back = topNavigationBackElement(app: app) { tapElementCenter(app: app, element: back) - return true + return .performed } return tapTopLeadingNavigationFallback(app: app) #endif @@ -81,16 +121,9 @@ extension RunnerTests { return navigationBackKeywords.firstIndex { text.contains($0) } } - // isFinite/>0 alone don't reject CGRect.infinite — its origin (~-9e307) is finite. - static func isUsableNavigationFrame(_ frame: CGRect) -> Bool { - guard frame.width.isFinite, frame.height.isFinite, frame.width > 0, frame.height > 0 else { - return false - } - return !frame.isInfinite - } - static func isTopNavigationControlFrame(_ candidate: CGRect, in window: CGRect) -> Bool { - guard isUsableNavigationFrame(candidate), isUsableNavigationFrame(window) else { + guard SnapshotGeometry.isPositiveFinite(candidate), SnapshotGeometry.isPositiveFinite(window) + else { return false } // Accept the compact navigation/search header band without matching deep content controls. @@ -99,7 +132,7 @@ extension RunnerTests { } static func topLeadingNavigationFallbackPoint(in frame: CGRect) -> CGPoint? { - guard isUsableNavigationFrame(frame) else { + guard SnapshotGeometry.isPositiveFinite(frame) else { return nil } // Aim at the standard leading navigation slot, bounded for compact and tablet widths. @@ -109,131 +142,162 @@ extension RunnerTests { return CGPoint(x: frame.minX + xOffset, y: frame.minY + yOffset) } - private func tapTopLeadingNavigationFallback(app: XCUIApplication) -> Bool { + private func tapTopLeadingNavigationFallback(app: XCUIApplication) -> InAppBackOutcome { #if os(iOS) let frame = onScreenWindowFrame(app: app) guard let point = Self.topLeadingNavigationFallbackPoint(in: frame) else { - return false + return .unavailable } - let before = captureNavigationFallbackVisualState() + let before = captureNavigationFallbackVisualState(app: app) let context = synthesizedCoordinateContext( app: app, policy: synthesizedGesturePolicy(.coordinateTap) )?.withReferenceFrame(frame) - let synthesized = performGesture(app, idleTimeout: false) { + switch performSynthesizedGesture(app, kind: .coordinateTap, context: context, synthesize: { synthesizedTapAt(app: app, x: point.x, y: point.y, context: context) - } - if case .performed = synthesized.outcome { - return didNavigationFallbackChangeVisualState(before: before) - } - let fallback = performGesture(app) { - tapAt(app: app, x: point.x, y: point.y) - } - if case .performed = fallback.outcome { - return didNavigationFallbackChangeVisualState(before: before) + }) { + case .performed: + return verifyNavigationFallbackOutcome(app: app, before: before) + case .refused: + return .unavailable + case .xctestFallback: + let fallback = performGesture(app) { + tapAt(app: app, x: point.x, y: point.y) + } + if case .performed = fallback.outcome { + return verifyNavigationFallbackOutcome(app: app, before: before) + } } #endif - return false + return .unavailable } - private func captureNavigationFallbackVisualState() -> Data? { + private func captureNavigationFallbackVisualState(app: XCUIApplication) -> NavigationVisualSample { #if os(iOS) - runnerPngData(for: XCUIScreen.main.screenshot().image) + return Self.navigationFallbackSample( + resolvingApp: { self.captureResolvedAppScreen(app: app) }, + systemSurface: { self.captureResolvedAppScreen(app: self.springboard) }, + encoding: { runnerPngData(for: $0.image) } + ) #else - return nil + return NavigationVisualSample(data: nil) #endif } - private func didNavigationFallbackChangeVisualState(before: Data?) -> Bool { - sleepFor(Self.navigationFallbackVerificationDelay) - let after = captureNavigationFallbackVisualState() - let changed = Self.didNavigationFallbackChangeVisualState(before: before, after: after) - if !changed { - NSLog("AGENT_DEVICE_RUNNER_IN_APP_BACK_FALLBACK_NO_STATE_CHANGE") - } - return changed - } - - static func didNavigationFallbackChangeVisualState(before: Data?, after: Data?) -> Bool { - guard let before, let after else { return false } - return before != after - } - - private func macOSNavigationBackElement(app: XCUIApplication) -> XCUIElement? { - let predicate = NSPredicate( - format: "identifier == %@ OR label == %@", - "go back", - "Back" - ) - let element = app.descendants(matching: .any).matching(predicate).firstMatch - return element.exists ? element : nil + /// The decision the in-app `back` fallback makes about WHAT to sample, kept apart from the live app + /// so the decision itself is testable. It samples only the app's own resolved screen: consulting the + /// system surface for an app that resolved no window would capture SpringBoard's home screen, which + /// reads as "unchanged" across a before/after pair and launders a wrong-process frame into a false + /// "no back control" — the opposite of the unknown-outcome answer the fallback owes (#2728). The + /// system surface is still threaded in, so sampling it is the tested contract: a caller that started + /// to consult it fails the fallback's test. `navigationVisualSample` then names the refusal. + static func navigationFallbackSample( + resolvingApp: () -> Result, + systemSurface: () -> Result, + encoding: (CapturedAppScreen) -> Data? + ) -> NavigationVisualSample { + _ = systemSurface + return navigationVisualSample(from: resolvingApp(), encoding: encoding) } -#if AGENT_DEVICE_RUNNER_UNIT_TESTS - func testTopLeadingNavigationFallbackPointTargetsHeaderControlBand() throws { - let point = try XCTUnwrap( - Self.topLeadingNavigationFallbackPoint( - in: CGRect(x: 0, y: 0, width: 430, height: 932) + /// Turns a capture answer into a navigation sample: the encoded frame when the display answered, + /// and the typed reason it refused otherwise. A resolved display whose image would not encode is + /// named as the capture failure it is, not as an unnamed no-sample that would default to "no + /// display resolved" (#2728). Kept apart from the query so the mapping itself is testable. + static func navigationVisualSample( + from outcome: Result, + encoding: (CapturedAppScreen) -> Data? + ) -> NavigationVisualSample { + switch outcome { + case .success(let captured): + guard let png = encoding(captured) else { + let refusal = RunnerAppScreenCaptureFailure.unrenderableImage + return NavigationVisualSample( + data: nil, + refusalCode: refusal.rawValue, + refusalHint: refusal.hint + ) + } + return NavigationVisualSample(data: png) + case .failure(let failure): + return NavigationVisualSample( + data: nil, + refusalCode: failure.rawValue, + refusalHint: failure.hint ) - ) - - XCTAssertEqual(point.x, 34.4, accuracy: 0.01) - XCTAssertEqual(point.y, 132, accuracy: 0.01) + } } - func testTopLeadingNavigationFallbackPointRejectsInvalidFrame() { - XCTAssertNil(Self.topLeadingNavigationFallbackPoint(in: .infinite)) - XCTAssertNil(Self.topLeadingNavigationFallbackPoint(in: .zero)) + private func verifyNavigationFallbackOutcome( + app: XCUIApplication, + before: NavigationVisualSample + ) -> InAppBackOutcome { + sleepFor(Self.navigationFallbackVerificationDelay) + let after = captureNavigationFallbackVisualState(app: app) + let observation = Self.navigationVisualObservation(before: before.data, after: after.data) + // The sample sizes and the observation name together tell a refused capture apart from an + // unchanged screen, and the fallback is rare enough that saying so every time costs nothing. + NSLog( + "AGENT_DEVICE_RUNNER_IN_APP_BACK_VISUAL_VERIFICATION beforeBytes=%ld afterBytes=%ld changed=%@", + before.data?.count ?? -1, + after.data?.count ?? -1, + observation.logToken + ) + return Self.inAppBackOutcome(observation: observation, before: before, after: after) } - func testNavigationBackControlRankPrefersBackThenCloseThenCancel() { - XCTAssertEqual(Self.navigationBackControlRank(label: "Back", identifier: ""), 0) - XCTAssertEqual(Self.navigationBackControlRank(label: "Close", identifier: ""), 1) - XCTAssertEqual(Self.navigationBackControlRank(label: "Cancel search", identifier: ""), 2) - XCTAssertNil(Self.navigationBackControlRank(label: "Search for more feeds", identifier: "")) + /// What an observation of the fallback's before/after samples concludes. Kept apart from the sleep + /// and the capture so the three-way decision is testable without a running app. + static func inAppBackOutcome( + observation: NavigationVisualObservation, + before: NavigationVisualSample, + after: NavigationVisualSample + ) -> InAppBackOutcome { + switch observation { + case .changed: + return .performed + case .unchanged: + return .unavailable + case .unobserved: + // No sample is not evidence of no navigation change: the fallback ran, so report the display + // refusal it hit rather than laundering an unobservable result into "back is not available". + return .unverified(Self.navigationFallbackErrorPayload(after: after, before: before)) + } } - func testNavigationBackPredicateUsesTheSharedKeywordTable() { - let predicate = Self.navigationBackPredicate() - - XCTAssertTrue(predicate.evaluate(with: ["label": "Back", "identifier": ""])) - XCTAssertTrue(predicate.evaluate(with: ["label": "", "identifier": "close-button"])) - XCTAssertFalse(predicate.evaluate(with: ["label": "Search for more feeds", "identifier": ""])) + static func navigationVisualObservation( + before: Data?, + after: Data? + ) -> NavigationVisualObservation { + guard let before, let after else { return .unobserved } + return before != after ? .changed : .unchanged } - func testTopNavigationControlFrameAcceptsOnlyHeaderBand() { - let window = CGRect(x: 0, y: 0, width: 430, height: 932) - - XCTAssertTrue( - Self.isTopNavigationControlFrame( - CGRect(x: 340, y: 84, width: 72, height: 44), - in: window - ) - ) - XCTAssertFalse( - Self.isTopNavigationControlFrame( - CGRect(x: 20, y: 760, width: 72, height: 44), - in: window - ) + /// The refusal the fallback most recently hit wins, so the code names the last thing it looked at + /// before giving up. A missing reason on both sides is unreachable on iOS (every nil sample carries + /// one) and defaults to the plain display-unresolved code. + static func navigationFallbackErrorPayload( + after: NavigationVisualSample, + before: NavigationVisualSample + ) -> ErrorPayload { + ErrorPayload( + code: + after.refusalCode + ?? before.refusalCode + ?? RunnerAppScreenCaptureFailure.unresolvedScreen.rawValue, + message: + "The in-app back fallback was dispatched, but no display could be sampled to confirm the result. This is an unknown outcome, not evidence that a back control is absent.", + hint: after.refusalHint ?? before.refusalHint ) - XCTAssertFalse(Self.isTopNavigationControlFrame(.infinite, in: window)) } - func testNavigationFallbackRequiresObservedVisualChange() { - XCTAssertTrue( - Self.didNavigationFallbackChangeVisualState( - before: Data([1, 2, 3]), - after: Data([1, 2, 4]) - ) - ) - XCTAssertFalse( - Self.didNavigationFallbackChangeVisualState( - before: Data([1, 2, 3]), - after: Data([1, 2, 3]) - ) + private func macOSNavigationBackElement(app: XCUIApplication) -> XCUIElement? { + let predicate = NSPredicate( + format: "identifier == %@ OR label == %@", + "go back", + "Back" ) - XCTAssertFalse(Self.didNavigationFallbackChangeVisualState(before: nil, after: Data([1]))) - XCTAssertFalse(Self.didNavigationFallbackChangeVisualState(before: Data([1]), after: nil)) + let element = app.descendants(matching: .any).matching(predicate).firstMatch + return element.exists ? element : nil } -#endif } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+PrivateAXPresentation.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+PrivateAXPresentation.swift index 202e2d061b..b95b1dc5a1 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+PrivateAXPresentation.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+PrivateAXPresentation.swift @@ -89,212 +89,3 @@ extension RunnerTests { (value as? Bool) ?? (value as? NSNumber)?.boolValue } } - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS -extension RunnerTests { - fileprivate static func privateAXFrame( - _ x: Double, _ y: Double, _ width: Double, _ height: Double - ) -> [String: Any] { - ["x": x, "y": y, "width": width, "height": height] - } - - /// A scroll container whose second row is scrolled out of the viewport, plus an unlabeled - /// decoration: the shapes the regular projection folds away and the raw projection must keep. - fileprivate static var privateAXScrolledFixture: [String: Any] { - let frame = privateAXFrame - return ["type": Int(XCUIElement.ElementType.application.rawValue), - "label": "Element", "frame": frame(0, 0, 402, 874), "children": [[ - "type": Int(XCUIElement.ElementType.scrollView.rawValue), "frame": frame(0, 96, 402, 700), - "actions": ["Scroll down"], - "children": [ - ["type": Int(XCUIElement.ElementType.button.rawValue), "label": "Profile picture", - "frame": frame(16, 120, 44, 44), - "children": [["type": Int(XCUIElement.ElementType.image.rawValue), - "frame": frame(16, 120, 1, 1)]]], - ["type": Int(XCUIElement.ElementType.button.rawValue), "label": "Theme", - "frame": frame(16, 900, 360, 44)]]]]] - } - - /// Acquire with the private-AX serializer, run the one normalization pass the production capture - /// plan runs (`captureWithBackend`), then present through the shared regular fold -- the production - /// route for this backend since the fold moved into presentation (#1797, #2661). - fileprivate func privateAXRegularPresentation( - rawRoot: [String: Any], - viewport: CGRect, - interactiveOnly: Bool = false - ) throws -> [PresentedNode] { - let hint = CaptureHint( - projection: .regular, depth: nil, regularPresentedDepth: nil, - interactiveOnly: interactiveOnly, customActions: false) - let nodes = privateAXNormalizedAcquisition( - rawRoot: rawRoot, hint: hint, viewport: viewport, - interfaceOrientation: RunnerInterfaceOrientation.portrait) - return try SnapshotPresentation.presentRegular( - SnapshotAcquisition( - hint: hint, nodes: nodes, truncated: false, effectiveDepth: nil, viewport: viewport, - interfaceOrientation: RunnerInterfaceOrientation.portrait), - options: PresentationOptions( - interactiveOnly: interactiveOnly, depth: nil, scope: nil, raw: false), - policy: .cursorProjected - ).nodes - } - - /// Acquire, then normalize once -- the exact pair `captureWithBackend` runs for this backend. - fileprivate func privateAXNormalizedAcquisition( - rawRoot: [String: Any], - hint: CaptureHint, - viewport: CGRect, - interfaceOrientation: Int - ) -> [RawAXNode] { - SnapshotGeometrySpace.normalized( - nodes: privateAXAcquisition(rawRoot: rawRoot, hint: hint), - viewport: viewport, - interfaceOrientation: interfaceOrientation - ) - } - - /// The one normalization pass has to reach the node it publishes: this asserts the rotated rect of - /// a key under a turned surface host, and an untouched sibling under the app's own window, so a - /// pass that drops the space a subtree declared fails here. - func testPrivateAXAcquisitionPublishesATurnedSurfaceHostInAppOrientationSpace() { - let frame = Self.privateAXFrame - let appWindow: [String: Any] = [ - "type": Int(XCUIElement.ElementType.window.rawValue), - "frame": frame(0, 0, 874, 402), - "children": [ - ["type": Int(XCUIElement.ElementType.button.rawValue), "label": "Home", - "frame": frame(204, 323, 91, 55), "children": []] - ] - ] - let keyboardWindow: [String: Any] = [ - "type": Int(XCUIElement.ElementType.window.rawValue), - "frame": frame(0, 0, 874, 402), - "children": [ - ["type": Int(XCUIElement.ElementType.other.rawValue), - "frame": frame(0, 0, 402, 874), - "children": [ - ["type": Int(XCUIElement.ElementType.key.rawValue), "label": "q", - "frame": frame(154, 77, 45, 72), "children": []] - ]] - ] - ] - let hint = CaptureHint( - projection: .raw, depth: nil, regularPresentedDepth: nil, - interactiveOnly: false, customActions: false) - let nodes = privateAXNormalizedAcquisition( - rawRoot: [ - "type": Int(XCUIElement.ElementType.application.rawValue), - "label": "Element", "frame": frame(0, 0, 874, 402), - "children": [appWindow, keyboardWindow] - ], - hint: hint, - viewport: CGRect(x: 0, y: 0, width: 874, height: 402), - interfaceOrientation: RunnerInterfaceOrientation.landscapeRight - ) - - // Measured on iPhone 17 Pro (26.2): the key plane's left column arrives 154 pt along the device's - // long axis and comes back 203 pt down the app's short one. - XCTAssertEqual( - nodes.first { $0.label == "q" }?.rect, - SnapshotRect(x: 77, y: 203, width: 72, height: 45) - ) - XCTAssertEqual( - nodes.first { $0.label == "Home" }?.rect, - SnapshotRect(x: 204, y: 323, width: 91, height: 55) - ) - } - - func testPrivateAXRegularPresentationProjectsToViewportAndKeepsScrollHint() throws { - let nodes = try privateAXRegularPresentation( - rawRoot: Self.privateAXScrolledFixture, - viewport: CGRect(x: 0, y: 0, width: 402, height: 874)) - XCTAssertEqual(nodes.compactMap(\.label), ["Element", "Profile picture"]) - let scrollView = nodes.first { $0.type == "ScrollView" } - XCTAssertEqual(scrollView?.hiddenContentBelow, true) - XCTAssertEqual(scrollView?.actions, ["Scroll down"]) - } - - /// #1797 D4: the raw projection is the acquired tree. The offscreen row and the sub-pixel - /// decoration the regular projection folds away are both present, at traversal depth, and every - /// regular node still appears -- `regular ⊆ raw` on the same capture. - func testPrivateAXRawProjectionKeepsEveryAcquiredNode() throws { - let viewport = CGRect(x: 0, y: 0, width: 402, height: 874) - let root = Self.privateAXScrolledFixture - let regular = try privateAXRegularPresentation(rawRoot: root, viewport: viewport, - interactiveOnly: true) - let raw = privateAXNormalizedAcquisition(rawRoot: root, - hint: CaptureHint( - projection: .raw, depth: nil, regularPresentedDepth: nil, - interactiveOnly: false, customActions: false), - viewport: viewport, - interfaceOrientation: RunnerInterfaceOrientation.portrait) - - XCTAssertEqual(raw.map(\.type), ["Application", "ScrollView", "Button", "Image", "Button"]) - XCTAssertEqual(raw.map(\.depth), [0, 1, 2, 3, 2]) - XCTAssertEqual(raw.map(\.parentIndex), [nil, 0, 1, 2, 1]) - XCTAssertEqual(raw.compactMap(\.label), ["Element", "Profile picture", "Theme"]) - // The offscreen row is a reported fact in raw, and reported facts do not become hittable - // just because the projection kept them. - XCTAssertEqual(raw.last?.hittable, false) - XCTAssertTrue(raw.allSatisfy { $0.hiddenContentAbove == nil && $0.hiddenContentBelow == nil }) - - let rawKeys = Set(raw.map { "\($0.type)-\($0.label ?? "")-\($0.rect.y)" }) - for node in regular { - XCTAssertTrue( - rawKeys.contains("\(node.type)-\(node.label ?? "")-\(node.rect.y)"), - "regular node \(node.type)/\(node.label ?? "") is missing from the raw projection" - ) - } - XCTAssertGreaterThan(raw.count, regular.count) - } - - /// Raw depth is traversal depth, so a raw `--depth` request is the one narrowing this backend - /// can prove complete. - func testPrivateAXRawProjectionAppliesRequestedTraversalDepth() { - let raw = privateAXNormalizedAcquisition(rawRoot: Self.privateAXScrolledFixture, - hint: CaptureHint( - projection: .raw, depth: 2, regularPresentedDepth: nil, - interactiveOnly: false, customActions: false), - viewport: CGRect(x: 0, y: 0, width: 402, height: 874), - interfaceOrientation: RunnerInterfaceOrientation.portrait) - XCTAssertEqual(raw.map(\.type), ["Application", "ScrollView", "Button", "Button"]) - XCTAssertEqual(raw.map(\.depth), [0, 1, 2, 2]) - } - - func testPrivateAXPresentationKeepsOffscreenSubtreeExcludedWhenChildFramesAreClamped() throws { - let frame = Self.privateAXFrame - let root: [String: Any] = ["type": Int(XCUIElement.ElementType.application.rawValue), - "label": "Element", "frame": frame(0, 0, 402, 874), "children": [[ - "type": Int(XCUIElement.ElementType.table.rawValue), "frame": frame(0, 96, 402, 700), - "children": [["type": Int(XCUIElement.ElementType.cell.rawValue), - "label": "Theme", "frame": frame(0, 900, 402, 44), "children": [[ - "type": Int(XCUIElement.ElementType.staticText.rawValue), - "label": "Theme", "frame": frame(16, 96, 120, 44)], - ["type": Int(XCUIElement.ElementType.switch.rawValue), - "label": "Theme", "frame": frame(340, 96, 46, 44)]]]]]]] - - let nodes = try privateAXRegularPresentation( - rawRoot: root, viewport: CGRect(x: 0, y: 0, width: 402, height: 874)) - - XCTAssertEqual(nodes.compactMap(\.label), ["Element"]) - XCTAssertEqual(nodes.first { $0.type == "Table" }?.hiddenContentBelow, true) - } - - func testPrivateAXGeometrylessSemanticsAreNeverActionableOrScrollContexts() throws { - let zero = ["x": 0, "y": 0, "width": 0, "height": 0] - let root: [String: Any] = ["type": Int(XCUIElement.ElementType.application.rawValue), - "label": "Element", "frame": ["x": 0, "y": 0, "width": 402, "height": 874], - "children": [["type": Int(XCUIElement.ElementType.scrollView.rawValue), - "label": "Settings semantics", "frame": zero, "children": [[ - "type": Int(XCUIElement.ElementType.button.rawValue), "label": "Theme", "frame": zero], - ["type": Int(XCUIElement.ElementType.other.rawValue), "frame": zero]]]]] - let nodes = try privateAXRegularPresentation( - rawRoot: root, viewport: CGRect(x: 0, y: 0, width: 402, height: 874), - interactiveOnly: true) - XCTAssertEqual(nodes.compactMap(\.label), ["Element", "Settings semantics", "Theme"]) - XCTAssertEqual(nodes.filter { $0.index != 0 }.map(\.hittable), [false, false]) - XCTAssertFalse(nodes.contains { $0.type == "Other" }) - XCTAssertTrue(nodes.allSatisfy { $0.hiddenContentAbove == nil && $0.hiddenContentBelow == nil }) - } -} -#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScreenRecorder.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScreenRecorder.swift index 6f55379c24..c70b0e3228 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScreenRecorder.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScreenRecorder.swift @@ -24,13 +24,21 @@ extension RunnerTests { private var isStopping = false private var startedSession = false private var startError: Error? + #if AGENT_DEVICE_RUNNER_UNIT_TESTS + private var appendedFramesForTesting: [RunnerImage] = [] + #endif init(outputPath: String, fps: Int32?) { self.outputPath = outputPath self.fps = fps } - func start(captureFrame: @escaping () -> RunnerImage?) throws { + /// `bootstrap` must produce the frame that sizes the writer and runs on the caller's thread. + /// `frame` answers each tick with an image, or `nil` to drop the tick. + func start( + bootstrap: @escaping () -> Result, + frame: @escaping () -> RunnerImage? + ) throws { let url = URL(fileURLWithPath: outputPath) let directory = url.deletingLastPathComponent() try FileManager.default.createDirectory( @@ -44,21 +52,34 @@ extension RunnerTests { var dimensions: CGSize = .zero var bootstrapImage: RunnerImage? + var lastFailure: RunnerAppScreenCaptureFailure? let bootstrapDeadline = Date().addingTimeInterval(2.0) while Date() < bootstrapDeadline { - if let image = captureFrame(), let cgImage = runnerCGImage(from: image) { - bootstrapImage = image - dimensions = CGSize(width: cgImage.width, height: cgImage.height) + switch bootstrap() { + case .success(let captured): + bootstrapImage = captured.image + dimensions = CGSize(width: captured.pixelWidth, height: captured.pixelHeight) + case .failure(let failure): + lastFailure = failure + } + if dimensions.width > 0, dimensions.height > 0 { break } Thread.sleep(forTimeInterval: 0.05) } guard dimensions.width > 0, dimensions.height > 0 else { - throw NSError( - domain: "AgentDeviceRunner.Record", - code: 1, - userInfo: [NSLocalizedDescriptionKey: "failed to capture initial frame"] - ) + // The bootstrap frame is required: the writer is sized from it. A capture that refused names + // why (no window, no display, unencodable image) so the host sees a typed reason rather than + // the generic "no frame" it used to collapse every refusal into (#2728). macOS keeps its + // host-display behavior and its original error, because nothing here is a panel question. + #if os(iOS) + throw RunnerTests.recordingBootstrapError(from: lastFailure) + #else + // macOS/tvOS preserve their original untyped record error regardless of why the host capture + // refused; the reason is read here only so the shared bootstrap loop carries no dead write. + _ = lastFailure + throw RunnerTests.recordingBootstrapError(from: nil) + #endif } let writer = try AVAssetWriter(outputURL: url, fileType: .mp4) @@ -114,7 +135,7 @@ extension RunnerTests { timer.setEventHandler { [weak self] in guard let self else { return } if self.shouldStop() { return } - guard let image = captureFrame() else { return } + guard let image = frame() else { return } self.append(image: image) } self.timer = timer @@ -212,6 +233,9 @@ extension RunnerTests { return } lastTimestampValue = timestampValue + #if AGENT_DEVICE_RUNNER_UNIT_TESTS + appendedFramesForTesting.append(image) + #endif } private func timestampCandidateValue(for nowUptime: TimeInterval) -> Int64 { @@ -269,6 +293,52 @@ extension RunnerTests { } } +extension RunnerTests { + /// Starts `recorder` on the frames `capture` produces. The bootstrap frame is taken on the calling + /// thread, which is main for `record start`. Each later tick is optional work: it hops to main only + /// while no other main-thread work is in flight, so it never queues behind a command. + /// A capture still running after `recordingFrameCaptureTimeout` is abandoned and its frame dropped; + /// its late result is never returned. + func startRecording( + _ recorder: ScreenRecorder, + capture: @escaping () -> Result + ) throws { + try recorder.start(bootstrap: capture) { [weak self] in + guard let self else { return nil } + return try? self.runMainThreadWorkIfIdle( + "recording_frame", + timeout: self.recordingFrameCaptureTimeout, + timeoutError: self.mainThreadExecutionTimeoutError + ) { + try capture().get().image + } + } + } + + /// The error a `record start` bootstrap raises when no initial frame arrived. On iOS the last capture + /// refusal (if any) is the honest reason and travels as its own typed code; only when nothing + /// refused — a macOS host capture, or a deadline that elapsed before any answer — does it fall back + /// to the original untyped record error, which keeps pre-panel behavior intact (#2728). + static func recordingBootstrapError(from lastFailure: RunnerAppScreenCaptureFailure?) -> Error { + lastFailure + ?? NSError( + domain: "AgentDeviceRunner.Record", + code: 1, + userInfo: [NSLocalizedDescriptionKey: "failed to capture initial frame"] + ) + } + + /// Maps a `record start` failure to the wire payload. A capture that refused carries a typed + /// `APP_SCREEN_*` reason, so a no-window bootstrap reaches the host as that code rather than the + /// generic record error it used to collapse into; a genuine writer failure keeps its message. + static func recordingStartErrorPayload(for error: Error) -> ErrorPayload { + if let failure = error as? RunnerAppScreenCaptureFailure { + return ErrorPayload(code: failure.rawValue, message: failure.message, hint: failure.hint) + } + return ErrorPayload(message: "failed to start recording: \(error.localizedDescription)") + } +} + #if AGENT_DEVICE_RUNNER_UNIT_TESTS extension RunnerTests.ScreenRecorder { @discardableResult @@ -279,5 +349,11 @@ extension RunnerTests.ScreenRecorder { lastTimestampValue = allocatedTimestamp return allocatedTimestamp } + + func appendedFrameSnapshotForTesting() -> [RunnerImage] { + lock.lock() + defer { lock.unlock() } + return appendedFramesForTesting + } } #endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollDragExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollDragExecution.swift new file mode 100644 index 0000000000..77c896c5ba --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollDragExecution.swift @@ -0,0 +1,271 @@ +import XCTest +import AgentDeviceSnapshotPresentation + +extension RunnerTests { + func invalidScrollDirectionResponse(commandName: String) -> Response { + Response( + ok: false, + error: ErrorPayload( + code: "INVALID_ARGS", + message: "\(commandName) requires direction up|down|left|right" + ) + ) + } + + func scrollDurationIsValid(_ durationMs: Double?) -> Bool { + guard let durationMs else { return true } + return durationMs.isFinite && durationMs >= 0 && durationMs <= 10000 + } + + func invalidScrollDurationResponse(commandName: String) -> Response { + return Response( + ok: false, + error: ErrorPayload( + code: "INVALID_ARGS", + message: "\(commandName) durationMs must be between 0 and 10000" + ) + ) + } + + func executeScrollDragGesture( + activeApp: XCUIApplication, + x: Double, + y: Double, + x2: Double, + y2: Double, + durationMs: Double, + message: String, + context: SynthesizedCoordinateContext, + releaseBehavior: ScrollReleaseBehavior? + ) -> Response { +#if os(iOS) + return executeDragGesture( + activeApp: activeApp, + x: x, + y: y, + x2: x2, + y2: y2, + durationMs: durationMs, + message: message, + synthesizedContext: context, + synthesized: (profile: scrollDragProfile(releaseBehavior: releaseBehavior), policyKind: .scroll) + ) +#else + return executeDragGesture( + activeApp: activeApp, + x: x, + y: y, + x2: x2, + y2: y2, + durationMs: durationMs, + message: message + ) +#endif + } + + /// Shared coordinate drag execution. Callers that pass `synthesized` take the iOS synthesized + /// lane with that profile and fallback policy; the rest perform an XCTest coordinate drag. + func executeDragGesture( + activeApp: XCUIApplication, + x: Double, + y: Double, + x2: Double, + y2: Double, + durationMs: Double?, + message: String, + synthesizedContext: SynthesizedCoordinateContext? = nil, + synthesized: (profile: SynthesizedDragProfile, policyKind: SynthesizedGesturePolicyKind)? = nil + ) -> Response { + let durationMs = durationMs ?? runnerDefaultDragDurationMs + let commandName = dragCommandName(message: message) + guard x.isFinite, y.isFinite, x2.isFinite, y2.isFinite else { + return Response( + ok: false, + error: ErrorPayload(code: "INVALID_ARGS", message: "\(commandName) requires finite coordinates") + ) + } + if let synthesized, let synthesizedResponse = executeSynthesizedDragGesture( + activeApp: activeApp, + x: x, + y: y, + x2: x2, + y2: y2, + durationMs: durationMs, + message: message, + context: synthesizedContext, + policyKind: synthesized.policyKind, + profile: synthesized.profile + ) { + return synthesizedResponse + } + let dragPoints = keyboardAvoidingDragPoints(app: activeApp, x: x, y: y, x2: x2, y2: y2) + let dragFrame = resolvedDragVisualizationFrame( + app: activeApp, + x: dragPoints.x, + y: dragPoints.y, + x2: dragPoints.x2, + y2: dragPoints.y2 + ) + let holdDuration = synthesized == nil + ? coordinateDragHoldDuration() + : synthesizedSwipeFallbackHoldDuration(durationMs: durationMs) + let (timing, outcome) = performGesture(activeApp) { + dragAt( + app: activeApp, + x: dragPoints.x, + y: dragPoints.y, + x2: dragPoints.x2, + y2: dragPoints.y2, + holdDuration: holdDuration + ) + } + if let response = unsupportedResponse(for: outcome) { + return response + } + return gestureResponse(message: message, timing: timing, frame: .drag(dragFrame)) + } + + private func executeSynthesizedDragGesture( + activeApp: XCUIApplication, + x: Double, + y: Double, + x2: Double, + y2: Double, + durationMs: Double, + message: String, + context: SynthesizedCoordinateContext?, + policyKind: SynthesizedGesturePolicyKind, + profile: SynthesizedDragProfile + ) -> Response? { +#if os(iOS) + let policy = synthesizedGesturePolicy(policyKind) + let context = context ?? synthesizedCoordinateContext(app: activeApp, policy: policy) + guard let plan = axFreeSynthesizedDragPlan( + app: activeApp, + x: x, + y: y, + x2: x2, + y2: y2, + context: context + ) + else { + logSynthesizedGesturePolicyDecision(kind: policyKind, context: context, fallbackAttempted: false) + return Response( + ok: false, + error: ErrorPayload( + code: "INVALID_ARGS", + message: "\(dragCommandName(message: message)) could not resolve a finite synthesized coordinate frame" + ) + ) + } + let durationMs = min(max(durationMs, 16), 10000) + let dragFrame = axFreeDragVisualizationFrame( + x: plan.points.x, + y: plan.points.y, + x2: plan.points.x2, + y2: plan.points.y2, + referenceFrame: plan.referenceFrame + ) + switch performSynthesizedGesture(activeApp, kind: policyKind, context: plan.context, synthesize: { + synthesizedDragAt( + app: activeApp, + x: plan.points.x, + y: plan.points.y, + x2: plan.points.x2, + y2: plan.points.y2, + durationMs: durationMs, + profile: profile, + context: plan.context + ) + }) { + case .performed(let timing): + return gestureResponse(message: message, timing: timing, frame: .drag(dragFrame)) + case .xctestFallback(let fallbackMessage, let hint): + return executeCoordinateDragFallback( + activeApp: activeApp, + x: plan.points.x, + y: plan.points.y, + x2: plan.points.x2, + y2: plan.points.y2, + durationMs: durationMs, + message: message, + fallback: GestureFallback(strategy: "xctest-coordinate-drag", message: fallbackMessage, hint: hint) + ) + case .refused(_, let refusalMessage, let hint): + return unsupportedResponse(message: refusalMessage, hint: hint) + } +#else + return nil +#endif + } + + private func executeCoordinateDragFallback( + activeApp: XCUIApplication, + x: Double, + y: Double, + x2: Double, + y2: Double, + durationMs: Double, + message: String, + fallback: GestureFallback + ) -> Response { + let dragPoints = keyboardAvoidingDragPoints(app: activeApp, x: x, y: y, x2: x2, y2: y2) + let dragFrame = resolvedDragVisualizationFrame( + app: activeApp, + x: dragPoints.x, + y: dragPoints.y, + x2: dragPoints.x2, + y2: dragPoints.y2 + ) + let holdDuration = synthesizedSwipeFallbackHoldDuration(durationMs: durationMs) + let (timing, outcome) = performGesture(activeApp) { + dragAt( + app: activeApp, + x: dragPoints.x, + y: dragPoints.y, + x2: dragPoints.x2, + y2: dragPoints.y2, + holdDuration: holdDuration + ) + } + if let response = unsupportedResponse(for: outcome) { + return response + } + return gestureResponse( + message: message, + timing: timing, + frame: .drag(dragFrame), + fallback: fallback + ) + } + + /// Adds the #2500 avoidance evidence to a scroll response. Only the frame resolver knows whether + /// it trimmed the swipe for a keyboard, and only `scroll` has this evidence to carry, so it is + /// attached where the frame was resolved rather than threaded through every gesture response. + /// The refusal a keyboard forces. It performs no gesture: swiping into the keys would leave the + /// surface where it was, which the daemon's no-progress fingerprint reads as a stuck container + /// (#2499) and an agent reads as a broken scroll. The TS owner maps the code to the + /// `scroll_keyboard_occludes_surface` reason and the "dismiss the keyboard" hint. + func scrollKeyboardOccludedResponse( + direction: String, + keyboardMinY: Double, + visibleHeight: Double + ) -> Response { + return Response( + ok: false, + error: ErrorPayload( + code: ScrollViewportPolicy.occlusionRunnerCode, + message: String( + format: + "scroll %@ refused: the keyboard leaves %.0fpt of visible surface above it, too little to swipe", + direction, + visibleHeight + ) + ) + ) + } + + private func dragCommandName(message: String) -> String { + return message == "scrolled" ? "scroll" : "drag" + } +} diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollGesture.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollGesture.swift index 6b3423777c..326eec698f 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollGesture.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollGesture.swift @@ -5,8 +5,9 @@ import XCTest // This is a deliberate two-place invariant: the daemon keeps the TS implementation (for Android, // recording, and reported-pixels), and the runner places the gesture with this Swift copy. Both // ports are asserted against the same table, contracts/fixtures/scroll-gesture.json (gated -// XCTest at the bottom of this file, vitest twin packages/contracts/src/scroll-gesture.test.ts) — -// if you change the math in either language, update the other and the table. +// XCTest in UnitTests/RunnerTests+ScrollGestureTests.swift, vitest twin +// packages/contracts/src/scroll-gesture.test.ts) — if you change the math in either language, +// update the other and the table. // // All inputs here are positive (reference dims, travel, center), so Swift's `.rounded()` // (half away from zero) matches JS `Math.round` (half up) on every value computed below. @@ -96,203 +97,3 @@ func runnerDragCommandDefaults(_ command: Command) -> RunnerDragCommandDefaults durationMs: command.durationMs ?? runnerDefaultDragDurationMs ) } - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS -private struct ScrollGestureFixture: Decodable { - struct Constants: Decodable { - let defaultIosScrollAmount: Double - let defaultMobileScrollDurationMs: Double - let defaultIosScrollDurationMs: Double - let defaultScrollAmount: Double - let defaultEdgePaddingFraction: Double - let ordinaryScrollReleaseBehavior: String - let edgeScrollReleaseBehavior: String - } - struct Expected: Decodable { - let x1: Double - let y1: Double - let x2: Double - let y2: Double - let pixels: Double - } - struct Case: Decodable { - let name: String - let direction: String - let amount: Double? - let pixels: Double? - let referenceWidth: Double - let referenceHeight: Double - let expected: Expected - } - - let constants: Constants - let cases: [Case] -} - -extension RunnerTests { - // Cross-language parity table: every case in contracts/fixtures/scroll-gesture.json must agree - // with the vitest twin (packages/contracts/src/scroll-gesture.test.ts). Add vectors there, - // never fork the math. - private func loadScrollGestureFixture() throws -> ScrollGestureFixture { - let fixtureURL = URL(fileURLWithPath: #filePath) - .deletingLastPathComponent() // AgentDeviceRunnerUITests - .deletingLastPathComponent() // AgentDeviceRunner - .deletingLastPathComponent() // runner - .deletingLastPathComponent() // apple - .deletingLastPathComponent() // repo root - .appendingPathComponent("contracts") - .appendingPathComponent("fixtures") - .appendingPathComponent("scroll-gesture.json") - return try JSONDecoder().decode(ScrollGestureFixture.self, from: Data(contentsOf: fixtureURL)) - } - - func testRunnerScrollGesturePlanMatchesParityTable() throws { - let fixture = try loadScrollGestureFixture() - XCTAssertFalse(fixture.cases.isEmpty, "parity table must not be empty") - for testCase in fixture.cases { - let plan = try XCTUnwrap( - runnerScrollGesturePlan( - direction: try XCTUnwrap(RunnerScrollDirection(rawValue: testCase.direction)), - amount: testCase.amount, - pixels: testCase.pixels, - referenceWidth: testCase.referenceWidth, - referenceHeight: testCase.referenceHeight - ), - testCase.name - ) - XCTAssertEqual(plan.x1, testCase.expected.x1, testCase.name) - XCTAssertEqual(plan.y1, testCase.expected.y1, testCase.name) - XCTAssertEqual(plan.x2, testCase.expected.x2, testCase.name) - XCTAssertEqual(plan.y2, testCase.expected.y2, testCase.name) - XCTAssertEqual(plan.travelPixels, testCase.expected.pixels, testCase.name) - } - } - - // The planner constants are private on both sides; the table pins them behaviourally on a - // 1000px axis where every rounding step is exact. - func testRunnerScrollGesturePlanUsesParityTableConstants() throws { - let constants = try loadScrollGestureFixture().constants - let defaultScroll = try JSONDecoder().decode( - Command.self, from: Data(#"{"command":"scroll"}"#.utf8)) - let defaults = runnerDragCommandDefaults(defaultScroll) - XCTAssertEqual(defaults.durationMs, constants.defaultIosScrollDurationMs) - XCTAssertEqual(defaults.scrollAmount, constants.defaultIosScrollAmount) - let defaulted = try XCTUnwrap( - runnerScrollGesturePlan( - direction: .down, amount: nil, pixels: nil, referenceWidth: 1000, referenceHeight: 1000 - ) - ) - XCTAssertEqual(defaulted.travelPixels, 1000 * constants.defaultScrollAmount) - let saturated = try XCTUnwrap( - runnerScrollGesturePlan( - direction: .down, amount: 10, pixels: nil, referenceWidth: 1000, referenceHeight: 1000 - ) - ) - XCTAssertEqual( - saturated.travelPixels, 1000 - 2 * 1000 * constants.defaultEdgePaddingFraction) - } - - func testRunnerScrollAndDragCommandDefaultsStayDistinct() throws { - func command(_ json: String) throws -> Command { - try JSONDecoder().decode(Command.self, from: Data(json.utf8)) - } - - let pixelScroll = runnerDragCommandDefaults( - try command(#"{"command":"scroll","pixels":120}"#)) - XCTAssertNil(pixelScroll.scrollAmount) - XCTAssertEqual(pixelScroll.durationMs, 400) - - let drag = runnerDragCommandDefaults(try command(#"{"command":"drag"}"#)) - XCTAssertNil(drag.scrollAmount) - XCTAssertEqual(drag.durationMs, 250) - - let explicitScroll = runnerDragCommandDefaults( - try command(#"{"command":"scroll","amount":0.5,"durationMs":125}"#)) - XCTAssertEqual(explicitScroll.scrollAmount, 0.5) - XCTAssertEqual(explicitScroll.durationMs, 125) - - let explicitDrag = runnerDragCommandDefaults( - try command(#"{"command":"drag","durationMs":125}"#)) - XCTAssertEqual(explicitDrag.durationMs, 125) - } - - func testRunnerScrollReleaseBehaviorSelectsTheDragProfile() throws { - let constants = try loadScrollGestureFixture().constants - let controlled = try XCTUnwrap(ScrollReleaseBehavior(rawValue: constants.ordinaryScrollReleaseBehavior)) - let inertial = try XCTUnwrap(ScrollReleaseBehavior(rawValue: constants.edgeScrollReleaseBehavior)) - XCTAssertEqual( - scrollDragProfile(releaseBehavior: nil), - .controlledScroll - ) - XCTAssertEqual( - scrollDragProfile(releaseBehavior: controlled), - .controlledScroll - ) - XCTAssertEqual( - scrollDragProfile(releaseBehavior: inertial), - .fastSwipe - ) - } - - func testControlledScrollProfileUsesReliableCadenceAndMonotonicDeceleration() { - XCTAssertEqual(RunnerControlledScrollFrameCount(350), 21) - XCTAssertEqual(RunnerControlledScrollFrameCount(400), 24) - XCTAssertEqual(RunnerControlledScrollFrameCount(500), 30) - XCTAssertEqual(RunnerControlledScrollFrameCount(1_000), 30) - XCTAssertEqual(RunnerControlledScrollFrameCount(10_000), 30) - - let frameCount = RunnerControlledScrollFrameCount(400) - let progress = (0...frameCount).map { - RunnerControlledScrollProgress(Double($0) / Double(frameCount)) - } - let deltas = zip(progress.dropFirst(), progress).map { $0.0 - $0.1 } - XCTAssertEqual(progress.first, 0) - XCTAssertEqual(progress.last, 1) - XCTAssertTrue(zip(deltas, deltas.dropFirst()).allSatisfy { $0.1 <= $0.0 }) - - let iPhoneViewportPoints = 874.0 - let defaultFingerTravel = iPhoneViewportPoints * 0.65 - let finalSampleTravel = (1 - progress[progress.count - 2]) * defaultFingerTravel - XCTAssertLessThan(finalSampleTravel, 0.1) - } - - func testContinuousDragRetainsSixteenMillisecondSamplingAtLongDurations() { - XCTAssertEqual(RunnerContinuousDragFrameCount(1_000), 62) - XCTAssertEqual(RunnerContinuousDragFrameCount(10_000), 625) - } - - func testRunnerScrollGesturePlanRejectsUnknownDirection() { - XCTAssertNil(RunnerScrollDirection(rawValue: "sideways")) - } - - func testRunnerScrollGesturePlanRejectsInvalidAmountAndPixels() { - XCTAssertNil( - runnerScrollGesturePlan( - direction: .down, - amount: 0, - pixels: nil, - referenceWidth: 300, - referenceHeight: 600 - ) - ) - XCTAssertNil( - runnerScrollGesturePlan( - direction: .down, - amount: nil, - pixels: -10, - referenceWidth: 300, - referenceHeight: 600 - ) - ) - XCTAssertNil( - runnerScrollGesturePlan( - direction: .down, - amount: .infinity, - pixels: nil, - referenceWidth: 300, - referenceHeight: 600 - ) - ) - } -} -#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift index 8b0b5da622..9bc0481861 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift @@ -143,51 +143,21 @@ extension RunnerTests { ) -> SequenceStepOutcome { let x = step.x ?? 0 let y = step.y ?? 0 - // Synthesized HID tap fast path mirrors the individual `tap` command (idleTimeout:false, with - // a tapAt fallback when synthesis is unsupported), so fusing a jittered tap series does not - // change the touch mechanism for these inputs. - if step.kind == "tap", step.synthesized == true { - let policyKind = SynthesizedGesturePolicyKind.synthesizedDrag -#if os(iOS) - guard let synthesizedContext else { - let nowMs = ProcessInfo.processInfo.systemUptime * 1000 - logSynthesizedGesturePolicyDecision(kind: policyKind, context: nil, fallbackAttempted: false) - return SequenceStepOutcome( - outcome: .unsupported( - message: "synthesized coordinate tap could not resolve a finite coordinate frame", - hint: "Retry after the app is foregrounded, or use a plain screenshot to choose coordinates." - ), - gestureStartUptimeMs: nowMs, - gestureEndUptimeMs: nowMs - ) - } -#endif - let (timing, outcome) = performGesture(activeApp, idleTimeout: false) { + if let policyKind = synthesizedPolicyKind(forSequenceStep: step) { + switch performSynthesizedGesture(activeApp, kind: policyKind, context: synthesizedContext, synthesize: { synthesizedTapAt(app: activeApp, x: x, y: y, context: synthesizedContext) - } - if case .performed = outcome { - logSynthesizedGesturePolicyDecision(kind: policyKind, context: synthesizedContext, fallbackAttempted: false) - if let pauseMs = step.pauseMs, pauseMs > 0 { - sleepFor(min(max(pauseMs, 0), 10000) / 1000.0) - } - return SequenceStepOutcome( - outcome: outcome, - gestureStartUptimeMs: timing.gestureStartUptimeMs, - gestureEndUptimeMs: timing.gestureEndUptimeMs + }) { + case .performed(let timing): + return finishedSequenceStep(step, timing: timing, outcome: .performed) + case .refused(let timing, let message, let hint): + return finishedSequenceStep( + step, + timing: timing, + outcome: .unsupported(message: message, hint: hint) ) + case .xctestFallback: + break } -#if os(iOS) - guard synthesizedContext.allowsXCTestCoordinateFallback else { - logSynthesizedGesturePolicyDecision(kind: policyKind, context: synthesizedContext, fallbackAttempted: false) - return SequenceStepOutcome( - outcome: outcome, - gestureStartUptimeMs: timing.gestureStartUptimeMs, - gestureEndUptimeMs: timing.gestureEndUptimeMs - ) - } - logSynthesizedGesturePolicyDecision(kind: policyKind, context: synthesizedContext, fallbackAttempted: true) -#endif - // Synthesis unsupported (e.g. macOS) — fall through to the drag-based tapAt below. } let (timing, outcome) = performGesture(activeApp) { switch step.kind { @@ -201,6 +171,14 @@ extension RunnerTests { return tapAt(app: activeApp, x: x, y: y) } } + return finishedSequenceStep(step, timing: timing, outcome: outcome) + } + + private func finishedSequenceStep( + _ step: SequenceStep, + timing: (gestureStartUptimeMs: Double, gestureEndUptimeMs: Double), + outcome: RunnerInteractionOutcome + ) -> SequenceStepOutcome { // Sleep AFTER the step — pauseMs is the inter-step gap — but only when the step performed. // assembleSequenceExecution stops at the first unsupported outcome, so pausing after a failed // step would burn up to 10s of watchdog budget with no following step to separate from. @@ -239,188 +217,3 @@ extension RunnerTests { Response(ok: false, error: ErrorPayload(code: "INVALID_ARGS", message: message)) } } - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS -// MARK: - In-bundle unit tests (device-free) - -extension RunnerTests { - func testSequenceDecodesStepsFromWire() throws { - let json = """ - {"command":"sequence","commandId":"seq-1","steps":[ - {"kind":"tap","x":100,"y":200}, - {"kind":"doubleTap","x":101,"y":200}, - {"kind":"longPress","x":102,"y":200,"durationMs":300,"pauseMs":50} - ]} - """ - let command = try JSONDecoder().decode(Command.self, from: Data(json.utf8)) - XCTAssertEqual(command.command, .sequence) - XCTAssertEqual(command.steps?.count, 3) - XCTAssertEqual(command.steps?[0].kind, "tap") - XCTAssertEqual(command.steps?[1].kind, "doubleTap") - XCTAssertEqual(command.steps?[2].pauseMs, 50) - } - - func testSequenceAcceptsDoubleTapKind() { - // A doubleTap step missing coords must fail on the coords check, not the kind allowlist — - // proving "doubleTap" passes validateSequenceStep without needing a device to execute on. - let response = executeSequenceForTest(steps: [ - sequenceStep(kind: "doubleTap", x: nil) - ]) - XCTAssertEqual(response.ok, false) - XCTAssertEqual(response.error?.code, "INVALID_ARGS") - XCTAssertTrue(response.error?.message.contains("requires finite x and y") ?? false) - XCTAssertFalse(response.error?.message.contains("unsupported kind") ?? true) - } - - func testSequenceRejectsUnknownKind() throws { - let response = executeSequenceForTest(steps: [ - sequenceStep(kind: "tap", x: 1, y: 2), - sequenceStep(kind: "pinch", x: 3, y: 4), - ]) - XCTAssertEqual(response.ok, false) - XCTAssertEqual(response.error?.code, "INVALID_ARGS") - XCTAssertTrue(response.error?.message.contains("step 1") ?? false) - XCTAssertTrue(response.error?.message.contains("pinch") ?? false) - } - - func testSequenceRejectsEmpty() { - let response = executeSequenceForTest(steps: []) - XCTAssertEqual(response.ok, false) - XCTAssertEqual(response.error?.code, "INVALID_ARGS") - } - - func testSequenceRejectsTooManySteps() { - let steps = (0..<21).map { _ in sequenceStep(kind: "tap", x: 1, y: 2) } - let response = executeSequenceForTest(steps: steps) - XCTAssertEqual(response.ok, false) - XCTAssertEqual(response.error?.code, "INVALID_ARGS") - XCTAssertTrue(response.error?.message.contains("at most 20") ?? false) - } - - func testSequenceHasSynthesizedCoordinateStep() { - XCTAssertTrue( - sequenceHasSynthesizedCoordinateStep([ - sequenceStep(kind: "tap", x: 1, y: 2, synthesized: true), - ]) - ) - XCTAssertFalse( - sequenceHasSynthesizedCoordinateStep([ - sequenceStep(kind: "tap", x: 1, y: 2), - sequenceStep(kind: "doubleTap", x: 1, y: 2, synthesized: true), - ]) - ) - } - - func testAssembleSequencePreservesOrderOnSuccess() { - let steps = [ - sequenceStep(kind: "tap", x: 1, y: 1), - sequenceStep(kind: "longPress", x: 2, y: 2), - sequenceStep(kind: "tap", x: 3, y: 3), - ] - var calls: [Int] = [] - let execution = assembleSequenceExecution(steps: steps) { index, _ in - calls.append(index) - return SequenceStepOutcome( - outcome: .performed, - gestureStartUptimeMs: Double(index * 10), - gestureEndUptimeMs: Double(index * 10 + 5) - ) - } - XCTAssertEqual(calls, [0, 1, 2]) - XCTAssertEqual(execution.completedSteps, 3) - XCTAssertNil(execution.failedStepIndex) - XCTAssertEqual(execution.results.map { $0.kind }, ["tap", "longPress", "tap"]) - XCTAssertEqual(execution.gestureStartUptimeMs, 0) - XCTAssertEqual(execution.gestureEndUptimeMs, 25) - } - - func testAssembleSequenceStopsAtFirstFailure() { - let steps = [ - sequenceStep(kind: "tap", x: 1, y: 1), - sequenceStep(kind: "longPress", x: 2, y: 2), - sequenceStep(kind: "tap", x: 3, y: 3), - ] - var calls: [Int] = [] - let execution = assembleSequenceExecution(steps: steps) { index, _ in - calls.append(index) - if index == 1 { - return SequenceStepOutcome( - outcome: .unsupported(message: "long press unsupported", hint: nil), - gestureStartUptimeMs: 10, - gestureEndUptimeMs: 15 - ) - } - return SequenceStepOutcome(outcome: .performed, gestureStartUptimeMs: 0, gestureEndUptimeMs: 5) - } - // Step 2 is never invoked. - XCTAssertEqual(calls, [0, 1]) - XCTAssertEqual(execution.completedSteps, 1) - XCTAssertEqual(execution.failedStepIndex, 1) - // results.count == completedSteps + 1 (the failed step). - XCTAssertEqual(execution.results.count, 2) - XCTAssertEqual(execution.results[1].ok, false) - XCTAssertEqual(execution.results[1].errorCode, "UNSUPPORTED_OPERATION") - XCTAssertEqual(execution.results[1].errorMessage, "long press unsupported") - } - - func testSequenceWorstCaseResponseStaysUnderJournalCap() throws { - let longMessage = String(repeating: "e", count: 200) - let results = (0..<20).map { index in - SequenceStepResult( - ok: index < 19, - kind: "longPress", - errorCode: index < 19 ? nil : "UNSUPPORTED_OPERATION", - errorMessage: index < 19 ? nil : longMessage, - gestureStartUptimeMs: 123456.789, - gestureEndUptimeMs: 123466.789 - ) - } - let response = Response( - ok: true, - data: DataPayload( - message: "sequence", - completedSteps: 19, - failedStepIndex: 19, - sequenceResults: results - ) - ) - let encoded = try JSONEncoder().encode(response) - XCTAssertLessThan(encoded.count, 16 * 1024) - } - - private func sequenceStep( - kind: String, - x: Double?, - y: Double? = nil, - synthesized: Bool? = nil - ) -> SequenceStep { - SequenceStep( - kind: kind, - x: x, - y: y, - durationMs: nil, - pauseMs: nil, - synthesized: synthesized - ) - } - - /// Validation runs before any executor call, so the INVALID_ARGS paths are exercised without - /// reaching the device executor (which is never invoked when validation rejects). - private func executeSequenceForTest(steps: [SequenceStep]) -> Response { - let command = makeSequenceCommand(steps: steps) - return executeSequence(command: command, activeApp: app) - } - - /// Build a sequence Command via JSON so the test does not depend on the memberwise init's - /// parameter order. - private func makeSequenceCommand(steps: [SequenceStep]) -> Command { - struct SequenceCommandFixture: Encodable { - let command = "sequence" - let commandId = "seq-test" - let steps: [SequenceStep] - } - let data = try! JSONEncoder().encode(SequenceCommandFixture(steps: steps)) - return try! JSONDecoder().decode(Command.self, from: data) - } -} -#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift index 40b23e6a08..b21d55b1da 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift @@ -2,15 +2,15 @@ import XCTest import AgentDeviceSnapshotPresentation extension RunnerTests { - private static let axSnapshotErrorCode = "IOS_AX_SNAPSHOT_FAILED" - private static let axSnapshotFailureMessage = + static let axSnapshotErrorCode = "IOS_AX_SNAPSHOT_FAILED" + static let axSnapshotFailureMessage = "iOS XCTest snapshot failed while serializing the accessibility tree." - private static let axSnapshotUnavailableReason = "ax_snapshot_unavailable" - private static let axSnapshotHint = + static let axSnapshotUnavailableReason = "ax_snapshot_unavailable" + static let axSnapshotHint = "Snapshot state is unavailable because XCTest could not serialize this iOS accessibility tree. This can be specific to the current screen. Use plain screenshot, not screenshot --overlay-refs, as visual truth; navigate with coordinate commands if needed; then retry snapshot -i after reaching another screen. If you own the app and need full-tree inspection, simplify this screen's accessibility tree and expose stable ids on actionable controls." - private static let rawSnapshotTooLargeCode = "IOS_RAW_SNAPSHOT_TOO_LARGE" - private static let rawSnapshotMaxNodes = 5_000 - private static let rawSnapshotTooLargeHint = + static let rawSnapshotTooLargeCode = "IOS_RAW_SNAPSHOT_TOO_LARGE" + static let rawSnapshotMaxNodes = 5_000 + static let rawSnapshotTooLargeHint = "Raw iOS snapshot exceeded the runner payload guard. Use regular snapshot for visible UI, or scope/depth-limit raw snapshot when inspecting a large accessibility tree." // Runaway guard for the regular tree walk: a work bound only. A screen that trips it raises this // number in ADR 0004's name rather than bounding the walk by geometry again. @@ -21,9 +21,8 @@ extension RunnerTests { struct SnapshotTraversalContext { let queryRoot: XCUIElement let rootSnapshot: XCUIElementSnapshot - let viewport: CGRect - /** Which way the app's interface is turned from the device's native space (#2612). */ - let interfaceOrientation: Int + /** Carries which way the app's interface is turned from the device's native space (#2612). */ + let viewport: SnapshotViewport /** * The keyboard band this capture measured, published beside the tree so the daemon's tap guard * measures against the producer's own reading rather than a band it derives from these rects @@ -32,7 +31,7 @@ extension RunnerTests { let keyboardBand: RunnerKeyboardBandFact? } - private struct SnapshotEvaluation { + struct SnapshotEvaluation { let label: String let identifier: String let valueText: String? @@ -117,7 +116,7 @@ extension RunnerTests { "ScrollView" ] - private static let collapsedTabCandidateTypes: Set = [ + static let collapsedTabCandidateTypes: Set = [ .button, .link, .menuItem, @@ -132,6 +131,20 @@ extension RunnerTests { ] static let flatInteractiveFallbackBudget: TimeInterval = 1.0 + /// The least slice time a sweep query may start with. XCTest cannot cancel a query, so one that + /// starts later outlives the slice its caller waits for and holds the main thread (#2783). + static let flatInteractiveQueryBudget: TimeInterval = 0.1 + + /// The deadline the query-sweep tier's caller waits for: one slice, clamped to the plan deadline. + /// Interactive and non-interactive requests share it, since the caller discards a later result. + static func querySweepSliceDeadline(startedAt: Date, planDeadline: Date) -> Date { + min(startedAt.addingTimeInterval(flatInteractiveFallbackBudget), planDeadline) + } + + /// Whether a sweep query started at `now` can still finish before the slice `deadline`. + static func querySweepCanStartQuery(deadline: Date, now: Date) -> Bool { + deadline.timeIntervalSince(now) >= flatInteractiveQueryBudget + } /// What one capture may spend reading the keyboard band before it gives up on the fact and lets the /// tap guard fall back to the tree rule. The scroll path pays this query per gesture and stays well @@ -145,14 +158,17 @@ extension RunnerTests { // `boundedBlockingSystemAlertSnapshot`'s probe closure (see `systemModalProbeOverrideForTesting` // in RunnerTests.swift), so reverting this entry point to bypass the bounded probe fails the // regression test. - func snapshotFast(app: XCUIApplication, options: PresentationOptions) throws -> DataPayload { + func snapshotFast(target: SnapshotCaptureTarget, options: PresentationOptions) throws -> DataPayload { let deadline = Date().addingTimeInterval(Self.snapshotPlanBudget) - if let blocking = boundedBlockingSystemAlertSnapshot(deadline: deadline) { + if let blocking = boundedBlockingSystemAlertSnapshot( + deadline: deadline, + penaltyTarget: .prepared(bundleId: target.bundleId) + ) { return blocking } return try runSnapshotCapturePlan( Self.regularVisiblePlan, - app: app, + target: target, options: options, terminal: .sparseWithFatalOnAXFailure, deadline: deadline @@ -261,20 +277,22 @@ extension RunnerTests { nodes: nodes, truncated: false, effectiveDepth: nil, - viewport: context.viewport, - interfaceOrientation: context.interfaceOrientation + viewport: context.viewport ) } // See `snapshotFast` above: the single production entry point, no unit-test overload. - func snapshotRaw(app: XCUIApplication, options: PresentationOptions) throws -> DataPayload { + func snapshotRaw(target: SnapshotCaptureTarget, options: PresentationOptions) throws -> DataPayload { let deadline = Date().addingTimeInterval(Self.snapshotPlanBudget) - if let blocking = boundedBlockingSystemAlertSnapshot(deadline: deadline) { + if let blocking = boundedBlockingSystemAlertSnapshot( + deadline: deadline, + penaltyTarget: .prepared(bundleId: target.bundleId) + ) { return blocking } return try runSnapshotCapturePlan( Self.rawDiagnosticPlan, - app: app, + target: target, options: options, terminal: .throwOnAXFailure, deadline: deadline @@ -283,8 +301,15 @@ extension RunnerTests { /// Runs the pre-plan SpringBoard system-modal probe as a bounded capture tier sharing the plan /// deadline, so a slow alert enumeration cannot bypass the snapshot timeout and stall (#1244). - func boundedBlockingSystemAlertSnapshot(deadline: Date) -> DataPayload? { - boundedBlockingSystemAlertSnapshotBody(deadline: deadline) { probeDeadline in + /// An abandoned probe penalizes the XCTest channel for `penaltyTarget`. + func boundedBlockingSystemAlertSnapshot( + deadline: Date, + penaltyTarget: SnapshotProbePenaltyTarget + ) -> DataPayload? { + boundedBlockingSystemAlertSnapshotBody( + deadline: deadline, + penaltyTarget: penaltyTarget + ) { probeDeadline in #if AGENT_DEVICE_RUNNER_UNIT_TESTS if let override = self.systemModalProbeOverrideForTesting { return override(probeDeadline) @@ -301,6 +326,7 @@ extension RunnerTests { /// production runs and what the unit tests exercise. private func boundedBlockingSystemAlertSnapshotBody( deadline: Date, + penaltyTarget: SnapshotProbePenaltyTarget, probe: @escaping (Date) -> DataPayload? ) -> DataPayload? { #if os(macOS) @@ -316,6 +342,7 @@ extension RunnerTests { } let probeDeadline = Date().addingTimeInterval(slice) let startedAt = Date() + let penaltyIdentity = SnapshotProbePenaltyIdentity(penaltyTarget) do { return try runMainThreadWork( "system_modal_probe", @@ -329,12 +356,13 @@ extension RunnerTests { }, onAbandoned: { self.penalizeSnapshotXCTestChannel( - bundleId: self.currentBundleId, + bundleId: penaltyIdentity.penalizedBundleId, reason: "system_modal_probe_timeout" ) } ) { - probe(probeDeadline) + penaltyIdentity.captureFromMain(bundleId: self.currentBundleId) + return probe(probeDeadline) } } catch { NSLog( @@ -405,45 +433,40 @@ extension RunnerTests { nodes: nodes, truncated: false, effectiveDepth: nil, - viewport: context.viewport, - interfaceOrientation: context.interfaceOrientation + viewport: context.viewport ) } func querySweepSnapshotAcquisition( app: XCUIApplication, hint: CaptureHint, - planDeadline: Date = .distantFuture - ) -> SnapshotAcquisition { + sliceDeadline deadline: Date + ) -> (acquisition: SnapshotAcquisition, outcome: SnapshotTierOutcome) { var nodes: [RawAXNode] = [ interactiveRootNode(rect: .zero) ] if hint.rawTraversalDepth == 0 || hint.regularPresentedDepth == 0 { - return SnapshotAcquisition( - hint: hint, - nodes: nodes, - truncated: false, - effectiveDepth: nil, - viewport: .infinite, - interfaceOrientation: RunnerInterfaceOrientation.unknown + return ( + SnapshotAcquisition( + hint: hint, + nodes: nodes, + truncated: false, + effectiveDepth: nil, + viewport: .missing(reason: .notProvided) + ), + .completed ) } - // Bounded by both its own sweep budget and the umbrella capture-plan deadline, so a - // chained recovery tier can never push the plan past the main-thread watchdog (#1105). - let sweepDeadline = hint.interactiveOnly - ? Date().addingTimeInterval(Self.flatInteractiveFallbackBudget) - : Date.distantFuture - let deadline = min(sweepDeadline, planDeadline) - let viewport = safeSnapshotViewport(app: app) + let viewport = safeSnapshotViewport(app: app, readingOrientation: false) var seen = Set() var candidates: [RawAXNode] = [] let flatElements = flatInteractiveElements(app: app, deadline: deadline) - var truncated = flatElements.truncated + var outcome = flatElements.outcome for element in flatElements.elements { - if Date() >= deadline { + if !Self.querySweepCanStartQuery(deadline: deadline, now: Date()) { NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_FLAT_FALLBACK_DEADLINE") - truncated = true + outcome = .deadlineExhausted break } guard let node = flatSnapshotNode(element: element, index: 0, parentIndex: 0) else { @@ -465,11 +488,9 @@ extension RunnerTests { } // The synthetic root doubles as the daemon's viewport (find.ts prefers on-screen matches - // inside nodes[0].rect): use the real screen viewport when capture produced a finite one, - // so off-screen candidates can never inflate the root and masquerade as on-screen. - let rootRect = viewport.isInfinite || viewport.isNull || viewport.isEmpty - ? interactiveRootFrame(for: candidates) - : viewport + // inside nodes[0].rect): use the real screen viewport when the capture resolved one, so + // off-screen candidates can never inflate the root and masquerade as on-screen. + let rootRect = viewport.rect ?? interactiveRootFrame(for: candidates) nodes[0] = interactiveRootNode(rect: rootRect) for candidate in candidates { nodes.append( @@ -491,26 +512,30 @@ extension RunnerTests { ) ) } - return SnapshotAcquisition( - hint: hint, - nodes: nodes, - truncated: truncated, - effectiveDepth: nil, - viewport: viewport, - interfaceOrientation: RunnerInterfaceOrientation.unknown + return ( + SnapshotAcquisition( + hint: hint, + nodes: nodes, + truncated: outcome == .deadlineExhausted, + effectiveDepth: nil, + viewport: viewport + ), + outcome ) } func snapshotAccessibilityUnavailable(failure: SnapshotCaptureFailure) -> DataPayload { NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_AX_UNAVAILABLE=%@", failure.message) - runnerAccessibilityHealth = .unavailable - invalidateCachedTarget(reason: Self.axSnapshotUnavailableReason) + applyMainOwnedSnapshotState("ax_unavailable_invalidation") { + self.runnerAccessibilityHealth = .unavailable + self.invalidateCachedTarget(reason: Self.axSnapshotUnavailableReason) + } // This is a planned terminal result, so it carries the structured verdict like every other // planned snapshot — downstream sparse handling keys off the verdict, not node shapes. return sparseTruncatedSnapshotPayload( message: recoveredSnapshotMessage(failure), snapshotQuality: SnapshotQuality( - state: "sparse", + state: .sparse, backend: SnapshotBackendKind.recursiveTree.rawValue, reason: failure.message, reasonCode: "ax-rejected", @@ -523,11 +548,11 @@ extension RunnerTests { ) } - private func recoveredSnapshotMessage(_ failure: SnapshotCaptureFailure) -> String { + func recoveredSnapshotMessage(_ failure: SnapshotCaptureFailure) -> String { return "\(failure.message) Hint: \(failure.hint)" } - private func rawSnapshotTooLargeFailure(nodeCount: Int) -> SnapshotCaptureFailure { + func rawSnapshotTooLargeFailure(nodeCount: Int) -> SnapshotCaptureFailure { SnapshotCaptureFailure( code: Self.rawSnapshotTooLargeCode, message: "iOS raw snapshot exceeded \(Self.rawSnapshotMaxNodes) nodes while walking node \(nodeCount).", @@ -558,803 +583,4 @@ extension RunnerTests { runnerFatalReason: runnerFatalReason ) } - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS - func testSnapshotAccessibilityUnavailableMarksSparseSnapshotRunnerFatal() { - currentApp = app - currentBundleId = "com.example.app" - - let payload = snapshotAccessibilityUnavailable( - failure: SnapshotCaptureFailure( - code: Self.axSnapshotErrorCode, - message: Self.axSnapshotFailureMessage, - hint: Self.axSnapshotHint - ) - ) - - XCTAssertEqual(payload.message, "\(Self.axSnapshotFailureMessage) Hint: \(Self.axSnapshotHint)") - XCTAssertEqual(payload.nodes?.count, 1) - XCTAssertEqual(payload.nodes?.first?.type, "Application") - XCTAssertEqual(payload.truncated, true) - XCTAssertEqual(payload.runnerFatal, true) - XCTAssertEqual(payload.runnerFatalReason, Self.axSnapshotUnavailableReason) - // The planned terminal result carries the structured verdict like every other planned - // snapshot — downstream sparse handling keys off it, not off node shapes. - XCTAssertEqual(payload.snapshotQuality?.state, "sparse") - XCTAssertEqual(payload.snapshotQuality?.reasonCode, "ax-rejected") - XCTAssertEqual(payload.snapshotQuality?.reason, Self.axSnapshotFailureMessage) - XCTAssertNil(currentApp) - XCTAssertNil(currentBundleId) - } - - func testRecoveredSnapshotMessagePreservesHint() { - let message = recoveredSnapshotMessage( - SnapshotCaptureFailure( - code: Self.axSnapshotErrorCode, - message: Self.axSnapshotFailureMessage, - hint: Self.axSnapshotHint - ) - ) - - XCTAssertTrue(message.contains(Self.axSnapshotFailureMessage)) - XCTAssertTrue(message.contains(Self.axSnapshotHint)) - } - - func testRawSnapshotTooLargeFailureIsStructured() { - let failure = rawSnapshotTooLargeFailure(nodeCount: Self.rawSnapshotMaxNodes + 1) - - XCTAssertEqual(failure.code, Self.rawSnapshotTooLargeCode) - XCTAssertTrue(failure.message.contains("\(Self.rawSnapshotMaxNodes) nodes")) - XCTAssertEqual(failure.hint, Self.rawSnapshotTooLargeHint) - } - - func testSystemModalProbeSliceSharesAndClampsToPlanDeadline() { - // Fresh plan deadline: the probe gets its full dedicated budget. - XCTAssertEqual(Self.systemModalProbeSlice(budget: 4, deadlineRemaining: 20), 4) - // Nearly-spent plan deadline: the probe is clamped so it can't run past the shared budget. - XCTAssertEqual(Self.systemModalProbeSlice(budget: 4, deadlineRemaining: 1.5), 1.5) - // Exactly/already exhausted deadline: skip the probe entirely (0), never a negative timeout. - XCTAssertEqual(Self.systemModalProbeSlice(budget: 4, deadlineRemaining: 0), 0) - XCTAssertEqual(Self.systemModalProbeSlice(budget: 4, deadlineRemaining: -5), 0) - } - - // Simulator-only: the bounded probe body returns nil on macOS (no SpringBoard host), so the - // timeout/penalty/drain machinery below only exists on the iOS branch. -#if os(iOS) - /// Regression for #1244/#1248: drives the bounded system-modal probe through a real, - /// production-only command entry point (`snapshotFast` or `snapshotRaw` -- see the two test - /// methods below), not `boundedBlockingSystemAlertSnapshot` directly, with - /// `systemModalProbeOverrideForTesting` set to a closure that blocks past the probe's real - /// slice, forcing a real `runMainThreadWork` timeout. This is revert-sensitive on both halves - /// of the fix, for either entry point: - /// - if the entry point reverted to calling the unbounded `blockingSystemAlertSnapshot` - /// directly (or dropped the `runMainThreadWork` wrap), nothing here would ever time out, - /// so the mid-flight busy/penalty assertions below would never be met; - /// - if the `onAbandoned` penalty hook or the abandoned-work accounting were dropped, the - /// timeout would still fire, but the busy/penalty and drain assertions would not hold. - /// - /// The drain assertion is synchronized on the *real* release rather than raced: after - /// signaling the probe to finish, the background queue polls `hasAbandonedMainThreadWork()` - /// (bounded) and only then fulfills `drained`, which the test `wait(for:timeout:)`s on before - /// asserting `.idle`/`hasAbandonedMainThreadWork() == false` below -- so a slow drain fails that - /// assertion instead of racing a fixed-timing guess. - private func assertBoundedSystemModalProbeTimeoutRecoversThenReleasesOnDrain( - entryPointName: String, - callEntryPoint: @escaping (XCUIApplication, PresentationOptions) throws -> DataPayload - ) { - let targetBundleId = "com.callstack.agentdevice.runner.missing.snapshot-timeout-test" - let snapshotTarget = XCUIApplication(bundleIdentifier: targetBundleId) - let probeReleaseGate = DispatchSemaphore(value: 0) - currentApp = snapshotTarget - currentBundleId = targetBundleId - defer { - probeReleaseGate.signal() - currentApp = nil - currentBundleId = nil - systemModalProbeOverrideForTesting = nil - clearSnapshotXCTestChannelPenalty(reason: "test-cleanup") - } - - final class ResultBox { - var payload: DataPayload? - var wasBusyBeforeDrain = false - var hadAbandonedCaptureBeforeDrain = false - var wasPenalizedBeforeDrain = false - } - let box = ResultBox() - // The test owns release of the injected probe. A fixed timeout races the capture plan's - // independent fallback tiers on loaded CI hosts and can drain before the test records the - // abandoned-work state. The defer above still releases the probe if an earlier assertion or - // expectation fails. - systemModalProbeOverrideForTesting = { _ in - probeReleaseGate.wait() - return nil - } - - let completion = expectation( - description: "\(entryPointName) recovered while the probe was abandoned, then released it" - ) - let drained = expectation(description: "\(entryPointName) modal probe drained") - DispatchQueue(label: "agent-device.runner.tests.modal-probe-timeout").async { - box.payload = try? callEntryPoint( - snapshotTarget, - PresentationOptions(interactiveOnly: false, depth: nil, scope: nil, raw: false) - ) - - // 1) Penalty/busy accounting: must already be in place by the time the entry point - // returns, well before we release the still-blocked probe below. - if case .busy = self.currentMainThreadBusyState() { - box.wasBusyBeforeDrain = true - } - box.hadAbandonedCaptureBeforeDrain = self.hasAbandonedMainThreadWork() - box.wasPenalizedBeforeDrain = self.isSnapshotXCTestChannelPenalized(bundleId: self.currentBundleId) - - // 2) `box.payload` above was already produced -- through the capture plan's recovery - // tiers -- while the probe is still blocked on `probeReleaseGate`, i.e. recovered before - // drain, not queued behind it. - completion.fulfill() - - // 3) Only now let the abandoned probe finish, then block this queue (never the test's - // main-thread wait) on the *real* drain signal -- the abandoned-work count reaching zero - // -- bounded so a revert that never drains fulfills `drained` anyway and lets the - // assertions below report the regression explicitly instead of just timing out. - probeReleaseGate.signal() - let drainDeadline = Date().addingTimeInterval(5) - while self.hasAbandonedMainThreadWork(), Date() < drainDeadline { - self.sleepFor(0.002) - } - drained.fulfill() - } - - wait(for: [completion], timeout: 15) - - // 1) Penalty/busy accounting. - XCTAssertTrue( - box.wasBusyBeforeDrain, - "expected RUNNER_BUSY while the \(entryPointName) modal probe timeout is outstanding" - ) - XCTAssertTrue( - box.hadAbandonedCaptureBeforeDrain, - "onAbandoned must retain the abandoned XCTest channel work for \(entryPointName)" - ) - XCTAssertTrue( - box.wasPenalizedBeforeDrain, - "a timed-out modal probe must penalize the XCTest snapshot channel for \(entryPointName)" - ) - - // 2) Recovered response before drain. - XCTAssertNotNil( - box.payload, - "\(entryPointName) must recover a payload through the capture plan while the probe drains" - ) - - // 3) Bounded, deterministic drain barrier, then release assertions. - wait(for: [drained], timeout: 6) - guard case .idle = currentMainThreadBusyState() else { - return XCTFail("expected the runner to be idle once the abandoned \(entryPointName) probe drained") - } - XCTAssertFalse( - hasAbandonedMainThreadWork(), - "the drained probe must release the main thread for \(entryPointName)" - ) - } - - func testBoundedSystemModalProbeTimeoutRecoversThenReleasesOnDrain() { - assertBoundedSystemModalProbeTimeoutRecoversThenReleasesOnDrain(entryPointName: "snapshotFast") { - target, options in - try self.snapshotFast(app: target, options: options) - } - } - - func testBoundedSystemModalProbeTimeoutRecoversThenReleasesOnDrainForSnapshotRaw() { - assertBoundedSystemModalProbeTimeoutRecoversThenReleasesOnDrain(entryPointName: "snapshotRaw") { - target, options in - try self.snapshotRaw(app: target, options: options) - } - } -#endif - - func testDispatchRecoverySkipsBookkeepingWhileXCTestChannelOccupied() { - // The #1244 recovery shape: the modal probe abandoned an XCTest query that is still grinding on - // main, the capture recovered independently, and its response is ready. The recovery loop must - // return it without re-entering the main queue for recorded-failure/retry bookkeeping (that hop - // would block behind the abandoned query and re-stall the command), and a later command must - // still see the runner busy until the abandoned work drains. Removing the guard regresses this. - let command = try! JSONDecoder().decode( - Command.self, - from: Data(#"{"command":"snapshot","commandId":"recovery-guard"}"#.utf8) - ) - let recovered = Response(ok: false, error: ErrorPayload(message: "target is not available")) - - setAbandonedMainThreadWork(1) - defer { setAbandonedMainThreadWork(0) } - guard case .busy = currentMainThreadBusyState() else { - return XCTFail("expected RUNNER_BUSY while abandoned XCTest work is outstanding") - } - - var occupiedCalls = 0 - let occupied = try! executeDispatchedWithRecovery(command: command) { - occupiedCalls += 1 - return recovered - } - XCTAssertEqual(occupiedCalls, 1, "recovered response must not retry behind abandoned XCTest work") - XCTAssertEqual(occupied.ok, false) - - setAbandonedMainThreadWork(0) - guard case .idle = currentMainThreadBusyState() else { - return XCTFail("runner should be idle once the abandoned work drained") - } - var drainedCalls = 0 - _ = try! executeDispatchedWithRecovery(command: command) { - drainedCalls += 1 - return recovered - } - XCTAssertEqual(drainedCalls, 2, "with the channel free the read-only retry runs once") - } - - private func setAbandonedMainThreadWork(_ count: Int) { - mainThreadWorkLock.lock() - abandonedMainThreadWorkCount = count - abandonedMainThreadWorkSince = count > 0 ? Date(timeIntervalSinceNow: -1) : nil - mainThreadWorkLock.unlock() - } -#endif - - private func interactiveRootNode(rect: CGRect) -> RawAXNode { - RawAXNode( - index: 0, - type: "Application", - label: nil, - identifier: nil, - value: nil, - rect: SnapshotRect(rect), - enabled: true, - focused: nil, - selected: nil, - hittable: false, - depth: 0, - parentIndex: nil, - hiddenContentAbove: nil, - hiddenContentBelow: nil - ) - } - - private func interactiveRootFrame(for candidates: [RawAXNode]) -> CGRect { - guard !candidates.isEmpty else { - return .zero - } - let maxX = candidates.map { CGFloat($0.rect.x + $0.rect.width) }.max() ?? 0 - let maxY = candidates.map { CGFloat($0.rect.y + $0.rect.height) }.max() ?? 0 - return CGRect(x: 0, y: 0, width: max(1, maxX), height: max(1, maxY)) - } - - // MARK: - Snapshot Filtering - - func makeSnapshotTraversalContext( - app: XCUIApplication, - hint: CaptureHint, - captureDeadline: Date = .distantFuture, - treeCaptureSliceBudgetOverride: TimeInterval? = nil - ) throws -> SnapshotTraversalContext? { - // The viewport and the interface orientation are one hop: geometry that arrives in the device's - // native space can only be placed relative to the app's own frame and rotation, and asking for - // the pair twice would read them at two different moments of a rotation. - let geometry = try runMainThreadWork( - "snapshot_viewport", - timeout: min(1.0, max(0.1, captureDeadline.timeIntervalSinceNow)), - timeoutError: snapshotMainThreadTimeoutError("preparing tree snapshot") - ) { - ( - viewport: self.safeSnapshotViewport(app: app), - interfaceOrientation: self.capturedInterfaceOrientation(app: app) - ) - } - let viewport = geometry.viewport - let interfaceOrientation = geometry.interfaceOrientation - let treeSliceBudget = treeCaptureSliceBudgetOverride ?? treeCaptureSliceBudget - let slice = min(treeSliceBudget, max(0.5, captureDeadline.timeIntervalSinceNow)) - guard let rootSnapshot = try captureSnapshotRootBounded(app, sliceSeconds: slice) else { - return nil - } - - // Read after the tree, so the band is never older than the tree it will be compared against: a - // keyboard that appeared while the tree was being captured would otherwise publish `absent` - // beside key nodes that the tap guard would then have to trust less than the absence (#2660). - // It keeps its own hop and slice rather than joining the geometry pair above, because a keyboard - // this capture cannot measure must cost the fact and not the tree tier behind it. - let keyboardBand = captureKeyboardBandFact(app: app, deadline: captureDeadline) - - return SnapshotTraversalContext( - queryRoot: app, - rootSnapshot: rootSnapshot, - viewport: viewport, - interfaceOrientation: interfaceOrientation, - keyboardBand: keyboardBand - ) - } - - static let xCTestSnapshotTimeoutCode = "IOS_TREE_CAPTURE_TIMEOUT" - - /// Runs the blocking tree-snapshot XPC on the main thread bounded by `sliceSeconds`. On - /// timeout the XPC keeps running on main (it cannot be cancelled); the capture is marked - /// abandoned so plans avoid XCTest-backed tiers until it drains, the timed-out attempt - /// penalizes the tree backend for this bundle (unless the fresh-process warmup exemption - /// applies), and the plan moves to the platform's independent recovery tier when one exists - /// (#1105/#1122). - private func captureSnapshotRootBounded( - _ element: XCUIElement, - sliceSeconds: TimeInterval - ) throws -> XCUIElementSnapshot? { - if Thread.isMainThread { - return try captureSnapshotRoot(element) - } - return try runMainThreadWork( - "tree_capture", - timeout: sliceSeconds, - timeoutError: treeCaptureTimeoutError(sliceSeconds: sliceSeconds) - ) { - try self.captureSnapshotRoot(element) - } - } - - private func treeCaptureTimeoutError(sliceSeconds: TimeInterval) -> () -> Error { - { - SnapshotCaptureFailure( - code: Self.xCTestSnapshotTimeoutCode, - message: "the XCTest tree capture exceeded its \(Int(sliceSeconds))s time slice", - hint: "The capture plan will avoid or tightly bound XCTest-backed snapshot tiers on this screen." - ) - } - } - - func snapshotMainThreadTimeoutError(_ operation: String) -> () -> Error { - { - SnapshotCaptureFailure( - code: Self.xCTestSnapshotTimeoutCode, - message: "timed out while \(operation) on the XCTest main thread", - hint: "The capture plan will skip XCTest-backed snapshot tiers while the previous main-thread work drains." - ) - } - } - - private func captureSnapshotRoot(_ element: XCUIElement) throws -> XCUIElementSnapshot? { - var rootSnapshot: XCUIElementSnapshot? - var swiftErrorMessage: String? - let exceptionMessage = RunnerObjCExceptionCatcher.catchException({ - do { - rootSnapshot = try element.snapshot() - } catch { - swiftErrorMessage = describeSnapshotError(error) - } - }) - - if let rootSnapshot { - return rootSnapshot - } - let message = exceptionMessage ?? swiftErrorMessage ?? "snapshot returned no root" - if Self.isAxIllegalArgument(message) { - throw axSnapshotFailure(message) - } - return nil - } - - func safeSnapshotViewport(app: XCUIApplication) -> CGRect { - safely("SNAPSHOT_VIEWPORT", CGRect.infinite) { snapshotViewport(app: app) } - } - - private func describeSnapshotError(_ error: Error) -> String { - let localized = error.localizedDescription - let debug = String(describing: error) - if localized.isEmpty { return debug } - if debug == localized { return localized } - return "\(localized) (\(debug))" - } - - private func axSnapshotFailure(_ message: String) -> SnapshotCaptureFailure { - let detail = message.trimmingCharacters(in: .whitespacesAndNewlines) - let failureMessage: String - if detail.isEmpty { - failureMessage = Self.axSnapshotFailureMessage - } else { - failureMessage = "\(Self.axSnapshotFailureMessage) \(detail)" - } - return SnapshotCaptureFailure( - code: Self.axSnapshotErrorCode, - message: failureMessage, - hint: Self.axSnapshotHint - ) - } - - private static func isAxIllegalArgument(_ message: String) -> Bool { - let normalized = message.lowercased() - return normalized.contains("kaxerrorillegalargument") - || (normalized.contains("illegal argument") && normalized.contains("snapshot")) - } - - static func isAxSnapshotFailure(_ failure: SnapshotCaptureFailure) -> Bool { - failure.code == Self.axSnapshotErrorCode || isAxIllegalArgument(failure.message) - } - - private func evaluateSnapshot(_ snapshot: XCUIElementSnapshot) -> SnapshotEvaluation { - let label = aggregatedLabel(for: snapshot) ?? snapshot.label.trimmingCharacters(in: .whitespacesAndNewlines) - let identifier = snapshot.identifier.trimmingCharacters(in: .whitespacesAndNewlines) - let valueText = snapshotValueText(snapshot) - return SnapshotEvaluation( - label: label, - identifier: identifier, - valueText: valueText, - focused: snapshotHasFocus(snapshot), - selected: snapshotIsSelected(snapshot) - ) - } - - private func makeSnapshotNode( - snapshot: XCUIElementSnapshot, - evaluation: SnapshotEvaluation, - depth: Int, - index: Int, - parentIndex: Int? - ) -> RawAXNode { - // Acquisition carries the frame the platform reported; `SnapshotGeometrySpace.normalized` turns - // it into the app's orientation space and recomputes `hittable` from that one pass (#2661). - return RawAXNode( - index: index, - type: elementTypeName(snapshot.elementType), - label: evaluation.label.isEmpty ? nil : evaluation.label, - identifier: evaluation.identifier.isEmpty ? nil : evaluation.identifier, - value: evaluation.valueText, - rect: SnapshotRect(snapshot.frame), - enabled: snapshot.isEnabled, - focused: evaluation.focused ? true : nil, - selected: evaluation.selected ? true : nil, - hittable: false, - depth: depth, - parentIndex: parentIndex, - hiddenContentAbove: nil, - hiddenContentBelow: nil - ) - } - - /// The app's own interface orientation: the fact that names which way the device's native space is - /// turned from the space the capture publishes. Unreadable or unnamed means no rotation. - func capturedInterfaceOrientation(app: XCUIApplication) -> Int { - safely("SNAPSHOT_INTERFACE_ORIENTATION", RunnerInterfaceOrientation.unknown) { - Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)) - } - } - - private func snapshotValueText(_ snapshot: XCUIElementSnapshot) -> String? { - guard let value = snapshot.value else { return nil } - let text = String(describing: value).trimmingCharacters(in: .whitespacesAndNewlines) - return text.isEmpty ? nil : text - } - - private func snapshotViewport(app: XCUIApplication) -> CGRect { - let appFrame = app.frame - if !appFrame.isNull && !appFrame.isEmpty { - return appFrame - } - return .infinite - } - - static func snapshotTraversalIdentity( - elementType: XCUIElement.ElementType, - label: String, - identifier: String, - frame: CGRect - ) -> String { - #if os(iOS) - "\(elementType)-\(label)-\(identifier)-\(frame.origin.x)-\(frame.origin.y)-\(frame.width)-\(frame.height)" - #else - return "\(elementType)-\(label)-\(identifier)-\(frame.origin.x)-\(frame.origin.y)" - #endif - } - - private func aggregatedLabel(for snapshot: XCUIElementSnapshot, depth: Int = 0) -> String? { - if depth > 4 { return nil } - let text = snapshot.label.trimmingCharacters(in: .whitespacesAndNewlines) - if !text.isEmpty { return text } - if let valueText = snapshotValueText(snapshot) { return valueText } - for child in snapshot.children { - if let childLabel = aggregatedLabel(for: child, depth: depth + 1) { - return childLabel - } - } - return nil - } - - private func appendCollapsedTabFallbackNodes( - to nodes: inout [RawAXNode], - containerSnapshot: XCUIElementSnapshot, - resolveElements: () -> [XCUIElement], - depth: Int, - parentIndex: Int - ) { - let fallbackNodes = collapsedTabFallbackNodes( - for: containerSnapshot, - resolveElements: resolveElements, - startingIndex: nodes.count, - depth: depth, - parentIndex: parentIndex - ) - nodes.append(contentsOf: fallbackNodes) - } - - private func collapsedTabFallbackNodes( - for containerSnapshot: XCUIElementSnapshot, - resolveElements: () -> [XCUIElement], - startingIndex: Int, - depth: Int, - parentIndex: Int - ) -> [RawAXNode] { - if !containerSnapshot.children.isEmpty { return [] } - // This fallback reads live element frames, which XCTest reports in the app's own space, and a - // collapsed tab container sits under the app's own window. `SnapshotGeometrySpace.normalized` - // keys rotation on ancestry, so these nodes never enter a declared native space and need no - // special case here; the containment and area rules below compare reported frames to each other. - guard shouldExpandCollapsedTabContainer(containerSnapshot) else { return [] } - let containerFrame = containerSnapshot.frame - if containerFrame.isNull || containerFrame.isEmpty { return [] } - - // Collapsed tab containers should be rare, so a full descendant scan is acceptable once per - // snapshot as a fallback for XCTest omitting the tab children from the snapshot tree. - let elements = resolveElements() - let candidates = elements.compactMap { element in - collapsedTabCandidateNode( - element: element, - containerSnapshot: containerSnapshot, - containerFrame: containerFrame - ) - } - .sorted { left, right in - if left.rect.x != right.rect.x { - return left.rect.x < right.rect.x - } - return left.rect.y < right.rect.y - } - - if candidates.count < 2 { return [] } - let rowMidpoints = candidates.map { $0.rect.y + ($0.rect.height / 2) } - let rowSpread = (rowMidpoints.max() ?? 0) - (rowMidpoints.min() ?? 0) - // Allow modest vertical jitter and short two-row wraps while still rejecting unrelated controls. - if rowSpread > max(24.0, Double(containerFrame.height) * 0.6) { return [] } - - var seen = Set() - let uniqueCandidates = candidates.filter { node in - let key = "\(node.type)-\(node.label ?? "")-\(node.identifier ?? "")-\(node.value ?? "")-\(node.rect.x)-\(node.rect.y)-\(node.rect.width)-\(node.rect.height)" - if seen.contains(key) { return false } - seen.insert(key) - return true - } - if uniqueCandidates.count < 2 { return [] } - - return uniqueCandidates.enumerated().map { offset, node in - RawAXNode( - index: startingIndex + offset, - type: node.type, - label: node.label, - identifier: node.identifier, - value: node.value, - rect: node.rect, - enabled: node.enabled, - focused: node.focused, - selected: node.selected, - hittable: node.hittable, - depth: depth, - parentIndex: parentIndex, - hiddenContentAbove: nil, - hiddenContentBelow: nil - ) - } - } - - private func collapsedTabCandidateNode( - element: XCUIElement, - containerSnapshot: XCUIElementSnapshot, - containerFrame: CGRect - ) -> RawAXNode? { - var node: RawAXNode? - let exceptionMessage = RunnerObjCExceptionCatcher.catchException({ - if !element.exists { return } - let elementType = element.elementType - if !Self.collapsedTabCandidateTypes.contains(elementType) { return } - let frame = element.frame - if frame.isNull || frame.isEmpty { return } - if frame.equalTo(containerFrame) { return } - let area = max(CGFloat(1), frame.width * frame.height) - let containerArea = max(CGFloat(1), containerFrame.width * containerFrame.height) - if area >= containerArea * 0.9 { return } - let center = CGPoint(x: frame.midX, y: frame.midY) - if !containerFrame.contains(center) { return } - - let label = element.label.trimmingCharacters(in: .whitespacesAndNewlines) - let identifier = element.identifier.trimmingCharacters(in: .whitespacesAndNewlines) - let valueText = snapshotValueText(element) - let hasContent = !label.isEmpty || !identifier.isEmpty || valueText != nil - if !hasContent { return } - if sameSemanticElement( - containerSnapshot: containerSnapshot, - elementType: elementType, - label: label, - identifier: identifier - ) { - return - } - - // The containment and area rules above compared reported frames with each other; the node - // joins the tree carrying the frame the platform reported, and normalization places it. - node = RawAXNode( - index: 0, - type: elementTypeName(elementType), - label: label.isEmpty ? nil : label, - identifier: identifier.isEmpty ? nil : identifier, - value: valueText, - rect: SnapshotRect(frame), - enabled: element.isEnabled, - focused: elementHasFocus(element) ? true : nil, - selected: element.isSelected ? true : nil, - hittable: false, - depth: 0, - parentIndex: nil, - hiddenContentAbove: nil, - hiddenContentBelow: nil - ) - }) - if let exceptionMessage { - NSLog( - "AGENT_DEVICE_RUNNER_SNAPSHOT_TAB_FALLBACK_IGNORED_EXCEPTION=%@", - exceptionMessage - ) - return nil - } - return node - } - - private func snapshotHasFocus(_ snapshot: XCUIElementSnapshot) -> Bool { - var focused = false - _ = RunnerObjCExceptionCatcher.catchException({ - if let value = (snapshot as! NSObject).value(forKey: "hasFocus") as? Bool { - focused = value - } - }) - return focused - } - - private func snapshotIsSelected(_ snapshot: XCUIElementSnapshot) -> Bool { - return snapshot.isSelected - } - - private func shouldExpandCollapsedTabContainer(_ snapshot: XCUIElementSnapshot) -> Bool { - let frame = snapshot.frame - if frame.isNull || frame.isEmpty { return false } - if frame.width < max(CGFloat(160), frame.height * 1.75) { return false } - switch snapshot.elementType { - case .tabBar, .segmentedControl, .slider: - return true - default: - return false - } - } - - private func snapshotValueText(_ element: XCUIElement) -> String? { - let text = String(describing: element.value ?? "") - .trimmingCharacters(in: .whitespacesAndNewlines) - return text.isEmpty ? nil : text - } - - private func sameSemanticElement( - containerSnapshot: XCUIElementSnapshot, - elementType: XCUIElement.ElementType, - label: String, - identifier: String - ) -> Bool { - if containerSnapshot.elementType != elementType { return false } - let containerLabel = containerSnapshot.label.trimmingCharacters(in: .whitespacesAndNewlines) - let containerIdentifier = containerSnapshot.identifier - .trimmingCharacters(in: .whitespacesAndNewlines) - return containerLabel == label && containerIdentifier == identifier - } - - private func flatInteractiveElements( - app: XCUIApplication, - deadline: Date - ) -> (elements: [XCUIElement], truncated: Bool) { - let queries: [XCUIElementQuery] = [ - app.buttons, - app.links, - app.textFields, - app.secureTextFields, - app.searchFields, - app.textViews, - app.switches, - app.sliders, - app.segmentedControls, - app.cells, - app.collectionViews, - app.tables, - app.scrollViews, - app.pickers, - app.steppers, - app.tabBars, - app.menuItems, - app.staticTexts, - app.images - ] - - var elements: [XCUIElement] = [] - var truncated = false - for query in queries { - if Date() >= deadline { - NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_FLAT_FALLBACK_DEADLINE") - truncated = true - break - } - let result = snapshotElementsQuery { - query.allElementsBoundByIndex - } - elements.append(contentsOf: result.elements) - if result.axUnavailable { - break - } - } - return (elements, truncated) - } - - private func snapshotElementsQuery( - _ fetch: () -> [XCUIElement] - ) -> (elements: [XCUIElement], axUnavailable: Bool) { - let (elements, exceptionMessage) = catchingObjCException(fallback: [], fetch) - guard let exceptionMessage else { - return (elements, false) - } - NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_QUERY_IGNORED_EXCEPTION=%@", exceptionMessage) - if Self.isAxIllegalArgument(exceptionMessage) { - invalidateCachedTarget(reason: "ax_snapshot_query_unavailable") - return ([], true) - } - return ([], false) - } - - private func flatSnapshotNode( - element: XCUIElement, - index: Int, - parentIndex: Int? - ) -> RawAXNode? { - var node: RawAXNode? - let exceptionMessage = RunnerObjCExceptionCatcher.catchException({ - if !element.exists { return } - // Declared residue: a flat element query has no hierarchy for geometryless semantics to - // attach to, so frameless elements are dropped at acquisition rather than presented. - let frame = element.frame - if frame.isNull || frame.isEmpty { return } - let label = element.label.trimmingCharacters(in: .whitespacesAndNewlines) - let identifier = element.identifier.trimmingCharacters(in: .whitespacesAndNewlines) - let valueText = snapshotValueText(element) - let elementType = element.elementType - let enabled = element.isEnabled - - node = RawAXNode( - index: index, - type: elementTypeName(elementType), - label: label.isEmpty ? nil : label, - identifier: identifier.isEmpty ? nil : identifier, - value: valueText, - rect: SnapshotRect(frame), - enabled: enabled, - focused: elementHasFocus(element) ? true : nil, - selected: element.isSelected ? true : nil, - hittable: false, - depth: 1, - parentIndex: parentIndex, - hiddenContentAbove: nil, - hiddenContentBelow: nil - ) - }) - if let exceptionMessage { - NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_FLAT_IGNORED_EXCEPTION=%@", exceptionMessage) - return nil - } - return node - } - } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift new file mode 100644 index 0000000000..7ebc82596f --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift @@ -0,0 +1,579 @@ +import XCTest +import AgentDeviceSnapshotPresentation + +extension RunnerTests { + func interactiveRootNode(rect: CGRect) -> RawAXNode { + RawAXNode( + index: 0, + type: "Application", + label: nil, + identifier: nil, + value: nil, + rect: SnapshotRect(rect), + enabled: true, + focused: nil, + selected: nil, + hittable: false, + depth: 0, + parentIndex: nil, + hiddenContentAbove: nil, + hiddenContentBelow: nil + ) + } + + func interactiveRootFrame(for candidates: [RawAXNode]) -> CGRect { + guard !candidates.isEmpty else { + return .zero + } + let maxX = candidates.map { CGFloat($0.rect.x + $0.rect.width) }.max() ?? 0 + let maxY = candidates.map { CGFloat($0.rect.y + $0.rect.height) }.max() ?? 0 + return CGRect(x: 0, y: 0, width: max(1, maxX), height: max(1, maxY)) + } + + // MARK: - Snapshot Filtering + + func makeSnapshotTraversalContext( + app: XCUIApplication, + hint: CaptureHint, + captureDeadline: Date = .distantFuture, + treeCaptureSliceBudgetOverride: TimeInterval? = nil + ) throws -> SnapshotTraversalContext? { + // The viewport and the interface orientation are one hop: geometry that arrives in the device's + // native space can only be placed relative to the app's own frame and rotation, and asking for + // the pair twice would read them at two different moments of a rotation. + let viewport = try runMainThreadWork( + "snapshot_viewport", + timeout: min(1.0, max(0.1, captureDeadline.timeIntervalSinceNow)), + timeoutError: snapshotMainThreadTimeoutError("preparing tree snapshot") + ) { + self.safeSnapshotViewport(app: app, readingOrientation: true) + } + let treeSliceBudget = treeCaptureSliceBudgetOverride ?? treeCaptureSliceBudget + let slice = min(treeSliceBudget, max(0.5, captureDeadline.timeIntervalSinceNow)) + guard let rootSnapshot = try captureSnapshotRootBounded(app, sliceSeconds: slice) else { + return nil + } + + // Read after the tree, so the band is never older than the tree it will be compared against: a + // keyboard that appeared while the tree was being captured would otherwise publish `absent` + // beside key nodes that the tap guard would then have to trust less than the absence (#2660). + // It keeps its own hop and slice rather than joining the geometry pair above, because a keyboard + // this capture cannot measure must cost the fact and not the tree tier behind it. + let keyboardBand = captureKeyboardBandFact(app: app, deadline: captureDeadline) + + return SnapshotTraversalContext( + queryRoot: app, + rootSnapshot: rootSnapshot, + viewport: viewport, + keyboardBand: keyboardBand + ) + } + + static let xCTestSnapshotTimeoutCode = "IOS_TREE_CAPTURE_TIMEOUT" + + /// Runs the blocking tree-snapshot XPC on the main thread bounded by `sliceSeconds`. On + /// timeout the XPC keeps running on main (it cannot be cancelled); the capture is marked + /// abandoned so plans avoid XCTest-backed tiers until it drains, the timed-out attempt + /// penalizes the tree backend for this bundle (unless the fresh-process warmup exemption + /// applies), and the plan moves to the platform's independent recovery tier when one exists + /// (#1105/#1122). + private func captureSnapshotRootBounded( + _ element: XCUIElement, + sliceSeconds: TimeInterval + ) throws -> XCUIElementSnapshot? { + if Thread.isMainThread { + return try captureSnapshotRoot(element) + } + return try runMainThreadWork( + "tree_capture", + timeout: sliceSeconds, + timeoutError: treeCaptureTimeoutError(sliceSeconds: sliceSeconds) + ) { + try self.captureSnapshotRoot(element) + } + } + + private func treeCaptureTimeoutError(sliceSeconds: TimeInterval) -> () -> Error { + { + SnapshotCaptureFailure( + code: Self.xCTestSnapshotTimeoutCode, + message: "the XCTest tree capture exceeded its \(Int(sliceSeconds))s time slice", + hint: "The capture plan will avoid or tightly bound XCTest-backed snapshot tiers on this screen." + ) + } + } + + func snapshotMainThreadTimeoutError(_ operation: String) -> () -> Error { + { + SnapshotCaptureFailure( + code: Self.xCTestSnapshotTimeoutCode, + message: "timed out while \(operation) on the XCTest main thread", + hint: "The capture plan will skip XCTest-backed snapshot tiers while the previous main-thread work drains." + ) + } + } + + private func captureSnapshotRoot(_ element: XCUIElement) throws -> XCUIElementSnapshot? { + var rootSnapshot: XCUIElementSnapshot? + var swiftErrorMessage: String? + let exceptionMessage = RunnerObjCExceptionCatcher.catchException({ + do { + rootSnapshot = try element.snapshot() + } catch { + swiftErrorMessage = describeSnapshotError(error) + } + }) + + if let rootSnapshot { + return rootSnapshot + } + let message = exceptionMessage ?? swiftErrorMessage ?? "snapshot returned no root" + if Self.isAxIllegalArgument(message) { + throw axSnapshotFailure(message) + } + return nil + } + + /// The viewport as a declared fact; a read that raises is `.missing(reason: .notProvided)` (#2891). + /// `readingOrientation` reads the interface orientation in the same hop, for tiers whose frames can + /// arrive in the device's native space; without it the viewport cannot anchor a rotation. + func safeSnapshotViewport(app: XCUIApplication, readingOrientation: Bool) -> SnapshotViewport { + safely("SNAPSHOT_VIEWPORT", .missing(reason: .notProvided)) { + .reported( + box: snapshotAppFrame(app: app), + interfaceOrientation: readingOrientation + ? capturedInterfaceOrientation(app: app) + : RunnerInterfaceOrientation.unknown + ) + } + } + + private func describeSnapshotError(_ error: Error) -> String { + let localized = error.localizedDescription + let debug = String(describing: error) + if localized.isEmpty { return debug } + if debug == localized { return localized } + return "\(localized) (\(debug))" + } + + private func axSnapshotFailure(_ message: String) -> SnapshotCaptureFailure { + let detail = message.trimmingCharacters(in: .whitespacesAndNewlines) + let failureMessage: String + if detail.isEmpty { + failureMessage = Self.axSnapshotFailureMessage + } else { + failureMessage = "\(Self.axSnapshotFailureMessage) \(detail)" + } + return SnapshotCaptureFailure( + code: Self.axSnapshotErrorCode, + message: failureMessage, + hint: Self.axSnapshotHint + ) + } + + private static func isAxIllegalArgument(_ message: String) -> Bool { + let normalized = message.lowercased() + return normalized.contains("kaxerrorillegalargument") + || (normalized.contains("illegal argument") && normalized.contains("snapshot")) + } + + static func isAxSnapshotFailure(_ failure: SnapshotCaptureFailure) -> Bool { + failure.code == Self.axSnapshotErrorCode || isAxIllegalArgument(failure.message) + } + + func evaluateSnapshot(_ snapshot: XCUIElementSnapshot) -> SnapshotEvaluation { + let label = aggregatedLabel(for: snapshot) ?? snapshot.label.trimmingCharacters(in: .whitespacesAndNewlines) + let identifier = snapshot.identifier.trimmingCharacters(in: .whitespacesAndNewlines) + let valueText = snapshotValueText(snapshot) + return SnapshotEvaluation( + label: label, + identifier: identifier, + valueText: valueText, + focused: snapshotHasFocus(snapshot), + selected: snapshotIsSelected(snapshot) + ) + } + + func makeSnapshotNode( + snapshot: XCUIElementSnapshot, + evaluation: SnapshotEvaluation, + depth: Int, + index: Int, + parentIndex: Int? + ) -> RawAXNode { + // Acquisition carries the frame the platform reported; `SnapshotGeometrySpace.normalized` turns + // it into the app's orientation space and recomputes `hittable` from that one pass (#2661). + return RawAXNode( + index: index, + type: elementTypeName(snapshot.elementType), + label: evaluation.label.isEmpty ? nil : evaluation.label, + identifier: evaluation.identifier.isEmpty ? nil : evaluation.identifier, + value: evaluation.valueText, + rect: SnapshotRect(snapshot.frame), + enabled: snapshot.isEnabled, + focused: evaluation.focused ? true : nil, + selected: evaluation.selected ? true : nil, + hittable: false, + depth: depth, + parentIndex: parentIndex, + hiddenContentAbove: nil, + hiddenContentBelow: nil + ) + } + + /// The app's own interface orientation: the fact that names which way the device's native space is + /// turned from the space the capture publishes. Unreadable or unnamed means no rotation. + func capturedInterfaceOrientation(app: XCUIApplication) -> Int { + safely("SNAPSHOT_INTERFACE_ORIENTATION", RunnerInterfaceOrientation.unknown) { + Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)) + } + } + + private func snapshotValueText(_ snapshot: XCUIElementSnapshot) -> String? { + guard let value = snapshot.value else { return nil } + let text = String(describing: value).trimmingCharacters(in: .whitespacesAndNewlines) + return text.isEmpty ? nil : text + } + + private func snapshotAppFrame(app: XCUIApplication) -> CGRect { +#if os(iOS) + return onScreenWindowFrame(app: app) +#else + return app.frame +#endif + } + + static func snapshotTraversalIdentity( + elementType: XCUIElement.ElementType, + label: String, + identifier: String, + frame: CGRect + ) -> String { + #if os(iOS) + "\(elementType)-\(label)-\(identifier)-\(frame.origin.x)-\(frame.origin.y)-\(frame.width)-\(frame.height)" + #else + return "\(elementType)-\(label)-\(identifier)-\(frame.origin.x)-\(frame.origin.y)" + #endif + } + + private func aggregatedLabel(for snapshot: XCUIElementSnapshot, depth: Int = 0) -> String? { + if depth > 4 { return nil } + let text = snapshot.label.trimmingCharacters(in: .whitespacesAndNewlines) + if !text.isEmpty { return text } + if let valueText = snapshotValueText(snapshot) { return valueText } + for child in snapshot.children { + if let childLabel = aggregatedLabel(for: child, depth: depth + 1) { + return childLabel + } + } + return nil + } + + func appendCollapsedTabFallbackNodes( + to nodes: inout [RawAXNode], + containerSnapshot: XCUIElementSnapshot, + resolveElements: () -> [XCUIElement], + depth: Int, + parentIndex: Int + ) { + let fallbackNodes = collapsedTabFallbackNodes( + for: containerSnapshot, + resolveElements: resolveElements, + startingIndex: nodes.count, + depth: depth, + parentIndex: parentIndex + ) + nodes.append(contentsOf: fallbackNodes) + } + + private func collapsedTabFallbackNodes( + for containerSnapshot: XCUIElementSnapshot, + resolveElements: () -> [XCUIElement], + startingIndex: Int, + depth: Int, + parentIndex: Int + ) -> [RawAXNode] { + if !containerSnapshot.children.isEmpty { return [] } + // This fallback reads live element frames, which XCTest reports in the app's own space, and a + // collapsed tab container sits under the app's own window. `SnapshotGeometrySpace.normalized` + // keys rotation on ancestry, so these nodes never enter a declared native space and need no + // special case here; the containment and area rules below compare reported frames to each other. + guard shouldExpandCollapsedTabContainer(containerSnapshot) else { return [] } + let containerFrame = containerSnapshot.frame + if containerFrame.isNull || containerFrame.isEmpty { return [] } + + // Collapsed tab containers should be rare, so a full descendant scan is acceptable once per + // snapshot as a fallback for XCTest omitting the tab children from the snapshot tree. + let elements = resolveElements() + let candidates = elements.compactMap { element in + collapsedTabCandidateNode( + element: element, + containerSnapshot: containerSnapshot, + containerFrame: containerFrame + ) + } + .sorted { left, right in + if left.rect.x != right.rect.x { + return left.rect.x < right.rect.x + } + return left.rect.y < right.rect.y + } + + if candidates.count < 2 { return [] } + let rowMidpoints = candidates.map { $0.rect.y + ($0.rect.height / 2) } + let rowSpread = (rowMidpoints.max() ?? 0) - (rowMidpoints.min() ?? 0) + // Allow modest vertical jitter and short two-row wraps while still rejecting unrelated controls. + if rowSpread > max(24.0, Double(containerFrame.height) * 0.6) { return [] } + + var seen = Set() + let uniqueCandidates = candidates.filter { node in + let key = "\(node.type)-\(node.label ?? "")-\(node.identifier ?? "")-\(node.value ?? "")-\(node.rect.x)-\(node.rect.y)-\(node.rect.width)-\(node.rect.height)" + if seen.contains(key) { return false } + seen.insert(key) + return true + } + if uniqueCandidates.count < 2 { return [] } + + return uniqueCandidates.enumerated().map { offset, node in + RawAXNode( + index: startingIndex + offset, + type: node.type, + label: node.label, + identifier: node.identifier, + value: node.value, + rect: node.rect, + enabled: node.enabled, + focused: node.focused, + selected: node.selected, + hittable: node.hittable, + depth: depth, + parentIndex: parentIndex, + hiddenContentAbove: nil, + hiddenContentBelow: nil + ) + } + } + + private func collapsedTabCandidateNode( + element: XCUIElement, + containerSnapshot: XCUIElementSnapshot, + containerFrame: CGRect + ) -> RawAXNode? { + var node: RawAXNode? + let exceptionMessage = RunnerObjCExceptionCatcher.catchException({ + if !element.exists { return } + let elementType = element.elementType + if !Self.collapsedTabCandidateTypes.contains(elementType) { return } + let frame = element.frame + if frame.isNull || frame.isEmpty { return } + if frame.equalTo(containerFrame) { return } + let area = max(CGFloat(1), frame.width * frame.height) + let containerArea = max(CGFloat(1), containerFrame.width * containerFrame.height) + if area >= containerArea * 0.9 { return } + let center = CGPoint(x: frame.midX, y: frame.midY) + if !containerFrame.contains(center) { return } + + let label = element.label.trimmingCharacters(in: .whitespacesAndNewlines) + let identifier = element.identifier.trimmingCharacters(in: .whitespacesAndNewlines) + let valueText = snapshotValueText(element) + let hasContent = !label.isEmpty || !identifier.isEmpty || valueText != nil + if !hasContent { return } + if sameSemanticElement( + containerSnapshot: containerSnapshot, + elementType: elementType, + label: label, + identifier: identifier + ) { + return + } + + // The containment and area rules above compared reported frames with each other; the node + // joins the tree carrying the frame the platform reported, and normalization places it. + node = RawAXNode( + index: 0, + type: elementTypeName(elementType), + label: label.isEmpty ? nil : label, + identifier: identifier.isEmpty ? nil : identifier, + value: valueText, + rect: SnapshotRect(frame), + enabled: element.isEnabled, + focused: elementHasFocus(element) ? true : nil, + selected: element.isSelected ? true : nil, + hittable: false, + depth: 0, + parentIndex: nil, + hiddenContentAbove: nil, + hiddenContentBelow: nil + ) + }) + if let exceptionMessage { + NSLog( + "AGENT_DEVICE_RUNNER_SNAPSHOT_TAB_FALLBACK_IGNORED_EXCEPTION=%@", + exceptionMessage + ) + return nil + } + return node + } + + private func snapshotHasFocus(_ snapshot: XCUIElementSnapshot) -> Bool { + var focused = false + _ = RunnerObjCExceptionCatcher.catchException({ + if let value = (snapshot as! NSObject).value(forKey: "hasFocus") as? Bool { + focused = value + } + }) + return focused + } + + private func snapshotIsSelected(_ snapshot: XCUIElementSnapshot) -> Bool { + return snapshot.isSelected + } + + private func shouldExpandCollapsedTabContainer(_ snapshot: XCUIElementSnapshot) -> Bool { + let frame = snapshot.frame + if frame.isNull || frame.isEmpty { return false } + if frame.width < max(CGFloat(160), frame.height * 1.75) { return false } + switch snapshot.elementType { + case .tabBar, .segmentedControl, .slider: + return true + default: + return false + } + } + + private func snapshotValueText(_ element: XCUIElement) -> String? { + let text = String(describing: element.value ?? "") + .trimmingCharacters(in: .whitespacesAndNewlines) + return text.isEmpty ? nil : text + } + + private func sameSemanticElement( + containerSnapshot: XCUIElementSnapshot, + elementType: XCUIElement.ElementType, + label: String, + identifier: String + ) -> Bool { + if containerSnapshot.elementType != elementType { return false } + let containerLabel = containerSnapshot.label.trimmingCharacters(in: .whitespacesAndNewlines) + let containerIdentifier = containerSnapshot.identifier + .trimmingCharacters(in: .whitespacesAndNewlines) + return containerLabel == label && containerIdentifier == identifier + } + + func flatInteractiveElements( + app: XCUIApplication, + deadline: Date + ) -> (elements: [XCUIElement], outcome: SnapshotTierOutcome) { + let queries: [XCUIElementQuery] = [ + app.buttons, + app.links, + app.textFields, + app.secureTextFields, + app.searchFields, + app.textViews, + app.switches, + app.sliders, + app.segmentedControls, + app.cells, + app.collectionViews, + app.tables, + app.scrollViews, + app.pickers, + app.steppers, + app.tabBars, + app.menuItems, + app.staticTexts, + app.images + ] + + return Self.runFlatInteractiveQueries(queries, deadline: deadline) { query in + self.snapshotElementsQuery { + query.allElementsBoundByIndex + } + } + } + + /// Runs sweep queries in order until one reports AX unavailable, or until the next one could not + /// finish before `deadline` (`querySweepCanStartQuery`). A sweep stopped by the deadline reports + /// `.deadlineExhausted` rather than a truncation flag: what it collected is a partial tree, and + /// only the caller that owns the tier decides whether that counts as an answer (#2781). + static func runFlatInteractiveQueries( + _ queries: [Query], + deadline: Date, + now: () -> Date = { Date() }, + run: (Query) -> (elements: [Element], axUnavailable: Bool) + ) -> (elements: [Element], outcome: SnapshotTierOutcome) { + var elements: [Element] = [] + for query in queries { + if !querySweepCanStartQuery(deadline: deadline, now: now()) { + NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_FLAT_FALLBACK_DEADLINE") + return (elements, .deadlineExhausted) + } + let result = run(query) + elements.append(contentsOf: result.elements) + if result.axUnavailable { + break + } + } + return (elements, .completed) + } + + func snapshotElementsQuery( + _ fetch: () -> [XCUIElement] + ) -> (elements: [XCUIElement], axUnavailable: Bool) { + let (elements, exceptionMessage) = catchingObjCException(fallback: [], fetch) + guard let exceptionMessage else { + return (elements, false) + } + NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_QUERY_IGNORED_EXCEPTION=%@", exceptionMessage) + if Self.isAxIllegalArgument(exceptionMessage) { + invalidateCachedTarget(reason: "ax_snapshot_query_unavailable") + return ([], true) + } + return ([], false) + } + + func flatSnapshotNode( + element: XCUIElement, + index: Int, + parentIndex: Int? + ) -> RawAXNode? { + var node: RawAXNode? + let exceptionMessage = RunnerObjCExceptionCatcher.catchException({ + if !element.exists { return } + // Declared residue: a flat element query has no hierarchy for geometryless semantics to + // attach to, so frameless elements are dropped at acquisition rather than presented. + let frame = element.frame + if frame.isNull || frame.isEmpty { return } + let label = element.label.trimmingCharacters(in: .whitespacesAndNewlines) + let identifier = element.identifier.trimmingCharacters(in: .whitespacesAndNewlines) + let valueText = snapshotValueText(element) + let elementType = element.elementType + let enabled = element.isEnabled + + node = RawAXNode( + index: index, + type: elementTypeName(elementType), + label: label.isEmpty ? nil : label, + identifier: identifier.isEmpty ? nil : identifier, + value: valueText, + rect: SnapshotRect(frame), + enabled: enabled, + focused: elementHasFocus(element) ? true : nil, + selected: element.isSelected ? true : nil, + hittable: false, + depth: 1, + parentIndex: parentIndex, + hiddenContentAbove: nil, + hiddenContentBelow: nil + ) + }) + if let exceptionMessage { + NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_FLAT_IGNORED_EXCEPTION=%@", exceptionMessage) + return nil + } + return node + } + +} diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotBackendCapabilities.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotBackendCapabilities.swift index e4c43b54bd..4a5b478639 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotBackendCapabilities.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotBackendCapabilities.swift @@ -86,81 +86,3 @@ enum SnapshotBackendKind: String, CaseIterable { } } } - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS -private struct SnapshotBackendParityFixture: Decodable { - struct Availability: Decodable { - let simulator: Bool - let physicalDevice: Bool - } - - struct Backend: Decodable { - let name: String - let forceable: Bool - let supportsRawProjection: Bool - let regularDepth: String - let hittable: String - let availability: Availability - } - - let backends: [Backend] -} - -extension RunnerTests { - private func loadSnapshotBackendParityFixture() throws -> SnapshotBackendParityFixture { - let fixtureURL = URL(fileURLWithPath: #filePath) - .deletingLastPathComponent() // AgentDeviceRunnerUITests - .deletingLastPathComponent() // AgentDeviceRunner - .deletingLastPathComponent() // runner - .deletingLastPathComponent() // apple - .deletingLastPathComponent() // repo root - .appendingPathComponent("contracts") - .appendingPathComponent("fixtures") - .appendingPathComponent("ios-snapshot-backends.json") - return try JSONDecoder().decode( - SnapshotBackendParityFixture.self, - from: Data(contentsOf: fixtureURL) - ) - } - - /// The JSON table is the cross-runtime declaration used by the TypeScript capability registry - /// and this runner. A backend case, forceability branch, projection/depth claim, or availability - /// change that is not classified in both implementations fails before an iOS smoke can drift. - func testSnapshotBackendDeclarationsMatchCapabilityFixture() throws { - let fixture = try loadSnapshotBackendParityFixture() - XCTAssertEqual( - fixture.backends.map(\.name), - SnapshotBackendKind.allCases.map(\.rawValue) - ) - - for expected in fixture.backends { - guard let backend = SnapshotBackendKind(rawValue: expected.name) else { - XCTFail("fixture contains an unknown snapshot backend: \(expected.name)") - continue - } - XCTAssertEqual(backend.isForceable, expected.forceable, expected.name) - XCTAssertEqual(backend.supportsRawProjection, expected.supportsRawProjection, expected.name) - XCTAssertEqual( - backend.regularDepthCapability.rawValue, - expected.regularDepth, - "regular depth capability: \(expected.name)" - ) - XCTAssertEqual( - backend.hittableSemantics, - expected.hittable, - "hittable semantics: \(expected.name)" - ) - XCTAssertEqual( - backend.isAvailable(on: .simulator), - expected.availability.simulator, - "simulator availability: \(expected.name)" - ) - XCTAssertEqual( - backend.isAvailable(on: .physicalDevice), - expected.availability.physicalDevice, - "physical-device availability: \(expected.name)" - ) - } - } -} -#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift index 7c6bc29443..770dde5524 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift @@ -8,11 +8,21 @@ import AgentDeviceSnapshotPresentation // stamp the outcome with a structured quality verdict so the daemon renders state instead of // re-deriving it from node shapes. Recovery ordering is data here, never a per-call-site branch. +/// The closed set of verdict states the host accepts. The wire strings are the shared table at +/// `contracts/fixtures/ios-snapshot-quality-states.json`, which `allCases` is pinned to; `reasonCode` +/// stays open because an unknown one costs only its wording, never the verdict. +enum SnapshotQualityState: String, Codable, CaseIterable { + /// First backend produced a usable tree. + case healthy + /// A later backend did. + case recovered + /// No backend produced a usable tree; the best attempt is returned as-is. + case sparse +} + /// Structured quality verdict shipped with every iOS snapshot payload. struct SnapshotQuality: Codable { - /// healthy: first backend produced a usable tree. recovered: a later backend did. - /// sparse: no backend produced a usable tree; the best attempt is returned as-is. - let state: String + let state: SnapshotQualityState /// Backend that produced the returned payload: tree | queries | private-ax. let backend: String /// Why recovery ran (first failure), why the payload is degraded, or why an internal backend @@ -37,6 +47,16 @@ enum SnapshotXCTestChannelPlanState: Equatable { case boundedXCTestProbe } +/// How one tier's bounded work ended. `deadlineExhausted` is a tier timeout: the tier stopped +/// starting work it could not finish inside its own slice, so what it returns is a partial result +/// it never completed collecting. The plan keeps that result only as the fallback and lets the next +/// backend answer; a node count cannot tell the two apart, because a short sweep still collects +/// more than the sparse threshold (#2781). +enum SnapshotTierOutcome: Equatable { + case completed + case deadlineExhausted +} + struct EffectiveSnapshotCapturePlan { let plan: [SnapshotBackendKind] let xCTestChannelState: SnapshotXCTestChannelPlanState @@ -142,12 +162,6 @@ extension RunnerTests { return penalized == bundleId } - func consumeSnapshotXCTestPenaltyWarmupExemption() -> Bool { - let pending = snapshotXCTestPenaltyWarmupExemptionPending - snapshotXCTestPenaltyWarmupExemptionPending = false - return pending - } - /// The pre-seeded first-failure a penalized plan stamps into its verdict. The deferred case /// uses the dedicated 'deferred' code: the breaker pre-selected the backend, nothing new /// degraded on this capture, and the daemon keys warning suppression and the settle budget @@ -260,7 +274,7 @@ extension RunnerTests { func runSnapshotCapturePlan( _ plan: [SnapshotBackendKind], - app: XCUIApplication, + target: SnapshotCaptureTarget, options: PresentationOptions, terminal: SnapshotCaptureTerminalPolicy, deadline: Date? = nil @@ -270,7 +284,7 @@ extension RunnerTests { var axFailure: SnapshotCaptureFailure? // A caller may share the pre-plan system-modal probe's deadline; otherwise own the full budget (#1244). let deadline = deadline ?? Date().addingTimeInterval(Self.snapshotPlanBudget) - let suppressXCTestPenalty = consumeSnapshotXCTestPenaltyWarmupExemption() + let suppressXCTestPenalty = snapshotXCTestPenaltyWarmupExemption.consume() // Reorder is iOS-only because hostile screens can make XCTest tree/query work grind while // the app remains visually responsive. Simulators can avoid that channel through private AX; @@ -281,7 +295,7 @@ extension RunnerTests { var xCTestChannelPenalized = false var xCTestChannelPenalizedByBreaker = false #if os(iOS) - xCTestChannelPenalizedByBreaker = isSnapshotXCTestChannelPenalized(bundleId: currentBundleId) + xCTestChannelPenalizedByBreaker = isSnapshotXCTestChannelPenalized(bundleId: target.bundleId) xCTestChannelPenalized = Self.snapshotXCTestChannelTreatedAsPenalized( penalized: xCTestChannelPenalizedByBreaker, preferredBackend: options.preferredBackend @@ -305,9 +319,9 @@ extension RunnerTests { case .normal: break case .deferredToIndependentBackend: - NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_XCTEST_CHANNEL_DEFERRED bundle=%@", currentBundleId ?? "") + NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_XCTEST_CHANNEL_DEFERRED bundle=%@", target.bundleId ?? "") case .boundedXCTestProbe: - NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_XCTEST_CHANNEL_PROBE_BOUNDED bundle=%@", currentBundleId ?? "") + NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_XCTEST_CHANNEL_PROBE_BOUNDED bundle=%@", target.bundleId ?? "") } for kind in effectivePlan { @@ -332,7 +346,7 @@ extension RunnerTests { } let attempt = try captureWithBackend( kind, - app: app, + target: target, options: options, deadline: deadline, treeCaptureSliceBudgetOverride: effective.treeCaptureSliceBudgetOverride @@ -340,6 +354,7 @@ extension RunnerTests { recordXCTestSnapshotBackendAttemptIfNeeded( kind, attempt: attempt, + bundleId: target.bundleId, penaltySuppressed: suppressXCTestPenalty ) if case let .failed(failure, phase: _) = attempt.outcome { @@ -356,8 +371,19 @@ extension RunnerTests { } guard case let .captured(capture) = attempt.outcome else { continue } - if let sparseReason = Self.sparsePayloadReason(capture.qualityPayload ?? capture.payload) { - if firstFailure == nil { firstFailure = sparseReason } + if attempt.tierOutcome == .deadlineExhausted { + NSLog( + "AGENT_DEVICE_RUNNER_SNAPSHOT_TIER_DEADLINE_EXHAUSTED backend=%@ nodes=%d", + kind.rawValue, + Self.payloadNodeCount(capture.payload) + ) + } + if let rejection = Self.snapshotTierRejectionReason( + outcome: attempt.tierOutcome, + kind: kind, + payload: capture.qualityPayload ?? capture.payload + ) { + if firstFailure == nil { firstFailure = rejection } if Self.payloadNodeCount(capture.payload) > Self.payloadNodeCount(best?.capture.payload) { best = (kind, capture) } @@ -375,7 +401,7 @@ extension RunnerTests { return stampedSnapshotPayload( capture, backend: kind, - state: recovered ? "recovered" : "healthy", + state: recovered ? .recovered : .healthy, reason: recovered || firstFailure?.code == "requested-backend" ? firstFailure : nil ) } @@ -400,11 +426,11 @@ extension RunnerTests { } let fallbackPayload = - best.map { stampedSnapshotPayload($0.capture, backend: $0.kind, state: "sparse", reason: firstFailure) } + best.map { stampedSnapshotPayload($0.capture, backend: $0.kind, state: .sparse, reason: firstFailure) } ?? stampedSnapshotPayload( SnapshotBackendCapture(payload: sparseTruncatedSnapshotPayload(), effectiveDepth: nil), backend: effectivePlan.last ?? plan.last ?? .recursiveTree, - state: "sparse", + state: .sparse, reason: firstFailure ) return fallbackPayload @@ -412,19 +438,21 @@ extension RunnerTests { private func captureWithBackend( _ kind: SnapshotBackendKind, - app: XCUIApplication, + target: SnapshotCaptureTarget, options: PresentationOptions, deadline: Date, treeCaptureSliceBudgetOverride: TimeInterval? ) throws -> SnapshotBackendAttempt { + let app = target.app let hint = SnapshotPresentation.captureHint(for: options) var timer = SnapshotPhaseTimer() let acquisition: SnapshotAcquisition? + let tierOutcome: SnapshotTierOutcome // The band is read inside the tree tier's own bounded work, so it has to be lifted out of the // acquisition phase and carried to the stamping step, where the payload is assembled (#2660). var keyboardBand: RunnerKeyboardBandFact? do { - acquisition = try timer.measure(.acquisition) { + let measured = try timer.measure(.acquisition) { () -> (SnapshotAcquisition?, SnapshotTierOutcome) in switch kind { case .recursiveTree: guard @@ -435,10 +463,10 @@ extension RunnerTests { treeCaptureSliceBudgetOverride: treeCaptureSliceBudgetOverride ) else { - return nil + return (nil, .completed) } keyboardBand = context.keyboardBand - return try self.runMainThreadWork( + let tree = try self.runMainThreadWork( "tree_processing", timeout: min(self.treeCaptureSliceBudget, max(0.5, deadline.timeIntervalSinceNow)), timeoutError: self.snapshotMainThreadTimeoutError("processing tree snapshot") @@ -447,26 +475,34 @@ extension RunnerTests { ? try self.rawTreeSnapshotAcquisition(context: context, hint: hint) : try self.recursiveTreeSnapshotAcquisition(context: context, hint: hint) } + return (tree, .completed) case .querySweep: - return try self.runMainThreadWork( + let sliceDeadline = Self.querySweepSliceDeadline(startedAt: Date(), planDeadline: deadline) + let sweep = try self.runMainThreadWork( "query_sweep", - timeout: min(Self.flatInteractiveFallbackBudget, max(0.1, deadline.timeIntervalSinceNow)), + timeout: max(0.1, sliceDeadline.timeIntervalSinceNow), timeoutError: self.snapshotMainThreadTimeoutError("running query-sweep snapshot") ) { self.querySweepSnapshotAcquisition( app: app, hint: hint, - planDeadline: deadline + sliceDeadline: sliceDeadline ) } + return (sweep.acquisition, sweep.outcome) case .privateAX: - return self.privateAXSnapshotAcquisition( - app: app, - hint: hint, - deadline: deadline + return ( + self.privateAXSnapshotAcquisition( + target: target, + hint: hint, + deadline: deadline + ), + .completed ) } } + acquisition = measured.0 + tierOutcome = measured.1 } catch let failure as SnapshotCaptureFailure { return SnapshotBackendAttempt( outcome: .failed(failure, phase: .acquisition), @@ -476,7 +512,8 @@ extension RunnerTests { guard let acquisition else { return SnapshotBackendAttempt( outcome: .noCapture, - timing: timer.timing + timing: timer.timing, + tierOutcome: tierOutcome ) } @@ -486,8 +523,7 @@ extension RunnerTests { let normalizedAcquisition = acquisition.replacingNodes( SnapshotGeometrySpace.normalized( nodes: acquisition.nodes, - viewport: acquisition.viewport, - interfaceOrientation: acquisition.interfaceOrientation + viewport: acquisition.viewport ) ) @@ -511,12 +547,14 @@ extension RunnerTests { } catch let failure as SnapshotPresentationFailure { return SnapshotBackendAttempt( outcome: .failed(Self.snapshotCaptureFailure(for: failure), phase: .presentation), - timing: timer.timing + timing: timer.timing, + tierOutcome: tierOutcome ) } catch let failure as SnapshotCaptureFailure { return SnapshotBackendAttempt( outcome: .failed(failure, phase: .presentation), - timing: timer.timing + timing: timer.timing, + tierOutcome: tierOutcome ) } @@ -525,7 +563,8 @@ extension RunnerTests { capture.keyboardBand = keyboardBand?.payload return SnapshotBackendAttempt( outcome: .captured(capture), - timing: timer.timing + timing: timer.timing, + tierOutcome: tierOutcome ) } @@ -546,6 +585,24 @@ extension RunnerTests { // MARK: Quality classifier (the single source of "is this snapshot degraded") + /// Why a captured tier may not end the plan, or nil when it may. A tier that stopped starting work + /// at its own deadline is rejected as a timeout whatever it collected: keeping its payload as the + /// fallback is right, accepting it is not, and a node count cannot tell a finished capture from a + /// sweep the slice cut short (#2781). + static func snapshotTierRejectionReason( + outcome: SnapshotTierOutcome, + kind: SnapshotBackendKind, + payload: DataPayload + ) -> (reason: String, code: String)? { + if outcome == .deadlineExhausted { + return ( + "the \(kind.rawValue) backend spent its capture slice with the collection unfinished", + "budget" + ) + } + return sparsePayloadReason(payload) + } + /// Returns a degradation reason + machine code when the payload is too degraded to accept. static func sparsePayloadReason(_ payload: DataPayload) -> (reason: String, code: String)? { guard let nodes = payload.nodes, !nodes.isEmpty else { @@ -593,7 +650,7 @@ extension RunnerTests { guard Self.structuralOnlyNodeTypes.contains(node.type) else { return false } guard !isRootContainer else { return true } - let isFullScreenContainer = !node.hittable && rootRects.contains { rootRect in + let isFullScreenContainer = node.hittable != true && rootRects.contains { rootRect in rootRect.x == node.rect.x && rootRect.y == node.rect.y && rootRect.width == node.rect.width && rootRect.height == node.rect.height } @@ -629,13 +686,16 @@ extension RunnerTests { // MARK: Outcome stamping - private func stampedSnapshotPayload( + func stampedSnapshotPayload( _ capture: SnapshotBackendCapture, backend: SnapshotBackendKind, - state: String, + state: SnapshotQualityState, reason: (reason: String, code: String)? ) -> DataPayload { - runnerAccessibilityHealth = reason?.code == "ax-rejected" ? .unavailable : .healthy + let health: RunnerAccessibilityHealth = reason?.code == "ax-rejected" ? .unavailable : .healthy + applyMainOwnedSnapshotState("accessibility_health") { + self.runnerAccessibilityHealth = health + } let payload = capture.payload let quality = SnapshotQuality( state: state, @@ -648,14 +708,13 @@ extension RunnerTests { timing: capture.timing ) return DataPayload( - // Legacy human text for older daemons that read message instead of snapshotQuality. - message: Self.legacyQualityMessage(quality) ?? payload.message, + message: payload.message, nodes: payload.nodes, // Completeness, never provenance: a whole tree that a later backend produced (state // "recovered") stays untruncated, so strict absence reads can trust it. Only a real cap // (payload truncation, a depth-limited private AX capture) or a sparse terminal payload // is truncated. - truncated: payload.truncated == true || state == "sparse" || capture.effectiveDepth != nil, + truncated: payload.truncated == true || state == .sparse || capture.effectiveDepth != nil, qualityPayload: capture.qualityPayload.flatMap { quality in guard let nodes = quality.nodes else { return nil } return SnapshotQualityPayload(nodes: nodes, truncated: quality.truncated == true) @@ -666,549 +725,4 @@ extension RunnerTests { runnerFatalReason: payload.runnerFatalReason ) } - - /// Response level, one line per incompleteness. An unread merged element is - /// byte-identical to one with no actions, and a clipped list looks complete, - /// so both have to name themselves. - static func customActionCoverageWarnings(_ coverage: SnapshotCustomActionCoverage) -> [String] { - var lines: [String] = [] - if coverage.blocked { - // Scrolling is the remedy for a budget stop, not for this one — saying it - // here would send the reader off doing something that cannot help. - lines.append( - "Custom actions were not read: an earlier accessibility read is still hung, so this " - + "capture skipped the read pass instead of queueing behind it. No element's actions " - + "list is authoritative here. Reads resume once that call returns.") - } else if coverage.read < coverage.candidates { - lines.append( - "Custom actions were read for \(coverage.read) of \(coverage.candidates) merged elements, " - + "on-screen ones first; the remaining \(coverage.candidates - coverage.read) were not read, " - + "so an absent actions list on those is not evidence that they have none. " - + "Scroll them into view and re-run to read them.") - } - if coverage.truncated > 0 { - lines.append( - "\(coverage.truncated) element(s) published more custom actions than are shown; those " - + "lists are clipped to the first 8 names, and long names are shortened.") - } - return lines - } - - static func legacyQualityMessage(_ quality: SnapshotQuality) -> String? { - let customActionWarnings = - quality.customActions.map { Self.customActionCoverageWarnings($0) } ?? [] - guard quality.state != "healthy" || quality.collapsedLeafIndexes != nil - || !customActionWarnings.isEmpty - else { return nil } - var parts: [String] = [] - if quality.state == "recovered" { - let meaning: String - switch quality.reasonCode { - case "budget", "deferred": - meaning = " The primary capture ran out of its time budget (busy app or simulator); the recovered tree is authoritative for this screen." - case "presentation-failed": - meaning = " The runner rejected a regular presentation because its cumulative clip invariant failed; report this as a runner bug and treat screenshot as visual truth." - default: - meaning = " This usually means the app publishes an unhealthy accessibility tree — fixing the app's accessibility is the real cure. Treat screenshot as visual truth when this warning appears." - } - parts.append( - "Detected an overly complex or slow accessibility tree. Fell back to the \(quality.backend) snapshot backend" - + (quality.reason.map { " after: \($0)." } ?? ".") - + meaning - ) - } - if quality.state == "sparse" { - parts.append( - "No snapshot backend could read this screen" - + (quality.reason.map { " (\($0))" } ?? "") - + ". Use screenshot as visual truth and coordinate taps." - ) - } - parts.append(contentsOf: customActionWarnings) - if let depth = quality.effectiveDepth { - // No --depth remedy here: an explicit --depth capture disables the - // frontier extension, so following it would return strictly less than - // this capture did. A plain re-run retries with a fresh extension budget. - parts.append( - "The accessibility server rejected deeper requests; content below depth \(depth) may be missing — re-run snapshot to retry deeper content." - ) - } - return parts.isEmpty ? nil : parts.joined(separator: " ") - } -} - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS -// MARK: - In-bundle unit tests - -extension RunnerTests { - private func planTestNode( - index: Int, - type: String, - label: String? = nil, - identifier: String? = nil, - hittable: Bool = false, - parentIndex: Int? = nil - ) -> PresentedNode { - SnapshotPresentation.singleElementRead( - RawAXNode( - index: index, - type: type, - label: label, - identifier: identifier, - value: nil, - rect: SnapshotRect(.zero), - enabled: true, - focused: nil, - selected: nil, - hittable: hittable, - depth: parentIndex == nil ? 0 : 1, - parentIndex: parentIndex, - hiddenContentAbove: nil, - hiddenContentBelow: nil - ) - ) - } - - func testSparsePayloadReasonMatrix() { - let root = planTestNode(index: 0, type: "Application", label: "Example App", hittable: true) - let window = planTestNode(index: 1, type: "Window", parentIndex: 0) - let button = planTestNode(index: 1, type: "Button", label: "Ok", hittable: true, parentIndex: 0) - let shell = planTestNode( - index: 1, - type: "Other", - identifier: "appShell", - parentIndex: 0 - ) - let serializationPlaceholder = planTestNode( - index: 2, - type: "Other", - label: "[object Object]", - parentIndex: 1 - ) - - // Labeled, hittable root over a bare window is still sparse. - XCTAssertNotNil(Self.sparsePayloadReason(DataPayload(nodes: [root, window], truncated: false))) - // Deadline-truncated near-empty sweep needs recovery even with one real control. - XCTAssertNotNil(Self.sparsePayloadReason(DataPayload(nodes: [root, button], truncated: true))) - // The same tiny tree from a completed sweep is a legitimately minimal screen. - XCTAssertNil(Self.sparsePayloadReason(DataPayload(nodes: [root, button], truncated: false))) - // Container metadata plus a stringified serialization placeholder is not readable UI. - XCTAssertNotNil( - Self.sparsePayloadReason( - DataPayload(nodes: [root, shell, serializationPlaceholder], truncated: false) - ) - ) - let actionableShell = planTestNode( - index: 1, - type: "Other", - identifier: "checkout", - hittable: true, - parentIndex: 0 - ) - XCTAssertNil( - Self.sparsePayloadReason(DataPayload(nodes: [root, actionableShell], truncated: false)) - ) - // Empty payloads are degraded. - XCTAssertNotNil(Self.sparsePayloadReason(DataPayload(nodes: [], truncated: false))) - } - - func testCollapsedLeafIndexesFlagsMergedContainersOnly() { - let root = planTestNode(index: 0, type: "Application", label: "App") - let merged = planTestNode( - index: 1, - type: "Other", - label: (0...30).map { "Row \($0), Tap" }.joined(separator: ", "), - parentIndex: 0 - ) - let prose = planTestNode( - index: 2, - type: "StaticText", - label: (0...30).map { "clause \($0)" }.joined(separator: ", "), - parentIndex: 0 - ) - XCTAssertEqual(Self.collapsedLeafIndexes([root, merged, prose]), [1]) - XCTAssertNil(Self.collapsedLeafIndexes([root, prose])) - } - - func testLegacyQualityMessageStatesFallbackMeaning() { - let recovered = SnapshotQuality( - state: "recovered", - backend: "queries", - reason: "snapshot returned only structural application/window nodes", - reasonCode: "sparse-tree", - effectiveDepth: nil, - collapsedLeafIndexes: nil, - customActions: nil - ) - let message = Self.legacyQualityMessage(recovered) - XCTAssertTrue(message?.contains("queries snapshot backend") == true) - XCTAssertTrue(message?.contains("fixing the app's accessibility") == true) - XCTAssertTrue(message?.contains("screenshot as visual truth") == true) - XCTAssertNil( - Self.legacyQualityMessage( - SnapshotQuality( - state: "healthy", backend: "tree", reason: nil, reasonCode: nil, effectiveDepth: nil, - collapsedLeafIndexes: nil, customActions: nil) - ) - ) - } - func testTerminalFailsClosedOnInteractiveAxFailureRegardlessOfSparseBest() { - // Interactive AX failure must invalidate + fail closed; a later tier's sparse synthetic-root - // "best" must never downgrade this to a returned-sparse payload (regression: best == nil guard). - XCTAssertEqual( - Self.resolveSnapshotPlanTerminal(terminal: .sparseWithFatalOnAXFailure, interactiveOnly: true), - .failClosed - ) - XCTAssertEqual( - Self.resolveSnapshotPlanTerminal(terminal: .sparseWithFatalOnAXFailure, interactiveOnly: false), - .sparseBest - ) - XCTAssertEqual( - Self.resolveSnapshotPlanTerminal(terminal: .throwOnAXFailure, interactiveOnly: true), - .throwAxFailure - ) - } - - func testXCTestChannelStateFirstFailureStampsDeferredCodeOnlyForDeferral() { - XCTAssertNil(Self.xcTestChannelStateFirstFailure(.normal)) - XCTAssertEqual(Self.xcTestChannelStateFirstFailure(.deferredToIndependentBackend)?.code, "deferred") - XCTAssertEqual(Self.xcTestChannelStateFirstFailure(.boundedXCTestProbe)?.code, "budget") - } - - func testSnapshotQualityCarriesPhaseTimingAtResponseLevel() { - let timing = SnapshotCaptureTiming(acquisitionMs: 12, presentationMs: 34) - let capture = SnapshotBackendCapture( - payload: DataPayload( - nodes: [planTestNode(index: 0, type: "Application", label: "App")], - truncated: false - ), - effectiveDepth: nil, - timing: timing - ) - - let payload = stampedSnapshotPayload( - capture, - backend: .recursiveTree, - state: "healthy", - reason: nil - ) - - XCTAssertEqual(payload.snapshotQuality?.timing, timing) - XCTAssertEqual(payload.nodes?.count, 1) - } - - func testStampedPayloadTruncationTracksCompletenessNotRecoveryProvenance() { - let complete = SnapshotBackendCapture( - payload: DataPayload( - nodes: [ - planTestNode(index: 0, type: "Application", label: "App"), - planTestNode(index: 1, type: "Button", label: "Open", parentIndex: 0), - ], - truncated: false - ), - effectiveDepth: nil - ) - let deferred: (reason: String, code: String) = ( - "XCTest-backed snapshot tiers were deferred after recent slow accessibility work", "deferred" - ) - - // The CI signature behind `is absent ... capture was truncated`: a complete private AX - // tree selected while the XCTest channel is penalized is whole, and must say so. - let recovered = stampedSnapshotPayload( - complete, backend: .privateAX, state: "recovered", reason: deferred) - XCTAssertEqual(recovered.snapshotQuality?.state, "recovered") - XCTAssertEqual(recovered.truncated, false) - - let depthLimited = stampedSnapshotPayload( - SnapshotBackendCapture(payload: complete.payload, effectiveDepth: 56), - backend: .privateAX, state: "recovered", reason: deferred) - XCTAssertEqual(depthLimited.truncated, true) - - let cappedPayload = stampedSnapshotPayload( - SnapshotBackendCapture( - payload: DataPayload(nodes: complete.payload.nodes ?? [], truncated: true), - effectiveDepth: nil), - backend: .recursiveTree, state: "healthy", reason: nil) - XCTAssertEqual(cappedPayload.truncated, true) - - let sparse = stampedSnapshotPayload( - complete, backend: .querySweep, state: "sparse", - reason: ("snapshot returned no semantic controls or content", "sparse-tree")) - XCTAssertEqual(sparse.truncated, true) - } - - func testSnapshotQualityCarriesUnscopedQualityPayload() { - let quality = DataPayload( - nodes: [planTestNode(index: 0, type: "Application", label: "App")], - truncated: false - ) - let capture = SnapshotBackendCapture( - payload: quality, - effectiveDepth: nil, - qualityPayload: quality - ) - - let payload = stampedSnapshotPayload( - capture, - backend: .recursiveTree, - state: "healthy", - reason: nil - ) - - XCTAssertEqual(payload.qualityPayload?.nodes.count, 1) - XCTAssertEqual(payload.qualityPayload?.truncated, false) - XCTAssertNil(payload.qualityPayload?.scope) - } - - func testDirectPresentationDoesNotClaimPlanTiming() { - let options = PresentationOptions( - interactiveOnly: false, - depth: nil, - scope: nil, - raw: true - ) - let result = SnapshotPresentation.presentRaw( - SnapshotAcquisition( - hint: SnapshotPresentation.captureHint(for: options), - nodes: [], - truncated: false, - effectiveDepth: nil, - viewport: .infinite - ), - options: options - ) - let capture = Self.makeSnapshotBackendCapture(from: result) - - let payload = stampedSnapshotPayload( - capture, - backend: .recursiveTree, - state: "healthy", - reason: nil - ) - - XCTAssertNil(payload.snapshotQuality?.timing) - } - - /// The raw plan is derived from what each backend can actually serve, not from a second - /// hand-maintained list. Non-vacuity: flipping `querySweep.supportsRawProjection` to true adds it - /// to the plan and fails the first two assertions — which is exactly the shape of #1797 D4, a - /// `--raw` request answered by a backend that has no hierarchy to return. - func testRawDiagnosticPlanCarriesOnlyBackendsThatCanServeRaw() { - XCTAssertEqual(Self.rawDiagnosticPlan, [.recursiveTree, .privateAX]) - XCTAssertEqual( - SnapshotBackendKind.allCases.filter { !$0.supportsRawProjection }, [.querySweep]) - XCTAssertTrue(Self.rawDiagnosticPlan.allSatisfy(\.supportsRawProjection)) - // Tree-first error propagation is the raw plan's other contract (ADR 0004). - XCTAssertEqual(Self.rawDiagnosticPlan.first, .recursiveTree) - } - - /// A projection mismatch is a runner bug, not an accessibility failure: it must not take the - /// AX-failure terminal route (rethrow / fail-closed), just drop its tier with a named reason. - func testProjectionMismatchFailureIsStructuredAndNotAnAxFailure() { - let failure = Self.snapshotProjectionMismatchFailure( - .querySweep, requested: .raw, acquired: .regular) - XCTAssertEqual(failure.code, "IOS_SNAPSHOT_PROJECTION_MISMATCH") - XCTAssertTrue(failure.message.contains("queries")) - XCTAssertTrue(failure.message.contains("raw")) - XCTAssertFalse(Self.isAxSnapshotFailure(failure)) - } - - /// #1634 P2: the decoded wire field must reach capture options and its - /// applicable plan. A pinned REGULAR capture defers to privateAX-first; the - /// RAW diagnostic plan is never rerouted by the pin — raw keeps tree-first - /// error propagation, which is exactly why raw baselines are excluded from - /// corroboration daemon-side. - func testDecodedPreferredBackendReachesOptionsAndApplicablePlan() throws { - let json = #"{"command":"snapshot","preferredBackend":"private-ax"}"# - let command = try JSONDecoder().decode(Command.self, from: Data(json.utf8)) - let options = Self.presentationOptions(from: command) - XCTAssertEqual(options.preferredBackend, "private-ax") - XCTAssertFalse(options.raw) - - let treated = Self.snapshotXCTestChannelTreatedAsPenalized( - penalized: false, preferredBackend: options.preferredBackend) - let pinned = Self.effectiveSnapshotCapturePlan( - Self.regularVisiblePlan, - xCTestChannelPenalized: treated, - preferredBackend: options.preferredBackend - ) - XCTAssertEqual(pinned.plan, [.privateAX]) - XCTAssertEqual(pinned.xCTestChannelState, .deferredToIndependentBackend) - - let raw = Self.effectiveSnapshotCapturePlan( - Self.rawDiagnosticPlan, - xCTestChannelPenalized: treated, - preferredBackend: options.preferredBackend - ) - XCTAssertEqual(raw.plan, Self.rawDiagnosticPlan) - - // A command without the field decodes to no pin and a normal plan. - let bare = try JSONDecoder().decode( - Command.self, from: Data(#"{"command":"snapshot"}"#.utf8)) - XCTAssertNil(Self.presentationOptions(from: bare).preferredBackend) - } - - /// #1635: the force seam must select the recursive tree even when the XCTest - /// channel is currently penalized. Without the preferred-backend argument, - /// this call returns the independent private-AX recovery plan instead. - func testPreferredTreeBackendPinsRegularPlanAndLeavesStructuredEvidence() { - let forced = Self.effectiveSnapshotCapturePlan( - Self.regularVisiblePlan, - xCTestChannelPenalized: true, - preferredBackend: SnapshotBackendKind.recursiveTree.rawValue - ) - XCTAssertEqual(forced.plan, [.recursiveTree]) - XCTAssertEqual(forced.xCTestChannelState, .normal) - XCTAssertEqual( - Self.xcTestChannelStateFirstFailure( - forced.xCTestChannelState, - preferredBackend: forced.preferredBackend?.rawValue - )?.code, - "requested-backend" - ) - } - - /// Same-backend evidence probes: a daemon-pinned private-AX capture takes the - /// penalized route even with a healthy channel, so tap-outcome corroboration - /// baselines and probes are always captured by the same backend (backends are - /// never comparable views of a screen). Composed with the plan rule, the pin - /// yields the privateAX-first deferred plan. - func testPreferredPrivateAXBackendPlansAsPenalized() { - XCTAssertTrue( - Self.snapshotXCTestChannelTreatedAsPenalized(penalized: false, preferredBackend: "private-ax")) - XCTAssertTrue( - Self.snapshotXCTestChannelTreatedAsPenalized(penalized: true, preferredBackend: nil)) - XCTAssertFalse( - Self.snapshotXCTestChannelTreatedAsPenalized(penalized: false, preferredBackend: nil)) - XCTAssertFalse( - Self.snapshotXCTestChannelTreatedAsPenalized(penalized: false, preferredBackend: "tree")) - - let pinned = Self.effectiveSnapshotCapturePlan( - Self.regularVisiblePlan, - xCTestChannelPenalized: Self.snapshotXCTestChannelTreatedAsPenalized( - penalized: false, preferredBackend: "private-ax" - ), - preferredBackend: "private-ax" - ) - XCTAssertEqual(pinned.plan, [.privateAX]) - XCTAssertEqual(pinned.xCTestChannelState, .deferredToIndependentBackend) - } - - func testEffectiveSnapshotCapturePlanDefersXCTestBackedTiersOnlyWhenPenalizedRegularPlan() { - let regular = Self.effectiveSnapshotCapturePlan( - Self.regularVisiblePlan, - xCTestChannelPenalized: true - ) - XCTAssertEqual(regular.plan, [.privateAX]) - XCTAssertEqual(regular.xCTestChannelState, .deferredToIndependentBackend) - XCTAssertNil(regular.treeCaptureSliceBudgetOverride) - - let unpenalized = Self.effectiveSnapshotCapturePlan( - Self.regularVisiblePlan, - xCTestChannelPenalized: false - ) - XCTAssertEqual(unpenalized.plan, Self.regularVisiblePlan) - XCTAssertEqual(unpenalized.xCTestChannelState, .normal) - XCTAssertNil(unpenalized.treeCaptureSliceBudgetOverride) - - // The raw diagnostic plan preserves tree-first error propagation even under penalty. - let raw = Self.effectiveSnapshotCapturePlan( - Self.rawDiagnosticPlan, - xCTestChannelPenalized: true - ) - XCTAssertEqual(raw.plan, Self.rawDiagnosticPlan) - XCTAssertEqual(raw.xCTestChannelState, .normal) - XCTAssertNil(raw.treeCaptureSliceBudgetOverride) - } - - func testEffectiveSnapshotCapturePlanUsesBoundedXCTestProbeWhenNoIndependentBackendRuns() { - let physicalDevicePlan = Self.effectiveSnapshotCapturePlan( - Self.regularVisiblePlan, - xCTestChannelPenalized: true, - availableBackends: [.recursiveTree, .querySweep] - ) - - XCTAssertEqual(physicalDevicePlan.plan, [.recursiveTree, .querySweep]) - XCTAssertEqual(physicalDevicePlan.xCTestChannelState, .boundedXCTestProbe) - XCTAssertEqual( - physicalDevicePlan.treeCaptureSliceBudgetOverride, - Self.penalizedXCTestProbeTreeSliceBudget - ) - } - - func testSnapshotXCTestChannelPenaltyMatchesBundleAndExpires() { - defer { - snapshotXCTestChannelPenaltyBundleId = nil - snapshotXCTestChannelPenaltyUntil = .distantPast - } - - penalizeSnapshotXCTestChannel(bundleId: "xyz.blueskyweb.app", reason: "test") - XCTAssertTrue(isSnapshotXCTestChannelPenalized(bundleId: "xyz.blueskyweb.app")) - XCTAssertFalse(isSnapshotXCTestChannelPenalized(bundleId: "com.other.app")) - - // A penalty recorded without a bundle applies to any current target. - penalizeSnapshotXCTestChannel(bundleId: nil, reason: "test") - XCTAssertTrue(isSnapshotXCTestChannelPenalized(bundleId: "com.other.app")) - - // Expired penalties stop applying. - snapshotXCTestChannelPenaltyUntil = Date(timeIntervalSinceNow: -1) - XCTAssertFalse(isSnapshotXCTestChannelPenalized(bundleId: "com.other.app")) - } - - func testAbandonedMainThreadWorkSkipsOnlyXCTestBackedSnapshotTiers() { - abandonedMainThreadWorkCount = 1 - defer { abandonedMainThreadWorkCount = 0 } - - XCTAssertTrue(shouldSkipSnapshotBackendForAbandonedMainThreadWork(.recursiveTree)) - XCTAssertTrue(shouldSkipSnapshotBackendForAbandonedMainThreadWork(.querySweep)) - XCTAssertFalse(shouldSkipSnapshotBackendForAbandonedMainThreadWork(.privateAX)) - } - -#if os(iOS) - /// #2403: a plan pinned to private AX serves a regular `--depth` request through acquisition - /// and presentation. With a backend depth gate in `captureWithBackend`, private AX returns no - /// capture, the plan falls through to the synthetic sparse root, and the daemon rejects that - /// zero-rect root as a missing viewport. - func testPrivateAXPinnedRegularDepthReachesAcquisitionAndPresentation() throws { - app.launchArguments = ["--agent-device-selector-read-regression"] - app.launch() - currentApp = app - currentBundleId = nil - defer { - currentApp = nil - clearPrivateAXAcceptedDepth(reason: "test-cleanup") - app.terminate() - } - func capture(depth: Int?) throws -> DataPayload { - try runSnapshotCapturePlan( - Self.regularVisiblePlan, - app: app, - options: PresentationOptions( - interactiveOnly: false, - depth: depth, - scope: nil, - raw: false, - preferredBackend: SnapshotBackendKind.privateAX.rawValue - ), - terminal: .sparseWithFatalOnAXFailure - ) - } - - let capped = try capture(depth: 1) - - let quality = try XCTUnwrap(capped.snapshotQuality) - XCTAssertEqual(quality.backend, SnapshotBackendKind.privateAX.rawValue) - XCTAssertNotEqual(quality.state, "sparse") - let nodes = try XCTUnwrap(capped.nodes) - XCTAssertGreaterThan(nodes.count, 1) - XCTAssertEqual(nodes.map(\.depth).max(), 1) - XCTAssertNotEqual(nodes[0].rect, SnapshotRect(x: 0, y: 0, width: 0, height: 0)) - XCTAssertTrue(nodes.contains { $0.label == "Readable target" }) - - // The presented cut only ever narrows the unscoped capture from the same backend. - let unscoped = try XCTUnwrap(try capture(depth: nil).nodes) - XCTAssertLessThanOrEqual(nodes.count, unscoped.count) - } -#endif } -#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCaptureTarget.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCaptureTarget.swift new file mode 100644 index 0000000000..446db190f2 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCaptureTarget.swift @@ -0,0 +1,106 @@ +import XCTest + +// MARK: - Snapshot capture target (#2781) +// +// Target identity (`currentApp`, `currentBundleId`, `currentAppProcessIdentifier`) and +// `runnerAccessibilityHealth` are owned by main-thread lifecycle code. A capture plan runs on the +// command queue, so it reads the identity from a `SnapshotCaptureTarget` taken on main while the +// command is prepared, and writes health or invalidates the target only through +// `applyMainOwnedSnapshotState`. + +/// The target one capture plan reads, taken once on the main thread. +struct SnapshotCaptureTarget { + let app: XCUIApplication + let bundleId: String? + let processIdentifier: Int? +} + +/// What snapshot command preparation hands the off-main capture. +enum SnapshotCommandPreparation { + case response(Response) + case capture(SnapshotCaptureTarget, systemSurface: SystemSurfaceHost?) +} + +/// The target a bounded XCTest probe arms its abandonment penalty with. +/// +/// The hook that arms the penalty fires on the command queue the moment the probe's slice is spent, +/// while the probe's own work block may still be running on main. `currentBundleId` belongs to main, +/// so it is never read across that boundary: a caller that already took the identity on main hands it +/// over, and a caller that is on the command queue lets the probe's main-side block capture the +/// identity main holds once the work actually starts (#2781). +enum SnapshotProbePenaltyTarget: Equatable { + /// Identity a capture took on main when it prepared its target. + case prepared(bundleId: String?) + /// Read `currentBundleId` inside the probe's main-side block. + case mainOwnedTarget +} + +/// One probe's penalty identity: written by the probe's main-side block, read by the command queue's +/// abandonment hook through this lock. +final class SnapshotProbePenaltyIdentity { + private let lock = NSLock() + private var bundleId: String? + private let readsMainOwnedTarget: Bool + + init(_ target: SnapshotProbePenaltyTarget) { + readsMainOwnedTarget = target == .mainOwnedTarget + if case .prepared(let bundleId) = target { + self.bundleId = bundleId + } + } + + /// Called on the main thread inside the probe's work block, before it enumerates anything, so the + /// identity is the one main had settled on rather than one a queued write is about to replace. + func captureFromMain(bundleId: String?) { + guard readsMainOwnedTarget else { return } + lock.lock() + self.bundleId = bundleId + lock.unlock() + } + + var penalizedBundleId: String? { + lock.lock() + defer { lock.unlock() } + return bundleId + } +} + +extension RunnerTests { + /// Main thread only: reads the lifecycle-owned target identity. + func takeSnapshotCaptureTarget(app: XCUIApplication) -> SnapshotCaptureTarget { + SnapshotCaptureTarget( + app: app, + bundleId: currentBundleId, + processIdentifier: currentAppProcessIdentifier + ) + } + + /// Runs `write` against main-owned runner state for a capture that may be on the command queue. + /// Abandoned work ahead of the hop cannot be cancelled, so behind it the write queues without + /// waiting: the capture answers now and the next command still observes the write. + func applyMainOwnedSnapshotState(_ operation: String, _ write: @escaping () -> Void) { + if Thread.isMainThread { + write() + return + } + guard !hasAbandonedMainThreadWork() else { + NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_STATE_DEFERRED_XCTEST_OCCUPIED operation=%@", operation) + DispatchQueue.main.async(execute: write) + return + } + do { + try runMainThreadWork( + operation, + timeout: 1, + timeoutError: mainThreadExecutionTimeoutError, + write + ) + } catch { + NSLog( + "AGENT_DEVICE_RUNNER_SNAPSHOT_STATE_FAILED operation=%@ error=%@", + operation, + String(describing: error) + ) + } + } +} diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotExecution.swift new file mode 100644 index 0000000000..ec0b2c8faa --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotExecution.swift @@ -0,0 +1,142 @@ +import XCTest +import AgentDeviceSnapshotPresentation + +extension RunnerTests { + func executeSnapshotDispatched(command: Command) throws -> Response { + try executeDispatchedWithRecovery(command: command) { + try self.executeSnapshotDispatchedOnce(command: command) + } + } + + private func executeSnapshotDispatchedOnce(command: Command) throws -> Response { + let preparation: SnapshotCommandPreparation = try runMainThreadWork( + "command_preparation", + timeout: mainThreadExecutionTimeout, + timeoutError: mainThreadExecutionTimeoutError + ) { () -> SnapshotCommandPreparation in + switch try self.prepareActiveCommandContextSafely(command: command, routeToSpringboard: false) { + case .response(let response): + return .response(response) + case .context(let context): + return .capture( + self.takeSnapshotCaptureTarget(app: context.app), + systemSurface: context.systemSurface + ) + } + } + switch preparation { + case .response(let response): + return response + case .capture(let target, let systemSurface): + return try executeSnapshotPrepared( + command: command, + target: target, + systemSurface: systemSurface + ) + } + } + + private func prepareActiveCommandContextSafely( + command: Command, + routeToSpringboard: Bool + ) throws -> ActiveCommandPreparation { + var preparation: ActiveCommandPreparation? + let exceptionMessage = RunnerObjCExceptionCatcher.catchException({ + preparation = self.prepareActiveCommandContext( + command: command, + routeToSpringboard: routeToSpringboard + ) + }) + if let exceptionMessage { + throw NSError( + domain: RunnerErrorDomain.exception, + code: RunnerErrorCode.objcException, + userInfo: [NSLocalizedDescriptionKey: exceptionMessage] + ) + } + guard let preparation else { + throw NSError( + domain: RunnerErrorDomain.general, + code: RunnerErrorCode.commandReturnedNoResponse, + userInfo: [NSLocalizedDescriptionKey: "snapshot preflight returned no response"] + ) + } + return preparation + } + + /// Pure command→options projection, extracted so the runner unit bundle can + /// prove the decoded wire field actually reaches presentation options (#1634 P2). + static func presentationOptions(from command: Command) -> PresentationOptions { + let customActions = command.customActions ?? false + return PresentationOptions( + interactiveOnly: command.interactiveOnly ?? false, + depth: command.depth, + scope: command.scope, + raw: command.raw ?? false, + // Custom actions are only readable through the private AX client, so + // asking for them pins that backend rather than silently returning a + // capture that structurally cannot carry them. An explicit pin wins. + preferredBackend: command.preferredBackend + ?? (customActions ? SnapshotBackendKind.privateAX.rawValue : nil), + customActions: customActions + ) + } + + private func executeSnapshotPrepared( + command: Command, + target: SnapshotCaptureTarget, + systemSurface: SystemSurfaceHost? + ) throws -> Response { + let options = Self.presentationOptions(from: command) + do { + var payload: DataPayload + if options.raw { + payload = try snapshotRaw(target: target, options: options) + } else { + payload = try snapshotFast(target: target, options: options) + } + if let systemSurface { + payload.systemSurface = SystemSurfaceProvenancePayload( + bundleId: systemSurface.bundleId, + kind: systemSurface.kind.rawValue + ) + } + setNeedsPostSnapshotInteractionDelay() + return Response(ok: true, data: payload) + } catch let failure as SnapshotCaptureFailure { + invalidateCachedTargetAfterSnapshotFailure() + return Response( + ok: false, + error: ErrorPayload( + code: failure.code, + message: failure.message, + hint: failure.hint + ) + ) + } + } + + func setNeedsPostSnapshotInteractionDelay() { + guard !hasAbandonedMainThreadWork() else { + NSLog("AGENT_DEVICE_RUNNER_POST_SNAPSHOT_DELAY_MARK_SKIPPED_XCTEST_OCCUPIED") + return + } + do { + try runMainThreadWork( + "post_snapshot_delay_mark", + timeout: 1, + timeoutError: mainThreadExecutionTimeoutError + ) { + self.needsPostSnapshotInteractionDelay = true + } + } catch { + NSLog("AGENT_DEVICE_RUNNER_POST_SNAPSHOT_DELAY_MARK_FAILED=%@", String(describing: error)) + } + } + + func invalidateCachedTargetAfterSnapshotFailure() { + applyMainOwnedSnapshotState("target_invalidation") { + self.invalidateCachedTarget(reason: "ax_snapshot_failure") + } + } +} diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotTiming.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotTiming.swift index 442aa6d1b9..fd11051e0f 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotTiming.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotTiming.swift @@ -57,6 +57,35 @@ struct SnapshotPhaseTimer { } } +/// Keeps the first capture plan that runs against a fresh target process from penalizing the XCTest +/// channel for a slow tier. Lifecycle code arms and disarms it on main; the capture plan consumes it +/// on the command queue, so a snapshot that returns before running a plan leaves it pending. +final class SnapshotXCTestPenaltyWarmupExemption { + private let lock = NSLock() + private var pending = false + + var isPending: Bool { + get { + lock.lock() + defer { lock.unlock() } + return pending + } + set { + lock.lock() + pending = newValue + lock.unlock() + } + } + + func consume() -> Bool { + lock.lock() + defer { lock.unlock() } + let wasPending = pending + pending = false + return wasPending + } +} + extension RunnerTests { struct SnapshotBackendAttempt { enum Outcome { @@ -69,11 +98,25 @@ extension RunnerTests { /// or error text. let outcome: Outcome let timing: SnapshotCaptureTiming + /// Whether the tier finished collecting or stopped at its own deadline. A tier that stopped at + /// its deadline timed out even when it handed back a payload, so penalty and recovery policy + /// read this instead of classifying the payload (#2781). + let tierOutcome: SnapshotTierOutcome + + init( + outcome: Outcome, + timing: SnapshotCaptureTiming, + tierOutcome: SnapshotTierOutcome = .completed + ) { + self.outcome = outcome + self.timing = timing + self.tierOutcome = tierOutcome + } } /// The penalty breaker observes only acquisition facts. Presentation is a separate phase and /// cannot arm the breaker, even when it is slower than the acquisition that produced the tree. - private static func snapshotXCTestPenaltyReason( + static func snapshotXCTestPenaltyReason( kind: SnapshotBackendKind, attempt: SnapshotBackendAttempt, slowThresholdMs: Double @@ -84,6 +127,9 @@ extension RunnerTests { { return "\(kind.rawValue)_backend_timeout" } + if attempt.tierOutcome == .deadlineExhausted { + return "\(kind.rawValue)_backend_timeout" + } guard attempt.timing.acquisitionMs > slowThresholdMs else { return nil } return "slow_\(kind.rawValue)_capture_\(Int(attempt.timing.acquisitionMs))ms" } @@ -91,6 +137,7 @@ extension RunnerTests { func recordXCTestSnapshotBackendAttemptIfNeeded( _ kind: SnapshotBackendKind, attempt: SnapshotBackendAttempt, + bundleId: String?, penaltySuppressed: Bool ) { guard !penaltySuppressed else { return } @@ -102,84 +149,8 @@ extension RunnerTests { ) else { return } penalizeSnapshotXCTestChannel( - bundleId: currentBundleId, + bundleId: bundleId, reason: reason ) } } - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS -extension RunnerTests { - func testXCTestPenaltyDecisionSeparatesAcquisitionAndPresentation() { - let slowPresentation = SnapshotBackendAttempt( - outcome: .noCapture, - timing: SnapshotCaptureTiming(acquisitionMs: 100, presentationMs: 4_000) - ) - XCTAssertNil( - Self.snapshotXCTestPenaltyReason( - kind: .recursiveTree, - attempt: slowPresentation, - slowThresholdMs: 3_000 - ) - ) - - let slowAcquisition = SnapshotBackendAttempt( - outcome: .noCapture, - timing: SnapshotCaptureTiming(acquisitionMs: 3_001, presentationMs: 100) - ) - XCTAssertEqual( - Self.snapshotXCTestPenaltyReason( - kind: .recursiveTree, - attempt: slowAcquisition, - slowThresholdMs: 3_000 - ), - "slow_tree_capture_3001ms" - ) - - let timeout = SnapshotCaptureFailure( - code: Self.xCTestSnapshotTimeoutCode, - message: "test timeout", - hint: "test" - ) - let acquisitionFailure = SnapshotBackendAttempt( - outcome: .failed(timeout, phase: .acquisition), - timing: SnapshotCaptureTiming(acquisitionMs: 100, presentationMs: 100) - ) - XCTAssertEqual( - Self.snapshotXCTestPenaltyReason( - kind: .recursiveTree, - attempt: acquisitionFailure, - slowThresholdMs: 3_000 - ), - "tree_backend_timeout" - ) - - let presentationFailure = SnapshotBackendAttempt( - outcome: .failed(timeout, phase: .presentation), - timing: SnapshotCaptureTiming(acquisitionMs: 100, presentationMs: 100) - ) - XCTAssertNil( - Self.snapshotXCTestPenaltyReason( - kind: .recursiveTree, - attempt: presentationFailure, - slowThresholdMs: 3_000 - ) - ) - } - - func testSnapshotPhaseTimerReportsAcquisitionAndPresentationSeparately() { - var now = Date(timeIntervalSinceReferenceDate: 100) - var timer = SnapshotPhaseTimer(now: { now }) - - _ = timer.measure(.acquisition) { - now = now.addingTimeInterval(2) - } - _ = timer.measure(.presentation) { - now = now.addingTimeInterval(5) - } - - XCTAssertEqual(timer.timing.acquisitionMs, 2_000, accuracy: 0.001) - XCTAssertEqual(timer.timing.presentationMs, 5_000, accuracy: 0.001) - } -} -#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedCommitDeadline.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedCommitDeadline.swift index b88d2d5aa9..08edcf6cd8 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedCommitDeadline.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedCommitDeadline.swift @@ -1,11 +1,11 @@ import XCTest -// The synthesized text-entry commit wait, end to end: the deadline that bounds it, the two waits -// that run it, the observation/pacing they poll through, and the value-free cadence line that path -// is allowed to log. Split from RunnerTests+SynthesizedTextEntry.swift, which keeps the -// private-XCTest synthesis boundary and the route policies; the pure outcome functions these wrap -// stay there next to the rules they encode. Everything that touches the polled field value now -// lives in this one file, which is the surface apple-runner-log-redaction.test.ts guards. +// The synthesized replacement commit wait, end to end: the deadline that bounds it, the wait that +// runs it, the observation/pacing it polls through, and the value-free cadence line that path is +// allowed to log. RunnerTests+SynthesizedTextEntry.swift keeps the private-XCTest synthesis +// boundary, the route policies, and the pure outcome function this wraps. Everything that touches +// the polled field value lives in this one file, which is the surface +// apple-runner-log-redaction.test.ts guards. extension RunnerTests { /// One commit wait's running deadline. /// @@ -57,54 +57,17 @@ extension RunnerTests { } } - /// Blocks until the synthesized bare-type text is observable in the target field, so `type` - /// cannot report ok while trailing characters are still uncommitted on a slow simulator. - /// - /// Observation only. A stalled prefix cannot be told apart from a suffix still queued in the - /// event stream, so re-synthesizing the difference risks committing it twice after the command - /// already reported success (#1676 rejected exactly that repair). Reporting `.notObserved` is - /// what the caller does instead: the partial value is the agent's to resolve, and a named - /// failure beats a success that misdescribes the field. Text carrying a submit key is skipped - /// outright: the app may clear or rewrite the field on submit, so `textBefore + typedText` is - /// not the value to wait for. - func awaitSynthesizedFirstResponderCommit( - app: XCUIApplication, - target: TextEntryTarget, - textBefore: String?, - typedText: String - ) -> SynthesizedTextCommitOutcome { - guard let textBefore, !typedText.contains("\n"), !typedText.contains("\r") else { - return .unobservable - } - let expectedText = textBefore + typedText - let waitStartedAt = Date() - NSLog("[DEBUG-1874] wait start expectedLen=%ld route=append", expectedText.count) - let ingredients = synthesizedCommitPollingIngredients(app: app, target: target, expectedText: expectedText) - let outcome = Self.awaitSynthesizedCommitOutcome( - expectedText: expectedText, - placeholder: ingredients.placeholder, - now: { Date() }, - observe: ingredients.observe, - waitForNextObservation: ingredients.waitForNextObservation - ) - NSLog( - "[DEBUG-1874] wait outcome=%@ elapsedMs=%.0f route=append", - String(describing: outcome), - waitStartedAt.timeIntervalSinceNow * -1000 - ) - return outcome - } - /// Blocks until the synthesized replacement text (`fill`) is observable in the target field, so /// `fill` cannot report ok while the select-all-and-retype it posted is still uncommitted — or /// silently wrong — on a slow or channel-penalized simulator (this route runs only when the - /// XCTest channel is already penalized, and never resolves an `XCUIElement`, so it previously had - /// no verification at all). + /// XCTest channel is already penalized, and never resolves an `XCUIElement`). /// - /// Unlike the append route, the expected value is the final text itself — replacement mode - /// clears the field first, so there is no `textBefore` prefix to account for — and unlike the - /// append route, a settled mismatch is always reported rather than trusted: see - /// `awaitSynthesizedReplacementCommitOutcome`'s doc comment. + /// Observation only. A stalled prefix cannot be told apart from a suffix still queued in the + /// event stream, so re-synthesizing the difference risks committing it twice after the command + /// already reported success. The expected value is the + /// final text itself, and a settled mismatch is always reported: see + /// `awaitSynthesizedReplacementCommitOutcome`'s doc comment. Text carrying a submit key is + /// skipped outright: the app may clear or rewrite the field on submit. func awaitSynthesizedReplacementCommit( app: XCUIApplication, target: TextEntryTarget, @@ -160,16 +123,9 @@ extension RunnerTests { ) } - /// The placeholder/observe/pacing ingredients shared by the append route - /// (`awaitSynthesizedFirstResponderCommit`) and the replacement route - /// (`awaitSynthesizedReplacementCommit`). What must NOT be shared is which outcome function - /// consumes them: see `awaitSynthesizedReplacementCommitOutcome`'s doc comment for why append - /// mode's "trust a diverged value" rule is wrong for replacement mode. Each caller therefore - /// calls its own named outcome function directly, with real argument labels — deliberately not - /// a stored closure/function-value parameter here, which would erase those labels at the call - /// site and make the observe closure unrecognizable to the static content-redaction check in - /// `apple-runner-log-redaction.test.ts` (`extractObserveClosure` locates the labeled closure - /// literal by its text; a closure passed as a plain function value carries no such label). + /// The placeholder/observe/pacing ingredients the commit wait polls through. The observe closure + /// stays a labeled closure literal here: the static content-redaction check in + /// `apple-runner-log-redaction.test.ts` (`extractObserveClosure`) locates it by that text. private func synthesizedCommitPollingIngredients( app: XCUIApplication, target: TextEntryTarget, @@ -189,7 +145,7 @@ extension RunnerTests { treatingPlaceholderAsEmpty: true ) // Cadence evidence stays value-free: the polled value is user content typed through - // `type`/`fill` and must never reach runner.log. Lengths and the expected-prefix walk + // `fill` and must never reach runner.log. Lengths and the expected-prefix walk // are enough to distinguish throttling (prefix grows slowly) from a wedge (it freezes). Self.logCommitCadence( elapsedMs: Int(waitStartedAt.timeIntervalSinceNow * -1000), diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift index c22c0cfb66..212d136781 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift @@ -14,15 +14,12 @@ enum RunnerAccessibilityHealth: String, Equatable { enum SynthesizedKeyboardPolicy: String, Equatable, Hashable { case never - case whenAccessibilityHealthy case requiredWhenAvailable func allowsProbe(accessibilityHealth: RunnerAccessibilityHealth) -> Bool { switch self { case .never: return false - case .whenAccessibilityHealthy: - return accessibilityHealth == .healthy case .requiredWhenAvailable: return accessibilityHealth != .unavailable } @@ -59,23 +56,21 @@ struct SynthesizedGesturePolicy: Equatable, Hashable { struct SynthesizedCoordinateContext { let referenceFrame: CGRect + /// The window `referenceFrame` was measured on. Synthesized records route their display ID + /// through this same window so geometry and routing can never name different windows. + let resolvedWindow: XCUIElement let keyboardPolicy: SynthesizedKeyboardPolicy - let fallbackPolicy: SynthesizedFallbackPolicy let accessibilityHealth: RunnerAccessibilityHealth func withReferenceFrame(_ frame: CGRect) -> SynthesizedCoordinateContext { SynthesizedCoordinateContext( referenceFrame: frame, + resolvedWindow: resolvedWindow, keyboardPolicy: keyboardPolicy, - fallbackPolicy: fallbackPolicy, accessibilityHealth: accessibilityHealth ) } - var allowsXCTestCoordinateFallback: Bool { - fallbackPolicy.allowsXCTestCoordinateFallback(accessibilityHealth: accessibilityHealth) - } - var allowsKeyboardProbe: Bool { keyboardPolicy.allowsProbe(accessibilityHealth: accessibilityHealth) } @@ -111,10 +106,22 @@ func shouldProbeCoordinateTapTextInput(xCTestChannelPenalized: Bool) -> Bool { !xCTestChannelPenalized } -func sequenceHasSynthesizedCoordinateStep(_ steps: [SequenceStep]) -> Bool { - steps.contains { step in - step.synthesized == true && step.kind == "tap" - } +/// A synthesized `tap` step in a `sequence` is a standalone coordinate tap and follows its policy. +func synthesizedPolicyKind(forSequenceStep step: SequenceStep) -> SynthesizedGesturePolicyKind? { + step.synthesized == true && step.kind == "tap" ? .coordinateTap : nil +} + +/// A synthesized gesture's result for its call site, which owns the XCTest coordinate gesture. +enum SynthesizedGestureAttempt { + case performed(timing: (gestureStartUptimeMs: Double, gestureEndUptimeMs: Double)) + /// Synthesis failed and the policy allows the call site's XCTest coordinate gesture. + case xctestFallback(message: String, hint: String?) + /// Synthesis failed and the policy refuses an XCTest coordinate gesture. + case refused( + timing: (gestureStartUptimeMs: Double, gestureEndUptimeMs: Double), + message: String, + hint: String? + ) } extension RunnerTests { @@ -122,11 +129,36 @@ extension RunnerTests { steps: [SequenceStep], app: XCUIApplication ) -> SynthesizedCoordinateContext? { - guard sequenceHasSynthesizedCoordinateStep(steps) else { return nil } - return synthesizedCoordinateContext( - app: app, - policy: synthesizedGesturePolicy(.synthesizedDrag) + guard let kind = steps.lazy.compactMap(synthesizedPolicyKind(forSequenceStep:)).first else { + return nil + } + return synthesizedCoordinateContext(app: app, policy: synthesizedGesturePolicy(kind)) + } + + /// `context` is nil when no window frame resolved; `kind`'s fallback policy then reads the + /// runner's current accessibility health. + func performSynthesizedGesture( + _ app: XCUIApplication, + kind: SynthesizedGesturePolicyKind, + context: SynthesizedCoordinateContext?, + synthesize: () -> RunnerInteractionOutcome + ) -> SynthesizedGestureAttempt { + let (timing, outcome) = performGesture(app, idleTimeout: false, synthesize) + guard case .unsupported(let message, let hint) = outcome else { + logSynthesizedGesturePolicyDecision(kind: kind, context: context, fallbackAttempted: false) + return .performed(timing: timing) + } + let fallbackAllowed = synthesizedGesturePolicy(kind).fallbackPolicy.allowsXCTestCoordinateFallback( + accessibilityHealth: context?.accessibilityHealth ?? runnerAccessibilityHealth + ) + logSynthesizedGesturePolicyDecision( + kind: kind, + context: context, + fallbackAttempted: fallbackAllowed ) + return fallbackAllowed + ? .xctestFallback(message: message, hint: hint) + : .refused(timing: timing, message: message, hint: hint) } func logSynthesizedGesturePolicyDecision( @@ -135,104 +167,35 @@ extension RunnerTests { fallbackAttempted: Bool ) { #if os(iOS) - guard let context else { - NSLog( - "AGENT_DEVICE_RUNNER_SYNTHESIZED_GESTURE_POLICY kind=%@ context=unavailable fallbackAttempted=%@", - kind.rawValue, - fallbackAttempted.description - ) - return + let line = Self.synthesizedGesturePolicyLine( + kind: kind, + context: context, + fallbackAttempted: fallbackAttempted + ) + // The same decision for the same gesture kind on every command is one line; a changed policy + // (AX health, keyboard, fallback) is a new one. + if lastLoggedGesturePolicyLines[kind] != line { + lastLoggedGesturePolicyLines[kind] = line + runnerMarkerWriter(line) } - NSLog( - "AGENT_DEVICE_RUNNER_SYNTHESIZED_GESTURE_POLICY kind=%@ axHealth=%@ frameSource=screenshot keyboardPolicy=%@ fallbackPolicy=%@ fallbackAllowed=%@ fallbackAttempted=%@", - kind.rawValue, - context.accessibilityHealth.rawValue, - context.keyboardPolicy.rawValue, - context.fallbackPolicy.rawValue, - context.allowsXCTestCoordinateFallback.description, - fallbackAttempted.description - ) #endif } -} - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS -extension RunnerTests { - func testSynthesizedFallbackPolicyRequiresPrivateSynthesisForScrollWhenAxUnavailableOrUnknown() { - XCTAssertFalse( - SynthesizedFallbackPolicy.privateSynthesisRequired - .allowsXCTestCoordinateFallback(accessibilityHealth: .unavailable) - ) - XCTAssertFalse( - SynthesizedFallbackPolicy.privateSynthesisRequired - .allowsXCTestCoordinateFallback(accessibilityHealth: .unknown) - ) - XCTAssertFalse( - SynthesizedFallbackPolicy.privateSynthesisRequired - .allowsXCTestCoordinateFallback(accessibilityHealth: .healthy) - ) - } - func testSynthesizedDragCoordinateFallbackAllowsUnknownButNotUnavailableAccessibility() { - XCTAssertTrue( - SynthesizedFallbackPolicy.xctestCoordinateWhenAccessibilityAvailable - .allowsXCTestCoordinateFallback(accessibilityHealth: .healthy) - ) - XCTAssertFalse( - SynthesizedFallbackPolicy.xctestCoordinateWhenAccessibilityAvailable - .allowsXCTestCoordinateFallback(accessibilityHealth: .unavailable) - ) - XCTAssertTrue( - SynthesizedFallbackPolicy.xctestCoordinateWhenAccessibilityAvailable - .allowsXCTestCoordinateFallback(accessibilityHealth: .unknown) - ) - } - - /// Keyboard-policy semantics only. Which command gets which policy is the table below; a probe - /// that is merely permitted still costs a live AX fetch, so the two questions stay separate. - func testSynthesizedKeyboardPolicyAllowsProbeOnlyWhenAccessibilityPermitsIt() { - XCTAssertFalse( - SynthesizedKeyboardPolicy.whenAccessibilityHealthy - .allowsProbe(accessibilityHealth: .unknown) - ) - XCTAssertTrue( - SynthesizedKeyboardPolicy.requiredWhenAvailable - .allowsProbe(accessibilityHealth: .unknown) - ) - XCTAssertFalse( - SynthesizedKeyboardPolicy.requiredWhenAvailable - .allowsProbe(accessibilityHealth: .unavailable) - ) - } - - func testSynthesizedGesturePoliciesMatchCommandContracts() { - XCTAssertEqual( - synthesizedGesturePolicy(.coordinateTap), - SynthesizedGesturePolicy( - keyboardPolicy: .never, - fallbackPolicy: .xctestCoordinateAllowed - ) - ) - XCTAssertEqual( - synthesizedGesturePolicy(.scroll), - SynthesizedGesturePolicy( - keyboardPolicy: .requiredWhenAvailable, - fallbackPolicy: .privateSynthesisRequired - ) - ) - XCTAssertEqual( - synthesizedGesturePolicy(.synthesizedDrag), - SynthesizedGesturePolicy( - keyboardPolicy: .requiredWhenAvailable, - fallbackPolicy: .xctestCoordinateWhenAccessibilityAvailable - ) - ) - } - - func testCoordinateTapTextInputProbeSkipsPenalizedXCTestChannel() { - XCTAssertTrue(shouldProbeCoordinateTapTextInput(xCTestChannelPenalized: false)) - XCTAssertFalse(shouldProbeCoordinateTapTextInput(xCTestChannelPenalized: true)) + static func synthesizedGesturePolicyLine( + kind: SynthesizedGesturePolicyKind, + context: SynthesizedCoordinateContext?, + fallbackAttempted: Bool + ) -> String { + guard let context else { + return "AGENT_DEVICE_RUNNER_SYNTHESIZED_GESTURE_POLICY kind=\(kind.rawValue)" + + " context=unavailable fallbackAttempted=\(fallbackAttempted)" + } + let fallbackPolicy = synthesizedGesturePolicy(kind).fallbackPolicy + return "AGENT_DEVICE_RUNNER_SYNTHESIZED_GESTURE_POLICY kind=\(kind.rawValue)" + + " axHealth=\(context.accessibilityHealth.rawValue) frameSource=window" + + " keyboardPolicy=\(context.keyboardPolicy.rawValue)" + + " fallbackPolicy=\(fallbackPolicy.rawValue)" + + " fallbackAllowed=\(fallbackPolicy.allowsXCTestCoordinateFallback(accessibilityHealth: context.accessibilityHealth))" + + " fallbackAttempted=\(fallbackAttempted)" } - } -#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift new file mode 100644 index 0000000000..369f588cf9 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift @@ -0,0 +1,583 @@ +import XCTest +import AgentDeviceSnapshotPresentation + +extension RunnerTests { + func synthesizedDragAt( + app: XCUIApplication, + x: Double, + y: Double, + x2: Double, + y2: Double, + durationMs: Double, + profile: SynthesizedDragProfile, + context: SynthesizedCoordinateContext? = nil + ) -> RunnerInteractionOutcome { +#if os(iOS) + guard x.isFinite, y.isFinite, x2.isFinite, y2.isFinite else { + return .unsupported( + message: "synthesized coordinate drag requires finite coordinates", + hint: "Retry with finite x, y, x2, and y2 values." + ) + } + let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)) + guard let context = context ?? synthesizedCoordinateContext( + app: app, + policy: synthesizedGesturePolicy(.synthesizedDrag) + ) else { + return .unsupported( + message: "synthesized coordinate drag could not resolve an app window with a finite screen frame", + hint: "Retry after the app is foregrounded, or use a plain screenshot to choose coordinates." + ) + } + let frame = context.referenceFrame + let start = CoordinateSpaceRotation.native( + point: CGPoint(x: x, y: y), + in: frame, + interfaceOrientation: orientation + ) + let end = CoordinateSpaceRotation.native( + point: CGPoint(x: x2, y: y2), + in: frame, + interfaceOrientation: orientation + ) + logSynthesizedDispatch( + kind: "drag", + start: start, + end: end, + referenceFrame: frame, + orientation: orientation + ) + let message = switch profile { + case .controlledScroll: + RunnerSynthesizedGesture.synthesizeControlledScroll( + withApplication: app, + resolvedWindow: context.resolvedWindow, + x: Double(start.x), + y: Double(start.y), + x2: Double(end.x), + y2: Double(end.y), + durationMs: durationMs + ) + case .fastSwipe: + RunnerSynthesizedGesture.synthesizeSwipe( + withApplication: app, + resolvedWindow: context.resolvedWindow, + x: Double(start.x), + y: Double(start.y), + x2: Double(end.x), + y2: Double(end.y), + durationMs: durationMs + ) + } + if let message { + return .unsupported( + message: message, + hint: "Private XCTest event synthesis is required for AX-free coordinate drag on iOS; update Xcode if this persists." + ) + } + return .performed +#elseif os(tvOS) + return .unsupported( + message: "coordinate drag is not supported on tvOS", + hint: "tvOS has no coordinate input; use remote-driven swipe/scroll to move focus instead." + ) +#else + return .unsupported( + message: "coordinate drag is not supported on macOS", + hint: "macOS automation has no touchscreen; use mouse-driven interactions instead." + ) +#endif + } + + func synthesizedTapAt( + app: XCUIApplication, + x: Double, + y: Double, + context: SynthesizedCoordinateContext? = nil + ) -> RunnerInteractionOutcome { +#if os(iOS) + guard x.isFinite, y.isFinite else { + return .unsupported( + message: "synthesized coordinate tap requires finite coordinates", + hint: "Retry with finite x and y values." + ) + } + let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)) + guard let context = context ?? synthesizedCoordinateContext( + app: app, + policy: synthesizedGesturePolicy(.coordinateTap) + ) else { + return .unsupported( + message: "synthesized coordinate tap could not resolve an app window with a finite screen frame", + hint: "Retry after the app is foregrounded, or use a plain screenshot to choose coordinates." + ) + } + let point = CoordinateSpaceRotation.native( + point: CGPoint(x: x, y: y), + in: context.referenceFrame, + interfaceOrientation: orientation + ) + logSynthesizedDispatch( + kind: "tap", + start: point, + end: nil, + referenceFrame: context.referenceFrame, + orientation: orientation + ) + if let message = RunnerSynthesizedGesture.synthesizeTap( + withApplication: app, + resolvedWindow: context.resolvedWindow, + x: Double(point.x), + y: Double(point.y) + ) { + return .unsupported( + message: message, + hint: "Falling back to XCTest coordinate tap may be slower and can still need a healthy accessibility tree." + ) + } + return .performed +#elseif os(tvOS) + return .unsupported( + message: "coordinate tap is not supported on tvOS; move focus with swipe or scroll, then select the focused element", + hint: "tvOS has no coordinate input; move focus with swipe/scroll to the target, then select it." + ) +#else + return .unsupported( + message: "synthesized coordinate tap is not supported on macOS", + hint: "macOS automation has no touchscreen; use mouse-driven interactions instead." + ) +#endif + } + + func keyboardAvoidingDragPoints( + app: XCUIApplication, + x: Double, + y: Double, + x2: Double, + y2: Double + ) -> DragPoints { + let original = DragPoints(x: x, y: y, x2: x2, y2: y2) +#if os(iOS) + guard let keyboardFrame = visibleKeyboardFrame(app: app) else { + return original + } + let minX = min(x, x2) + let minY = min(y, y2) + let gestureBounds = CGRect( + x: CGFloat(minX), + y: CGFloat(minY), + width: CGFloat(max(abs(x2 - x), 1)), + height: CGFloat(max(abs(y2 - y), 1)) + ) + guard gestureBounds.intersects(keyboardFrame) else { + return original + } + + let appFrame = onScreenWindowFrame(app: app) + guard !appFrame.isEmpty else { + return original + } + + let padding: Double = 12 + let targetMaxY = Double(keyboardFrame.minY) - padding + let currentMaxY = max(y, y2) + let shift = currentMaxY - targetMaxY + guard shift > 0 else { + return original + } + + let adjustedY = y - shift + let adjustedY2 = y2 - shift + guard min(adjustedY, adjustedY2) >= Double(appFrame.minY) + padding else { + return original + } + + NSLog( + "AGENT_DEVICE_RUNNER_KEYBOARD_AVOIDING_DRAG from=(%.1f,%.1f)->(%.1f,%.1f) adjusted=(%.1f,%.1f)->(%.1f,%.1f) keyboardMinY=%.1f", + x, + y, + x2, + y2, + x, + adjustedY, + x2, + adjustedY2, + Double(keyboardFrame.minY) + ) + return DragPoints(x: x, y: adjustedY, x2: x2, y2: adjustedY2) +#else + return original +#endif + } + + func resolvedTouchVisualizationFrame(app: XCUIApplication, x: Double, y: Double) -> TouchVisualizationFrame { + let appFrame = app.frame + let referenceFrame = resolvedTouchReferenceFrame(app: app, appFrame: appFrame) + let originX = appFrame.isEmpty ? referenceFrame.minX : appFrame.minX + let originY = appFrame.isEmpty ? referenceFrame.minY : appFrame.minY + return TouchVisualizationFrame( + x: originX + x, + y: originY + y, + referenceWidth: referenceFrame.width, + referenceHeight: referenceFrame.height + ) + } + + func resolvedDragVisualizationFrame( + app: XCUIApplication, + x: Double, + y: Double, + x2: Double, + y2: Double + ) -> DragVisualizationFrame { + let start = resolvedTouchVisualizationFrame(app: app, x: x, y: y) + let end = resolvedTouchVisualizationFrame(app: app, x: x2, y: y2) + return DragVisualizationFrame( + x: start.x, + y: start.y, + x2: end.x, + y2: end.y, + referenceWidth: start.referenceWidth, + referenceHeight: start.referenceHeight + ) + } + + func resolvedTouchReferenceFrame(app: XCUIApplication, appFrame: CGRect) -> CGRect { + let resolved = resolveRunnerWindow(app: app) + if resolved.window != nil { + return frameAvoidingKeyboard(app: app, frame: resolved.frame) + } + if !appFrame.isEmpty { + return frameAvoidingKeyboard(app: app, frame: appFrame) + } + return CGRect(x: 0, y: 0, width: 0, height: 0) + } + + private func frameAvoidingKeyboard(app: XCUIApplication, frame: CGRect) -> CGRect { +#if os(iOS) + guard let keyboardFrame = visibleKeyboardFrame(app: app), !frame.isEmpty else { + return frame + } + let intersection = frame.intersection(keyboardFrame) + guard !intersection.isNull && intersection.height > 0 else { + return frame + } + let keyboardCoverage = intersection.width / max(frame.width, 1) + guard keyboardCoverage >= 0.5 else { + return frame + } + let safeHeight = keyboardFrame.minY - frame.minY + guard safeHeight >= frame.height * 0.25 else { + return frame + } + return CGRect(x: frame.minX, y: frame.minY, width: frame.width, height: safeHeight) +#else + return frame +#endif + } + + func axFreeSynthesizedDragPlan( + app: XCUIApplication, + x: Double, + y: Double, + x2: Double, + y2: Double, + context: SynthesizedCoordinateContext? = nil + ) -> SynthesizedDragPlan? { +#if os(iOS) + let context = context ?? synthesizedCoordinateContext( + app: app, + policy: synthesizedGesturePolicy(.synthesizedDrag) + ) + guard x.isFinite, y.isFinite, x2.isFinite, y2.isFinite, + let context + else { + return nil + } + let points = keyboardAvoidingSynthesizedDragPoints( + app: app, + x: x, + y: y, + x2: x2, + y2: y2, + context: context + ) + return SynthesizedDragPlan( + points: points, + context: context + ) +#else + return nil +#endif + } + + func axFreeDragVisualizationFrame( + x: Double, + y: Double, + x2: Double, + y2: Double, + referenceFrame: CGRect + ) -> DragVisualizationFrame { + return DragVisualizationFrame( + x: x, + y: y, + x2: x2, + y2: y2, + referenceWidth: Double(referenceFrame.width), + referenceHeight: Double(referenceFrame.height) + ) + } + + func logSynthesizedDispatch( + kind: String, + start: CGPoint, + end: CGPoint?, + referenceFrame: CGRect, + orientation: Int + ) { + if let end { + NSLog( + "AGENT_DEVICE_RUNNER_SYNTHESIZED_DISPATCH kind=%@ start=(%.1f,%.1f) end=(%.1f,%.1f) reference=(%.1f,%.1f,%.1f,%.1f) orientation=%d", + kind, + Double(start.x), + Double(start.y), + Double(end.x), + Double(end.y), + Double(referenceFrame.origin.x), + Double(referenceFrame.origin.y), + Double(referenceFrame.width), + Double(referenceFrame.height), + orientation + ) + return + } + NSLog( + "AGENT_DEVICE_RUNNER_SYNTHESIZED_DISPATCH kind=%@ point=(%.1f,%.1f) reference=(%.1f,%.1f,%.1f,%.1f) orientation=%d", + kind, + Double(start.x), + Double(start.y), + Double(referenceFrame.origin.x), + Double(referenceFrame.origin.y), + Double(referenceFrame.width), + Double(referenceFrame.height), + orientation + ) + } + + func synthesizedCoordinateContext( + app: XCUIApplication, + policy: SynthesizedGesturePolicy + ) -> SynthesizedCoordinateContext? { +#if os(iOS) + let health = runnerAccessibilityHealth + let resolved = resolveRunnerWindow(app: app) + guard let window = resolved.window else { + return nil + } + let referenceFrame = resolved.frame + guard referenceFrame.width.isFinite, referenceFrame.height.isFinite, + referenceFrame.width > 0, referenceFrame.height > 0 + else { + return nil + } + return SynthesizedCoordinateContext( + referenceFrame: referenceFrame, + resolvedWindow: window, + keyboardPolicy: policy.keyboardPolicy, + accessibilityHealth: health + ) +#else + return nil +#endif + } + + + func keyboardAvoidingSynthesizedDragPoints( + app: XCUIApplication, + x: Double, + y: Double, + x2: Double, + y2: Double, + context: SynthesizedCoordinateContext + ) -> DragPoints { +#if os(iOS) + guard context.allowsKeyboardProbe else { + return DragPoints(x: x, y: y, x2: x2, y2: y2) + } + return keyboardAvoidingDragPoints(app: app, x: x, y: y, x2: x2, y2: y2) +#else + return DragPoints(x: x, y: y, x2: x2, y2: y2) +#endif + } + + func swipe(app: XCUIApplication, direction: String) -> DragVisualizationFrame? { + if performTvRemoteSwipeIfAvailable(direction: direction) { + let frame = resolvedTouchReferenceFrame(app: app, appFrame: app.frame) + let midX = frame.midX + let midY = frame.midY + return DragVisualizationFrame( + x: midX, + y: midY, + x2: midX, + y2: midY, + referenceWidth: frame.width, + referenceHeight: frame.height + ) + } + return nil + } + + private func performTvRemoteSwipeIfAvailable(direction: String) -> Bool { + switch direction { + case "up": + return pressTvRemote(.up) + case "down": + return pressTvRemote(.down) + case "left": + return pressTvRemote(.left) + case "right": + return pressTvRemote(.right) + default: + return false + } + } + + func plannedGestureValidationError(_ plan: RunnerGesturePlan) -> String? { + guard plan.topology == "single" || plan.topology == "two" else { + return "planned gesture topology must be single or two" + } + let supportedIntent = plan.topology == "single" + ? plan.intent == "fling" || plan.intent == "pan" + : plan.intent == "pan" || plan.intent == "pinch" || plan.intent == "rotate" + || plan.intent == "transform" + guard supportedIntent else { return "planned gesture has unsupported intent for its topology" } + if plan.topology == "single" { + guard plan.executionProfile == "endpoint-hold" || plan.executionProfile == "timed-pan" else { + return "single-pointer gesture requires a supported execution profile" + } + } else if plan.executionProfile != nil { + return "multi-touch gesture cannot define a single-pointer execution profile" + } + guard plan.durationMs.isFinite, plan.durationMs >= 16, plan.durationMs <= 10_000 else { + return "planned gesture durationMs must be between 16 and 10000" + } + let viewport = plan.viewport + guard viewport.x.isFinite, viewport.y.isFinite, viewport.width.isFinite, + viewport.height.isFinite, viewport.width > 0, viewport.height > 0 + else { + return "planned gesture viewport must be finite and positive" + } + let expectedPointerCount = plan.topology == "single" ? 1 : 2 + guard plan.pointers.count == expectedPointerCount else { + return "planned gesture pointer count does not match topology" + } + for (index, pointer) in plan.pointers.enumerated() where pointer.pointerId != index { + return "planned gesture requires ordered pointer ids" + } + let firstSamples = plan.pointers[0].samples + guard firstSamples.count >= 2 else { return "planned pointer paths require at least two samples" } + for pointer in plan.pointers { + guard pointer.samples.count == firstSamples.count else { + return "planned pointer paths require matching samples" + } + var previousOffset = -1.0 + for (index, sample) in pointer.samples.enumerated() { + guard sample.offsetMs.isFinite, + sample.offsetMs == firstSamples[index].offsetMs, + sample.offsetMs > previousOffset + else { + return "planned pointer sample offsets must match and strictly increase" + } + let point = sample.point + guard point.x.isFinite, point.y.isFinite, + point.x >= viewport.x, + point.x <= viewport.x + viewport.width, + point.y >= viewport.y, + point.y <= viewport.y + viewport.height + else { + return "planned pointer sample lies outside the viewport" + } + previousOffset = sample.offsetMs + } + guard pointer.samples.first?.offsetMs == 0, + pointer.samples.last?.offsetMs == plan.durationMs + else { + return "planned pointer paths must start at 0 and end at durationMs" + } + } + if plan.topology == "two" { + guard let firstStart = firstSamples.first?.point, + let secondStart = plan.pointers[1].samples.first?.point, + hypot(firstStart.x - secondStart.x, firstStart.y - secondStart.y) > 0 + else { + return "planned pointer paths require a positive initial span" + } + } + return nil + } + + func plannedGestureExecution(for plan: RunnerGesturePlan) -> PlannedGestureExecution { + plan.topology == "single" && plan.executionProfile == "endpoint-hold" + ? .fastSwipe + : .sampled + } + + func sampledPlannedGesture( + app: XCUIApplication, + plan: RunnerGesturePlan + ) -> RunnerInteractionOutcome { +#if os(iOS) + let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)) + // The portable planner and validation use this exact viewport. Using app.frame here can + // diverge when XCTest unions transformed/off-screen descendants into the application frame. + let frame = CGRect( + x: plan.viewport.x, + y: plan.viewport.y, + width: plan.viewport.width, + height: plan.viewport.height + ) + let pointerSamples: [[[String: NSNumber]]] = plan.pointers.map { pointer in + pointer.samples.map { sample in + let point = CoordinateSpaceRotation.native( + point: CGPoint(x: sample.point.x, y: sample.point.y), + in: frame, + interfaceOrientation: orientation + ) + return [ + "x": NSNumber(value: Double(point.x)), + "y": NSNumber(value: Double(point.y)), + "offsetMs": NSNumber(value: sample.offsetMs), + ] + } + } + let resolvedWindow = resolveRunnerWindow(app: app).window + if let message = RunnerSynthesizedGesture.synthesizeGesture( + withApplication: app, + resolvedWindow: resolvedWindow, + pointerSamples: pointerSamples + ) { + return .unsupported( + message: message, + hint: "This gesture uses private XCTest event-synthesis APIs; rebuild the runner with a supported Xcode if this persists." + ) + } + return .performed +#elseif os(tvOS) + return .unsupported( + message: "two-finger gestures are not supported on tvOS", + hint: "tvOS has no touch input; use remote-driven navigation." + ) +#elseif os(visionOS) + return .unsupported( + message: "two-finger touch gestures are not supported on visionOS", + hint: "The current XCTest synthesizer supports iOS and iPadOS touch simulators only." + ) +#else + return .unsupported( + message: "two-finger gestures are not supported on macOS", + hint: "macOS automation has no multi-touch input; run on an iOS simulator." + ) +#endif + } + +} diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift index 357af4f8d3..0ea1c6e65e 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift @@ -127,10 +127,9 @@ extension RunnerTests { sleepFor(request.delaySeconds) } } - // The private synthesize call returns at post time, not commit time (same as bare `type`, - // see awaitSynthesizedFirstResponderCommit) — but this route never resolves an XCUIElement, - // so without this wait it had no way to notice a dropped or still-in-flight character at all - // and reported ok purely because the event posted. Wait here, on the same request.target + // The private synthesize call returns at post time, not commit time, and this route never + // resolves an XCUIElement, so without this wait it had no way to notice a dropped or + // still-in-flight character at all. Wait here, on the same request.target // (element nil, refreshPoint set) that gated this route, so each poll re-resolves via the // refresh point rather than trusting a stale element handle. let commit = awaitSynthesizedReplacementCommit( @@ -168,37 +167,16 @@ extension RunnerTests { !hasResolvedElement && hasRefreshPoint && xCTestChannelPenalized } + /// The tap-witness route carries only the bare submit key, the one text the daemon sends without + /// a text-entry mode. Ordinary `type` text arrives in `.append` mode and is typed through the + /// resolved XCUIElement, where it is verified. static func shouldUseSynthesizedFirstResponderType( repairMode: TextTypingRepairMode, + text: String, fromTapWitness: Bool, softwareKeyboardVisible: Bool ) -> Bool { - repairMode == .none && fromTapWitness && !softwareKeyboardVisible - } - - enum SynthesizedTextCommitProgress: Equatable { - case committed - case pending - case diverged - } - - // The private synthesize call returns once the event record is posted, not once the target - // app has committed the characters, so intermediate reads walk prefix-by-prefix toward the - // expected value. Anything off that prefix path means the app transformed the input - // (formatter, mid-text caret, autocomplete) and the runner must not second-guess it. An - // unreadable value — secure field, or the element stopped resolving — ends the wait the - // same way. - static func synthesizedTextCommitProgress( - observedText: String?, - expectedText: String - ) -> SynthesizedTextCommitProgress { - guard let observedText else { - return .diverged - } - if observedText == expectedText { - return .committed - } - return expectedText.hasPrefix(observedText) ? .pending : .diverged + repairMode == .none && text == "\n" && fromTapWitness && !softwareKeyboardVisible } /// Length of the shared prefix of two strings. Feeds value-free commit-wait logging: the @@ -213,73 +191,19 @@ extension RunnerTests { return length } - /// How the commit wait ended. Distinct from `SynthesizedTextCommitProgress`, which classifies a - /// single observation: this is the whole wait's verdict, and it exists so the deadline can be - /// told apart from success. The wait used to return `Void`, which made an expired deadline - /// indistinguishable from a committed one — `type` then reported ok with a partial value in the - /// field (#1874, #1844). + /// How the commit wait ended: the whole wait's verdict, so the deadline can be told apart from + /// success. enum SynthesizedTextCommitOutcome: Equatable { - /// The wait's success case, but its meaning is route-specific: for append mode (bare `type`), - /// the expected text committed OR the app transformed the input in a way the runner must not - /// second-guess; for replacement mode (`fill`), it means only an exact match — see - /// `awaitSynthesizedReplacementCommitOutcome`'s doc comment for why replacement mode has no - /// "trust it" case. + /// The field holds exactly the expected text. case settled - /// There was nothing to wait for — no readable baseline, or the text carries a submit key. + /// There was nothing to wait for: the text carries a submit key. case unobservable /// The deadline expired with the expected text still not observed. case notObserved } - /// The commit wait's decision, with observation, pacing and the clock injected so both deadline - /// branches are exercisable without a simulator (the macOS host lane runs this; the member - /// wrapper below binds the real XCUI reads). - /// - /// The deadline is a local `var`, started from this loop's own first `now()` and advanced from - /// the same observation the progress check reads, so "did the burst move" and "is time up" are - /// two statements in one loop. The budget defaults to the shipped one, so only a test that is - /// asking about time has to name it. - static func awaitSynthesizedCommitOutcome( - expectedText: String, - placeholder: String?, - stallBudget: TimeInterval = TextEntryTiming.synthesizedCommitStallTimeout, - ceiling: TimeInterval = TextEntryTiming.synthesizedCommitCeiling, - now: () -> Date = { Date() }, - observe: () -> String?, - waitForNextObservation: () -> Void - ) -> SynthesizedTextCommitOutcome { - // A placeholder-equal AX value cannot prove a commit: an input handler may clear even a - // previously non-empty field after dispatch, making the empty field render the same value. - // Refuse before polling because no later read can distinguish those states. - if Self.textMatchesPlaceholder(expectedText, placeholder: placeholder) { - return .notObserved - } - var deadline = SynthesizedCommitDeadline(startedAt: now(), stallBudget: stallBudget, ceiling: ceiling) - // The deadline is checked AFTER an observation, never before one, so the last thing that - // happens before condemning a commit is a read. Checking first would condemn a commit that - // landed during the final poll sleep — the exact loaded-host timing this wait exists for. - while true { - let observedText = observe() - switch synthesizedTextCommitProgress(observedText: observedText, expectedText: expectedText) { - case .committed, .diverged: - return .settled - case .pending: - // One clock sample, so the instant an observation is recorded at is the instant it is - // judged against. - let sampledAt = now() - deadline.record( - expectedPrefixLength: Self.commonPrefixLength(observedText ?? "", expectedText), - at: sampledAt - ) - if deadline.isExpired(at: sampledAt) { return .notObserved } - waitForNextObservation() - } - } - } - - /// The command-level consequence of a commit wait. `.unobservable` is not a failure: there was - /// no baseline to compare against, which is the pre-existing contract for submit-key text and - /// unreadable fields, not evidence that anything went wrong. + /// The command-level consequence of a commit wait. `.unobservable` is not a failure: the app + /// may clear or rewrite the field on submit, so there is no value to compare against. static func textEntryFailure( forCommitOutcome outcome: SynthesizedTextCommitOutcome ) -> TextEntryFailure? { @@ -291,22 +215,16 @@ extension RunnerTests { } } - /// The replacement-mode counterpart of `awaitSynthesizedCommitOutcome`. It must NOT reuse that - /// function's `synthesizedTextCommitProgress`: prefix-walk's `.diverged` case exists to trust an - /// app that transforms bare-`type` input (formatter, autocomplete) rather than second-guess it — - /// but that same rule silently accepts a dropped-character corruption too, because a value with a - /// hole in the middle ("ada@example" -> "aexample") is neither a matching prefix NOR the full - /// string, yet still gets classified `.diverged` -> `.settled` -> reported `ok: true`. This is not - /// a hypothetical: it is the exact shape of the corruption this wait exists to catch (`fill` - /// reporting success over "Avelace"/"aexample"-style drops), verified against both live examples - /// before writing this comment. + /// The replacement commit wait's decision, with observation, pacing and the clock injected so + /// both deadline branches are exercisable without a simulator (the macOS host lane runs this; + /// `awaitSynthesizedReplacementCommit` binds the real XCUI reads). The budget defaults to the + /// shipped one, so only a test that is asking about time has to name it. /// - /// `.replacement` mode does not need prefix tolerance for legitimate transforms either: - /// `isRepairableTextEntryMismatch` (RunnerTests+TextTyping.swift) already treats every mismatch in - /// `.replacement` mode as repairable unconditionally, with no formatter/autocomplete carve-out — - /// `fill` fully owns the field via select-all, so there is no legitimate reason for the settled - /// value to be anything other than exactly what was requested. A settled non-match is therefore - /// always the wait's failure case, never a `.settled` pass-through. + /// Only an exact match settles. A value with a hole in the middle ("ada@example" -> "aexample") + /// is the corruption this wait exists to catch, and `.replacement` mode has no formatter or + /// autocomplete carve-out: `isRepairableTextEntryMismatch` (RunnerTests+TextTyping.swift) treats + /// every `.replacement` mismatch as repairable, because `fill` owns the whole field via + /// select-all. A settled non-match is therefore always the wait's failure case. static func awaitSynthesizedReplacementCommitOutcome( expectedText: String, placeholder: String?, @@ -316,18 +234,24 @@ extension RunnerTests { observe: () -> String?, waitForNextObservation: () -> Void ) -> SynthesizedTextCommitOutcome { + // A placeholder-equal AX value cannot prove a commit: an input handler may clear the field + // after dispatch, making the empty field render the same value. Refuse before polling because + // no later read can distinguish those states. if Self.textMatchesPlaceholder(expectedText, placeholder: placeholder) { return .notObserved } var deadline = SynthesizedCommitDeadline(startedAt: now(), stallBudget: stallBudget, ceiling: ceiling) + // The deadline is checked AFTER an observation, never before one, so the last thing that + // happens before condemning a commit is a read. Checking first would condemn a commit that + // landed during the final poll sleep — the exact loaded-host timing this wait exists for. while true { let observedText = observe() if observedText == expectedText { return .settled } - // Prefix growth cannot settle this wait — a value with a hole in the middle is still a - // failure, see the doc comment above — but it is the same evidence that the burst is still - // landing, so it buys the same time here as it does in append mode. + // Prefix growth cannot settle this wait, but it is evidence that the burst is still + // landing, so it buys time. One clock sample, so the instant an observation is recorded at + // is the instant it is judged against. let sampledAt = now() deadline.record( expectedPrefixLength: Self.commonPrefixLength(observedText ?? "", expectedText), @@ -346,11 +270,4 @@ extension RunnerTests { ) -> Bool { repairMode == .replacement && hasX && hasY && xCTestChannelPenalized } - - static func shouldFallbackFromSynthesizedTextEntryFocus( - _ outcome: RunnerInteractionOutcome - ) -> Bool { - if case .unsupported = outcome { return true } - return false - } } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SystemModal.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SystemModal.swift index 5618d78358..c05d3fb2f9 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SystemModal.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SystemModal.swift @@ -69,15 +69,7 @@ extension RunnerTests { in springboard: XCUIApplication, deadline: Date = .distantFuture ) -> XCUIElement? { - let disableSafeProbe = RunnerEnv.isTruthy("AGENT_DEVICE_RUNNER_DISABLE_SAFE_MODAL_PROBE") - let queryElements: (() -> [XCUIElement]) -> [XCUIElement] = { fetch in - if disableSafeProbe { - return fetch() - } - return self.safeElementsQuery(fetch) - } - - let alerts = queryElements { + let alerts = safeElementsQuery { springboard.alerts.allElementsBoundByIndex } for alert in alerts { @@ -92,7 +84,7 @@ extension RunnerTests { return nil } - let sheets = queryElements { + let sheets = safeElementsQuery { springboard.sheets.allElementsBoundByIndex } for sheet in sheets { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift index c3b3ec4b05..8aed63e9a5 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift @@ -25,7 +25,7 @@ extension RunnerTests { case .notFocused: return "Focus a visible text input, then retry type or fill. If the input is not exposed by accessibility, use a coordinate focus command before typing." case .synthesisUnavailable: - return "Show the software keyboard, then retry type or fill." + return "Show the software keyboard, then retry type." case .commitNotObserved: return "The field may hold none, part, or all of the text. Run snapshot -i and inspect the field: if it already matches, continue; otherwise retry fill with the full text quoted and --delay-ms 80. Do not use type, which appends to whatever committed." } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift index da780c8270..6d49503938 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift @@ -1,8 +1,8 @@ import XCTest // Text-entry target acquisition: choosing the element a `type`/`fill` will address and getting -// focus onto it — the one-shot tap witness, the post-tap stabilization, and the two -// `focusTextInputForTextEntry` entry points the command layer calls. Whether that target is ready +// focus onto it — the one-shot tap witness, the post-tap stabilization, and the +// `focusTextInputForTextEntry` entry point the command layer calls. Whether that target is ready // is RunnerTests+TextEntryReadiness.swift's question, and this file asks it rather than answering // it. extension RunnerTests { @@ -152,41 +152,6 @@ extension RunnerTests { ) } - func focusTextInputForTextEntry(app: XCUIApplication, element: XCUIElement) -> TextEntryTarget { - let point = textEntryRefreshPoint(for: element) - let keyboardVisibleBeforeTap = isKeyboardVisible(app: app) - if let point { - _ = tapAt(app: app, x: point.x, y: point.y) - } - // See the coordinate-target path above: direct element typing keeps this scoped to the - // tapped target, while the first-character warmup and final verify still catch dropped input. - if keyboardVisibleBeforeTap { - return TextEntryTarget( - element: element, - refreshPoint: textEntryRefreshPoint(for: element) ?? point, - prefersFocusedElement: false - ) - } - let stabilized = stabilizeTextInputBeforeTyping( - app: app, - target: element, - keyboardVisibleBeforeTap: keyboardVisibleBeforeTap - ) - let readyTarget = TextEntryTarget( - element: stabilized.element ?? element, - refreshPoint: point, - prefersFocusedElement: false - ) - let resolved = stabilized.focusConfirmed - ? (stabilized.element ?? element) - : (waitForTextEntryReadiness(app: app, target: readyTarget) ?? stabilized.element ?? element) - return TextEntryTarget( - element: resolved, - refreshPoint: textEntryRefreshPoint(for: resolved) ?? point, - prefersFocusedElement: false - ) - } - private func textEntryRefreshPoint(for element: XCUIElement?) -> CGPoint? { guard let element else { return nil diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputCandidatePolicy.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputCandidatePolicy.swift index a46e48e80f..e8dc68912b 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputCandidatePolicy.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputCandidatePolicy.swift @@ -14,34 +14,3 @@ func isCoordinateTextInputCandidate( && point.y >= frame.minY - tolerance && point.y <= frame.maxY + tolerance } - -extension RunnerTests { -#if AGENT_DEVICE_RUNNER_UNIT_TESTS - func testCoordinateTextInputCandidateMustBeEnabledAndContainTheTouchPoint() { - let frame = CGRect(x: 10, y: 20, width: 100, height: 40) - let point = CGPoint(x: 50, y: 40) - - XCTAssertTrue( - isCoordinateTextInputCandidate( - enabled: true, - frame: frame, - point: point - ) - ) - XCTAssertFalse( - isCoordinateTextInputCandidate( - enabled: false, - frame: frame, - point: point - ) - ) - XCTAssertFalse( - isCoordinateTextInputCandidate( - enabled: true, - frame: frame, - point: CGPoint(x: 200, y: 200) - ) - ) - } -#endif -} diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift index 422da3e710..241d312dcb 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift @@ -77,6 +77,7 @@ extension RunnerTests { } let shouldUseSynthesizedFirstResponderType = Self.shouldUseSynthesizedFirstResponderType( repairMode: repairMode, + text: text, fromTapWitness: activeTarget.fromTapWitness, softwareKeyboardVisible: isKeyboardVisible(app: app) ) @@ -142,26 +143,9 @@ extension RunnerTests { } textEntryRoute = "synthesized-first-responder" NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_ROUTE route=synthesized-first-responder") - let textBefore = editableTextValue(for: currentTarget, treatingPlaceholderAsEmpty: true) switch synthesizer.enterText(app: app, text: value, replacingExistingText: false) { case .continueTyping: - // No refresh point: like the tap-witness target itself, the commit wait must observe - // only the element the tap selected, never rediscover a different field. - let commit = awaitSynthesizedFirstResponderCommit( - app: app, - target: TextEntryTarget( - element: currentTarget, - refreshPoint: nil, - prefersFocusedElement: false, - fromTapWitness: true - ), - textBefore: textBefore, - typedText: value - ) - // The characters were posted, so the element stays on the tuple for the caller's - // bookkeeping; the failure is what makes the command refuse. Reporting ok here is the - // defect this route had — the deadline was indistinguishable from a commit. - return (currentTarget, Self.textEntryFailure(forCommitOutcome: commit)) + return (currentTarget, nil) case .fallback: return (nil, .synthesisUnavailable) case .raise(let message): diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Transport.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Transport.swift index 5545ab6b9c..9db9f0f7e7 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Transport.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Transport.swift @@ -108,12 +108,8 @@ extension RunnerTests { do { let command = try JSONDecoder().decode(Command.self, from: body) - if command.command == .status { - completion((jsonResponse(status: 200, response: executeStatus(command: command)), false)) - return - } - if command.command == .uptime { - completion((jsonResponse(status: 200, response: executeUptime()), false)) + if let response = inlineResponse(for: command) { + completion((jsonResponse(status: 200, response: response), false)) return } // Re-sends of a still-executing commandId (the daemon's transport retry loop) attach to @@ -127,10 +123,9 @@ extension RunnerTests { command.command.rawValue, command.commandId ?? "" ) - commandJournal.accept(command: command) - commandExecutionQueue.async { - do { - let response = try self.executeAccepted(command: command) + enqueueAccepted(command: command) { result in + switch result { + case .success(let response): NSLog( "AGENT_DEVICE_RUNNER_COMMAND_COMPLETED command=%@ commandId=%@ ok=%d", command.command.rawValue, @@ -142,7 +137,7 @@ extension RunnerTests { result: (self.jsonResponse(status: 200, response: response), command.command == .shutdown), completion: completion ) - } catch { + case .failure(let error): NSLog( "AGENT_DEVICE_RUNNER_COMMAND_FAILED command=%@ commandId=%@ error=%@", command.command.rawValue, @@ -174,12 +169,38 @@ extension RunnerTests { } } + // MARK: - Command Routing + + /// Status and uptime read runner state without entering the journal or the command queue. + func inlineResponse(for command: Command) -> Response? { + switch command.command { + case .status: + return executeStatus(command: command) + case .uptime: + return executeUptime() + default: + return nil + } + } + + /// Journal-accepts `command` and executes it on `commandExecutionQueue`; `completion` runs on that + /// queue. + func enqueueAccepted( + command: Command, + completion: @escaping (Result) -> Void + ) { + commandJournal.accept(command: command) + commandExecutionQueue.async { + completion(Result { try self.executeAccepted(command: command) }) + } + } + // MARK: - In-Flight Command Coalescing /// Returns true when this send duplicated a still-executing commandId and was attached as a /// waiter of the in-flight execution. Otherwise marks the commandId in flight and returns /// false so the caller enqueues the (single) execution. - private func attachToInFlightCommandIfNeeded( + func attachToInFlightCommandIfNeeded( command: Command, completion: @escaping ((data: Data, shouldFinish: Bool)) -> Void ) -> Bool { @@ -200,7 +221,7 @@ extension RunnerTests { return false } - private func deliverCommandResult( + func deliverCommandResult( command: Command, result: (data: Data, shouldFinish: Bool), completion: ((data: Data, shouldFinish: Bool)) -> Void @@ -218,45 +239,6 @@ extension RunnerTests { } } -#if AGENT_DEVICE_RUNNER_UNIT_TESTS - func testDuplicateCommandIdCoalescesOntoInFlightExecution() throws { - let command = try JSONDecoder().decode( - Command.self, - from: Data(#"{"command":"snapshot","commandId":"snapshot-coalesce"}"#.utf8) - ) - var primaryData: Data? - var waiterData: Data? - defer { - inFlightCommandIds.removeAll() - inFlightCommandWaiters.removeAll() - } - - XCTAssertFalse( - attachToInFlightCommandIfNeeded(command: command) { result in - primaryData = result.data - } - ) - XCTAssertTrue( - attachToInFlightCommandIfNeeded(command: command) { result in - waiterData = result.data - } - ) - - let delivered = Data("single-result".utf8) - deliverCommandResult( - command: command, - result: (delivered, false) - ) { result in - primaryData = result.data - } - - XCTAssertEqual(primaryData, delivered) - XCTAssertEqual(waiterData, delivered) - XCTAssertFalse(inFlightCommandIds.contains("snapshot-coalesce")) - XCTAssertNil(inFlightCommandWaiters["snapshot-coalesce"]) - } -#endif - // MARK: - Response Encoding private func jsonResponse(status: Int, response: Response) -> Data { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TvRemote.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TvRemote.swift index 252352baf0..b312933bb8 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TvRemote.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TvRemote.swift @@ -18,28 +18,6 @@ enum TvRemoteButton: String { } extension RunnerTests { -#if AGENT_DEVICE_RUNNER_UNIT_TESTS - func testTvRemoteButtonMappingAcceptsSupportedNamesAndRejectsUnknown() { - let supported = [ - ("select", "select"), - ("SELECT", "select"), - ("menu", "menu"), - ("home", "home"), - ("up", "up"), - ("down", "down"), - ("left", "left"), - ("right", "right"), - ] - for (raw, expected) in supported { - XCTAssertEqual(tvRemoteButton(from: raw)?.rawValue, expected) - } - - for raw in [String?(nil), "", "volumeUp", "select "] { - XCTAssertNil(tvRemoteButton(from: raw)) - } - } -#endif - func resolveTvRemoteDoublePressDelay() -> TimeInterval { guard let raw = ProcessInfo.processInfo.environment["AGENT_DEVICE_TV_REMOTE_DOUBLE_PRESS_DELAY_MS"], diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift new file mode 100644 index 0000000000..5ec6f27826 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift @@ -0,0 +1,135 @@ +import XCTest +import AgentDeviceSnapshotPresentation + +extension RunnerTests { + func executeTypeCommand(activeApp: XCUIApplication, command: Command) -> Response { + guard let text = command.text else { + return Response(ok: false, error: ErrorPayload(message: "type requires text")) + } + let delaySeconds = Double(max(command.delayMs ?? 0, 0)) / 1000.0 + let textEntryMode = resolveTextEntryMode(command) + let target: TextEntryTarget + var resolvedCoordinateContext: SynthesizedCoordinateContext? + // The shared runtime has already resolved this node as non-hittable and + // deliberately selected Maestro's coordinate compatibility route. + let maestroNonHittableCoordinateFallbackUsed: Bool? = + command.allowNonHittableCoordinateFallback == true && command.x != nil && command.y != nil + ? true + : nil + let focusStartedAt = Date() +#if os(iOS) + let xCTestChannelPenalized = isSnapshotXCTestChannelPenalized(bundleId: currentBundleId) + var resolvedCoordinateTarget: TextEntryTarget? + if Self.shouldUseResolvedCoordinateTextEntryRoute( + repairMode: textEntryMode, + hasX: command.x != nil, + hasY: command.y != nil, + xCTestChannelPenalized: xCTestChannelPenalized + ), let x = command.x, let y = command.y { + let policyKind = SynthesizedGesturePolicyKind.coordinateTap + let context = synthesizedCoordinateContext( + app: activeApp, + policy: synthesizedGesturePolicy(policyKind) + ) + switch performSynthesizedGesture(activeApp, kind: policyKind, context: context, synthesize: { + synthesizedTapAt(app: activeApp, x: x, y: y, context: context) + }) { + case .performed: + resolvedCoordinateContext = context + resolvedCoordinateTarget = TextEntryTarget( + element: nil, + refreshPoint: CGPoint(x: x, y: y), + prefersFocusedElement: false + ) + case .xctestFallback: + break + case .refused(_, let message, let hint): + return unsupportedResponse(message: message, hint: hint) + } + } +#else + let xCTestChannelPenalized = false + let resolvedCoordinateTarget: TextEntryTarget? = nil +#endif + if let resolvedCoordinateTarget { + target = resolvedCoordinateTarget + } else { + target = focusTextInputForTextEntry(app: activeApp, x: command.x, y: command.y) + } + NSLog( + "AGENT_DEVICE_RUNNER_TEXT_ENTRY_PHASE commandId=%@ phase=focus durationMs=%.1f chars=%d mode=%@", + command.commandId ?? "", + Date().timeIntervalSince(focusStartedAt) * 1000.0, + text.count, + textEntryModeName(textEntryMode) + ) + if textEntryMode == .replacement { +#if os(iOS) + let canReplaceResolvedFirstResponder = Self.shouldUseSynthesizedFirstResponderReplacement( + hasResolvedElement: target.element != nil, + hasRefreshPoint: target.refreshPoint != nil, + xCTestChannelPenalized: xCTestChannelPenalized + ) +#else + let canReplaceResolvedFirstResponder = false +#endif + guard target.element != nil || canReplaceResolvedFirstResponder else { + let message = + (command.x != nil && command.y != nil) + ? "no text input found at the provided coordinates to clear" + : "no focused text input to clear" + return Response(ok: false, error: ErrorPayload(message: message)) + } + } + let textResult = typeTextReliably( + app: activeApp, + target: target, + text: text, + delaySeconds: delaySeconds, + repairMode: textEntryMode, + xCTestChannelPenalized: xCTestChannelPenalized, + synthesizer: PrivateXCTestTextEntrySynthesizer(), + commandId: command.commandId + ) + if let failure = textResult.failure { + return Response( + ok: false, + error: ErrorPayload(code: failure.rawValue, message: failure.message, hint: failure.hint) + ) + } + if textResult.verified == false { + let expected = textResult.expectedText ?? "" + let observed = textResult.observedText ?? "" + return Response( + ok: false, + error: ErrorPayload( + code: "TEXT_ENTRY_MISMATCH", + message: "text entry verification failed: expected \"\(expected)\", observed \"\(observed)\"" + ) + ) + } + let point = target.refreshPoint + let frame: CGRect + if let resolvedCoordinateContext { + frame = resolvedCoordinateContext.referenceFrame + } else if point != nil { + frame = activeApp.frame + } else { + // Bare `type` has no coordinate response to normalize. Avoid serializing the + // application AX tree only to emit unused reference dimensions. + frame = .zero + } + return Response( + ok: true, + data: DataPayload( + message: textResult.repaired ? "typed after repair" : "typed", + x: point.map { Double($0.x) }, + y: point.map { Double($0.y) }, + referenceWidth: frame.isEmpty ? nil : Double(frame.width), + referenceHeight: frame.isEmpty ? nil : Double(frame.height), + maestroNonHittableCoordinateFallbackUsed: maestroNonHittableCoordinateFallbackUsed, + textEntryRoute: textResult.textEntryRoute + ) + ) + } +} diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift index 594f4e71a4..725261ff2a 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift @@ -31,8 +31,10 @@ final class RunnerTests: XCTestCase { /// String codes the daemon keys behavior on. `RUNNER_BUSY` and `RUNNER_WEDGED` come from the busy /// gate; `MAIN_THREAD_TIMEOUT` is emitted by the transport when a command trips the execution /// watchdog, so the daemon can tell "the main thread is now occupied" from a generic failure. + /// `APP_NOT_RUNNING` refuses a read whose session app is not running. enum RunnerWireErrorCode { static let mainThreadTimeout = "MAIN_THREAD_TIMEOUT" + static let appNotRunning = "APP_NOT_RUNNING" } static let springboardBundleId = "com.apple.springboard" @@ -52,6 +54,8 @@ final class RunnerTests: XCTestCase { let commandExecutionQueue = DispatchQueue(label: "agent-device.runner.commands") let app = XCUIApplication() lazy var springboard = XCUIApplication(bundleIdentifier: Self.springboardBundleId) + // Main-thread owned, like `runnerAccessibilityHealth`: an off-main capture plan reads them only + // through its `SnapshotCaptureTarget` and writes them only through `applyMainOwnedSnapshotState`. var currentApp: XCUIApplication? var currentBundleId: String? var currentAppProcessIdentifier: Int? @@ -75,7 +79,18 @@ final class RunnerTests: XCTestCase { let xctestIdleKeepaliveInterval: TimeInterval = 60.0 let minRecordingFps = 1 let maxRecordingFps = 120 + // A recorder frame still capturing on main after this long is abandoned and dropped. It bounds a + // screenshot round trip, not the frame interval: a capture slower than the interval lowers the + // frame rate, and only a capture this slow counts as main-thread occupancy. + let recordingFrameCaptureTimeout: TimeInterval = 1 var needsPostSnapshotInteractionDelay = false + // Per-command markers that restate a fact of the bound target (the fast app guard, the + // synthesized gesture policy per gesture kind) write only when that fact changes; otherwise a + // long session fills runner.log with one identical line per command. Cleared with the rest of the + // target-bound state so a rebind states the fact once more. + var lastLoggedFastAppGuardLine: String? + var lastLoggedGesturePolicyLines: [SynthesizedGesturePolicyKind: String] = [:] + var runnerMarkerWriter: (String) -> Void = { NSLog("%@", $0) } /// When the first interaction after an activation may run, on the monotonic uptime clock. /// The guarantee is a minimum gap *since the activation*, not a pause at the interaction: /// a caller that already spent that gap elsewhere (an agent's round trip is 190-260 ms) @@ -95,6 +110,9 @@ final class RunnerTests: XCTestCase { // and post-capture bookkeeping stays off main (#1105/#1244). let mainThreadWorkLock = NSLock() var abandonedMainThreadWorkCount = 0 + // Dispatched main-queue work that has not finished yet, abandoned or not. Only optional work + // reads it, to stay off a main thread that commands are using; occupancy readers do not. + var mainThreadWorkInFlightCount = 0 var abandonedMainThreadWorkSince: Date? // Past this age the runner stops claiming "busy, retry soon" and reports itself wedged so // the daemon recycles it — the only cure once the main thread is stuck for good. @@ -107,7 +125,7 @@ final class RunnerTests: XCTestCase { var snapshotXCTestChannelPenaltyBundleId: String? var snapshotXCTestChannelPenaltyUntil = Date.distantPast let snapshotXCTestChannelPenaltyDuration: TimeInterval = 120 - var snapshotXCTestPenaltyWarmupExemptionPending = false + let snapshotXCTestPenaltyWarmupExemption = SnapshotXCTestPenaltyWarmupExemption() // Sticky per-bundle hint for the private AX depth ladder: deep RN screens reject the default // depth with kAXErrorIllegalArgument on EVERY capture, so once a shallower rung is accepted // later captures start there instead of re-paying the rejected deep request (~300ms per @@ -190,6 +208,9 @@ final class RunnerTests: XCTestCase { var blockingSystemModalPresenceOverrideForTesting: Bool? var alertResolutionOverrideForTesting: ((Date) -> RunnerAlert?)? var alertButtonHittabilityProbeOverrideForTesting: ((Date) -> Bool)? + // Runs on the waiting thread after `runMainThreadWork`'s wait timed out and before it takes the + // lock that decides between finished and abandoned, so a test can finish the work in that window. + var mainThreadWorkTimedOutForTesting: (() -> Void)? #endif // Observability for the record(_:) suppression below: how many AX-broken-screen snapshot // issues this session muted, so wedge investigations see the volume without grepping logs. @@ -263,11 +284,6 @@ final class RunnerTests: XCTestCase { @MainActor func testCommand() throws { - if RunnerEnv.isTruthy("AGENT_DEVICE_RUNNER_NOOP_STARTUP") { - NSLog("AGENT_DEVICE_RUNNER_NOOP_STARTUP=1") - return - } - doneExpectation = expectation(description: "agent-device command handled") NSLog("AGENT_DEVICE_RUNNER_HEADLESS_STARTUP=1") let desiredPort = RunnerEnv.resolvePort() diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h index 8b1499f828..b346785f24 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h @@ -10,7 +10,7 @@ NS_ASSUME_NONNULL_BEGIN // // Each caller additionally resolves whichever XCSynthesizedEventRecord init // overload and XCPointerEventPath factory/mutator selectors its own event kind -// needs: gesture uses the 2-arg `initWithName:interfaceOrientation:` plus +// needs: gesture uses `initWithName:displayID:interfaceOrientation:` plus // touch-path selectors (`initForTouchAtPoint:offset:`, `moveToPoint:atOffset:`, // `liftUpAtOffset:`); text entry uses the 1-arg `initWithName:` plus text-input // selectors (`initForTextInput`, `typeText:atOffset:typingSpeed:shouldRedact:`, @@ -30,6 +30,16 @@ typedef void (*RunnerMsgSendSetInteger)(id, SEL, NSInteger); typedef void (*RunnerMsgSendAddPath)(id, SEL, id); typedef BOOL (*RunnerMsgSendSynthesize)(id, SEL, NSError **); +// Why the display hosting the target app could not be named. Callers branch on this rather than +// on the message, so a refusal keeps its reason across the language boundary (#2728). +typedef NS_ENUM(NSUInteger, RunnerApplicationScreenFailure) { + RunnerApplicationScreenFailureNone, + // No window with a usable frame, so nothing resolves and its screen would only report main. + RunnerApplicationScreenFailureUnresolvedWindow, + // A resolved window whose screen or display identity the XCTest runtime would not answer. + RunnerApplicationScreenFailureUnresolvedScreen, +}; + // Resolves the shared core described above against the live XCTest runtime. // Returns nil and populates `bridge` on success. Returns a non-nil // "private XCTest synthesis unavailable: ..." message and leaves @@ -43,6 +53,30 @@ FOUNDATION_EXPORT NSString * _Nullable RunnerResolveXCTestEventBridge( RunnerXCTestEventBridge *bridge ); +// Resolves the `XCUIScreen` hosting the application's window and that screen's `displayID`, which +// is the only display identity that follows a foldable app to whichever panel it currently lights. +// Reading the window frame must precede reading its screen: an unresolved element still reports the +// main screen. Never selects a screen by number or by position in `XCUIScreen.screens`. +// Returns YES and writes the screen to capture into `screen` and its identity into `displayID`. +// Otherwise returns NO, leaves both empty, and names the reason in `failure` so callers branch on +// the reason rather than on message text (#2728). Private KVC raises rather than returns nil when +// XCTest has no such value, so both reads are guarded here and surface as a typed failure instead +// of an exception reaching the caller. +FOUNDATION_EXPORT BOOL RunnerResolveApplicationScreen( + id application, + id _Nullable *screen, + NSUInteger *displayID, + RunnerApplicationScreenFailure *failure +); + +// Reads the display ID off an already-resolved window. When a caller has resolved the app window +// for geometry, routing the synthesized gesture by this window keeps the record's display and the +// booked reference frame on the same window instead of re-walking `windows.firstMatch`. +FOUNDATION_EXPORT NSString * _Nullable RunnerResolveWindowDisplayID( + id window, + NSUInteger *displayID +); + FOUNDATION_EXPORT NSString * _Nullable RunnerRequireClass( Class cls, NSString *className, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m index 1b5829ce2a..169b79580e 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m @@ -1,4 +1,7 @@ #import "RunnerXCTestEventBridge.h" +#import "RunnerObjCExceptionCatcher.h" +#import +#import NSString * _Nullable RunnerResolveXCTestEventBridge( id application, @@ -87,3 +90,160 @@ NSString *reason = exception.reason ?: fallbackReason; return [NSString stringWithFormat:@"%@: %@", name, reason]; } + +static BOOL RunnerUsableWindowFrame(CGRect frame) { + if (CGRectIsEmpty(frame) || CGRectIsInfinite(frame) || CGRectIsNull(frame)) { + return NO; + } + return isfinite(frame.origin.x) && isfinite(frame.origin.y) + && isfinite(frame.size.width) && isfinite(frame.size.height); +} + +static NSString *RunnerApplicationScreenFailureDescription( + RunnerApplicationScreenFailure failure +) { + switch (failure) { + case RunnerApplicationScreenFailureUnresolvedWindow: + return @"no resolved application window"; + case RunnerApplicationScreenFailureUnresolvedScreen: + return @"no resolved window display ID"; + case RunnerApplicationScreenFailureNone: + return @"application screen resolved"; + } +} + +// One window read. XCTest answers with a value or raises. The window object is handed back because +// resolving this instance is what its `screen` will answer from — re-querying `firstMatch` would +// return an unresolved element that still names the main screen. +// +// Both answers are required. An app with no window answers `frame` with `(0,0 0x0)` without raising, +// which geometry alone would read as a window the runtime has not measured, and only the query +// itself says whether it has one. Measured on a live Duo: the runner host process, which has no +// window, answers `exists` NO and `frame` `(0,0 0x0)` (#2728). +// +// The frame the read produced is reported even when it is refused: a capture that fails closed has +// to say what it looked at, or the reason code cannot be told apart from a stale device. +static RunnerApplicationScreenFailure RunnerResolveUsableWindow( + id application, + id _Nullable *window, + CGRect *observedFrame +) { + __block id readWindow = nil; + __block CGRect frame = CGRectNull; + __block BOOL resolved = NO; + NSString *exception = [RunnerObjCExceptionCatcher catchException:^{ + id candidate = [[application valueForKey:@"windows"] valueForKey:@"firstMatch"]; + NSNumber *present = [candidate valueForKey:@"exists"]; + resolved = [present isKindOfClass:NSNumber.class] && present.boolValue; + NSValue *frameValue = [candidate valueForKey:@"frame"]; + [frameValue getValue:&frame size:sizeof(frame)]; + readWindow = candidate; + }]; + *observedFrame = frame; + if (exception != nil || !resolved || !RunnerUsableWindowFrame(frame)) { + NSLog( + @"AGENT_DEVICE_RUNNER_APP_SCREEN_UNRESOLVED stage=window frame=(%g,%g %gx%g) raised=%@ resolved=%@", + frame.origin.x, + frame.origin.y, + frame.size.width, + frame.size.height, + exception != nil ? @"yes" : @"no", + resolved ? @"yes" : @"no" + ); + return RunnerApplicationScreenFailureUnresolvedWindow; + } + *window = readWindow; + return RunnerApplicationScreenFailureNone; +} + +// The screen read that must follow window resolution on that same window instance. Before the +// frame above is asked for, the window's `screen` can still name main, so this is never merged +// into the read ahead of it. +static RunnerApplicationScreenFailure RunnerReadResolvedWindowScreen( + id window, + CGRect windowFrame, + id _Nullable *screen, + NSUInteger *displayID +) { + __block id readScreen = nil; + __block NSUInteger readDisplayID = 0; + NSString *exception = [RunnerObjCExceptionCatcher catchException:^{ + id windowScreen = [window valueForKey:@"screen"]; + NSNumber *identifier = [windowScreen valueForKey:@"displayID"]; + if (![identifier isKindOfClass:NSNumber.class] || identifier.unsignedIntegerValue == 0) { + return; + } + readScreen = windowScreen; + readDisplayID = identifier.unsignedIntegerValue; + }]; + if (exception != nil || readScreen == nil) { + NSLog( + @"AGENT_DEVICE_RUNNER_APP_SCREEN_UNRESOLVED stage=screen frame=(%g,%g %gx%g) raised=%@", + windowFrame.origin.x, + windowFrame.origin.y, + windowFrame.size.width, + windowFrame.size.height, + exception != nil ? @"yes" : @"no" + ); + return RunnerApplicationScreenFailureUnresolvedScreen; + } + *screen = readScreen; + *displayID = readDisplayID; + return RunnerApplicationScreenFailureNone; +} + +BOOL RunnerResolveApplicationScreen( + id application, + id _Nullable *screen, + NSUInteger *displayID, + RunnerApplicationScreenFailure *failure +) { + *screen = nil; + *displayID = 0; + id window = nil; + CGRect windowFrame = CGRectNull; + *failure = RunnerResolveUsableWindow(application, &window, &windowFrame); + if (*failure != RunnerApplicationScreenFailureNone) { + return NO; + } + *failure = RunnerReadResolvedWindowScreen(window, windowFrame, screen, displayID); + return *failure == RunnerApplicationScreenFailureNone; +} + +// Reads the display ID off a window the caller already resolved, so a synthesized record can be +// routed by the same window its reference frame was measured on. The window still resolves here: +// its frame is read before its screen, and a window that answers empty or raises fails closed with +// the same reasons the application walk gives. +NSString * _Nullable RunnerResolveWindowDisplayID(id window, NSUInteger *displayID) { + __block CGRect frame = CGRectNull; + NSString *frameException = [RunnerObjCExceptionCatcher catchException:^{ + NSValue *frameValue = [window valueForKey:@"frame"]; + [frameValue getValue:&frame size:sizeof(frame)]; + }]; + if (frameException != nil || !RunnerUsableWindowFrame(frame)) { + NSLog( + @"AGENT_DEVICE_RUNNER_APP_SCREEN_UNRESOLVED stage=window frame=(%g,%g %gx%g) raised=%@", + frame.origin.x, + frame.origin.y, + frame.size.width, + frame.size.height, + frameException != nil ? @"yes" : @"no" + ); + return [NSString stringWithFormat: + @"private XCTest event synthesis unavailable: %@", + RunnerApplicationScreenFailureDescription(RunnerApplicationScreenFailureUnresolvedWindow) + ]; + } + id screen = nil; + NSUInteger resolvedDisplayID = 0; + RunnerApplicationScreenFailure failure = + RunnerReadResolvedWindowScreen(window, frame, &screen, &resolvedDisplayID); + if (failure != RunnerApplicationScreenFailureNone) { + return [NSString stringWithFormat: + @"private XCTest event synthesis unavailable: %@", + RunnerApplicationScreenFailureDescription(failure) + ]; + } + *displayID = resolvedDisplayID; + return nil; +} diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AXRecoveryConformanceTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AXRecoveryConformanceTests.swift index 6e6364c310..cde158367b 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AXRecoveryConformanceTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AXRecoveryConformanceTests.swift @@ -426,8 +426,6 @@ extension RunnerTests { XCTAssertFalse(fixture.hintCases.isEmpty) defer { clearPrivateAXAcceptedDepth(reason: "test-cleanup") - currentBundleId = nil - currentAppProcessIdentifier = nil } for hintCase in fixture.hintCases { clearPrivateAXAcceptedDepth(reason: "fixture-case") @@ -440,8 +438,6 @@ extension RunnerTests { let target = try XCTUnwrap(step.target, name) let processIdentifier = try XCTUnwrap( Int(target.generation.filter(\.isNumber)), "\(name): generation must end in digits") - currentBundleId = target.id - currentAppProcessIdentifier = processIdentifier let explicit = step.explicitDepth ?? false let remembered = explicit @@ -459,6 +455,8 @@ extension RunnerTests { XCTAssertEqual(capture.truncated, outcome.complete == false, "\(name): bounded") // The production path records only after a successful ladder, exactly like this. recordPrivateAXAcceptedDepth( + bundleId: target.id, + processIdentifier: processIdentifier, exactDepthRequested: explicit, effectiveDepth: capture.effectiveDepth, attemptDepths: capture.attemptDepths) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AXSnapshotFallbackTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AXSnapshotFallbackTests.swift new file mode 100644 index 0000000000..4e4fd9ff44 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AXSnapshotFallbackTests.swift @@ -0,0 +1,529 @@ +import XCTest +import AgentDeviceSnapshotPresentation + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +// MARK: - In-bundle unit tests + +extension RunnerTests { + func testPrivateAXAttemptDepthsAppliesRememberedDepth() { + XCTAssertEqual( + Self.privateAXAttemptDepths(requestedDepth: 64, rememberedDepth: nil), + [64, 56, 40, 24, 12] + ) + XCTAssertEqual( + Self.privateAXAttemptDepths(requestedDepth: 64, rememberedDepth: 56), + [56, 40, 24, 12] + ) + XCTAssertEqual(Self.privateAXAttemptDepths(requestedDepth: 64, rememberedDepth: 12), [12]) + // Remembered at/above the requested depth changes nothing. + XCTAssertEqual( + Self.privateAXAttemptDepths(requestedDepth: 64, rememberedDepth: 64), + [64, 56, 40, 24, 12] + ) + // A shallower explicit request keeps its own rungs; deeper stale memory is ignored. + XCTAssertEqual(Self.privateAXAttemptDepths(requestedDepth: 24, rememberedDepth: 56), [24, 12]) + } + + /// Executed producer contract for the #1627 review blocker: a frontier whose + /// live element vanished, and one whose re-rooted request fails, must BOTH + /// count as missed — an all-miss extension reporting itself drained would + /// present a capped capture as complete. Goes red if either miss-path + /// increment in extendSnapshotFrontiers is removed. + func testDeepExtensionCountsMissedFrontiers() { + // Element vanished (list churn between serialization and extension): the + // fabricated snapshot answers nil for accessibilityElement — missed, and + // no request call is consumed. (An explicit nil property: bare NSObject + // resolves the key through a UIKit category and would take the call path.) + let orphan = RunnerAXSnapshotFrontier() + orphan.snapshot = FrontierSnapshotWithoutElementForTesting() + orphan.node = NSMutableDictionary() + // Re-rooted request fails: the element resolves but the client cannot + // serve requestSnapshotForElement — one consumed call AND a miss. + let unreachable = RunnerAXSnapshotFrontier() + unreachable.snapshot = FrontierSnapshotWithElementForTesting() + unreachable.node = NSMutableDictionary() + + var nodeCount = 0 + var truncated = ObjCBool(false) + let outcome = RunnerAXSnapshotBridge.extend( + NSMutableArray(array: [orphan, unreachable]), + axClient: NSObject(), + attributes: [], + maxDepth: 56, + maxNodes: 5_000, + nodeCount: &nodeCount, + truncated: &truncated, + callsAllowed: 8, + mergedLeaves: nil, + deadline: nil + ) + + XCTAssertEqual(outcome?[RunnerAXSnapshotDeepExtensionMissedKey] as? Int, 2) + XCTAssertEqual(outcome?[RunnerAXSnapshotDeepExtensionCallsKey] as? Int, 1) + XCTAssertEqual(outcome?[RunnerAXSnapshotDeepExtensionPendingKey] as? Int, 0) + XCTAssertEqual(outcome?[RunnerAXSnapshotDeepExtensionNodesAddedKey] as? Int, 0) + XCTAssertFalse(truncated.boolValue) + // And the consumer verdict over exactly this outcome: still depth-limited. + XCTAssertTrue( + Self.privateAXDepthLimited( + effectiveDepth: 56, requestedDepth: 64, pendingFrontiers: 0, missedFrontiers: 2)) + } + + func testPrivateAXDepthLimitedRequiresEveryFrontierResolved() { + // Un-capped capture is never depth-limited, extension or not. + XCTAssertFalse( + Self.privateAXDepthLimited( + effectiveDepth: 64, requestedDepth: 64, pendingFrontiers: nil, missedFrontiers: nil)) + // Capped with no extension outcome (never ran) stays depth-limited. + XCTAssertTrue( + Self.privateAXDepthLimited( + effectiveDepth: 56, requestedDepth: 64, pendingFrontiers: nil, missedFrontiers: nil)) + // Fully drained extension clears the verdict. + XCTAssertFalse( + Self.privateAXDepthLimited( + effectiveDepth: 56, requestedDepth: 64, pendingFrontiers: 0, missedFrontiers: 0)) + // Budget exhaustion (pending frontiers) keeps it. + XCTAssertTrue( + Self.privateAXDepthLimited( + effectiveDepth: 56, requestedDepth: 64, pendingFrontiers: 2, missedFrontiers: 0)) + // The #1627 review blocker: an all-miss extension (elements vanished or + // re-rooted requests failed) resolved nothing — it must NOT present the + // capture as complete just because the queue emptied. + XCTAssertTrue( + Self.privateAXDepthLimited( + effectiveDepth: 56, requestedDepth: 64, pendingFrontiers: 0, missedFrontiers: 8)) + } + + func testPrivateAXAcceptedDepthMemoryMatchesBundleProcessAndExpires() { + defer { clearPrivateAXAcceptedDepth(reason: "test-cleanup") } + + rememberPrivateAXAcceptedDepth(bundleId: "xyz.blueskyweb.app", processIdentifier: 111, depth: 56) + XCTAssertEqual( + rememberedPrivateAXAcceptedDepth(bundleId: "xyz.blueskyweb.app", processIdentifier: 111), + 56 + ) + XCTAssertNil(rememberedPrivateAXAcceptedDepth(bundleId: "com.other.app", processIdentifier: 111)) + // A relaunch changes the PID; the new process must re-probe the full depth even inside the + // TTL, and an unknown current PID (post-invalidation) must never match. + XCTAssertNil(rememberedPrivateAXAcceptedDepth(bundleId: "xyz.blueskyweb.app", processIdentifier: 222)) + XCTAssertNil(rememberedPrivateAXAcceptedDepth(bundleId: "xyz.blueskyweb.app", processIdentifier: nil)) + + // Expired memory stops applying (the expiry re-probes the full requested depth). + privateAXAcceptedDepthUntil = Date(timeIntervalSinceNow: -1) + XCTAssertNil(rememberedPrivateAXAcceptedDepth(bundleId: "xyz.blueskyweb.app", processIdentifier: 111)) + } + + func testPrivateAXAcceptedDepthMemoryRequiresProcessIdentifierToRecord() { + defer { clearPrivateAXAcceptedDepth(reason: "test-cleanup") } + + rememberPrivateAXAcceptedDepth(bundleId: "xyz.blueskyweb.app", processIdentifier: nil, depth: 56) + XCTAssertNil( + rememberedPrivateAXAcceptedDepth(bundleId: "xyz.blueskyweb.app", processIdentifier: nil) + ) + } + + func testViewportReadSkippedWhileXCTestChannelPenalized() { + // Pins the viewport fast path (#1587 review): every penalized private AX capture used to burn + // the full 1s main-thread timeout on a doomed viewport read before falling back. + let bundleId = "xyz.blueskyweb.app" + defer { + clearSnapshotXCTestChannelPenalty(reason: "test-cleanup") + abandonedMainThreadWorkCount = 0 + } + + XCTAssertTrue(shouldReadPrivateAXViewportViaXCTest(bundleId: bundleId)) + + penalizeSnapshotXCTestChannel(bundleId: bundleId, reason: "test") + XCTAssertFalse(shouldReadPrivateAXViewportViaXCTest(bundleId: bundleId)) + + clearSnapshotXCTestChannelPenalty(reason: "test") + XCTAssertTrue(shouldReadPrivateAXViewportViaXCTest(bundleId: bundleId)) + + abandonedMainThreadWorkCount = 1 + XCTAssertFalse(shouldReadPrivateAXViewportViaXCTest(bundleId: bundleId)) + } + + /// The wire field must reach both capture options AND the backend pin: custom + /// actions are only readable through the private AX client, so a capture that + /// asked for them but planned the XCTest tree backend would return a payload + /// that structurally cannot carry them. + func testCustomActionsRequestPinsPrivateAXBackend() throws { + let asked = try JSONDecoder().decode( + Command.self, from: Data(#"{"command":"snapshot","customActions":true}"#.utf8)) + let options = Self.presentationOptions(from: asked) + XCTAssertTrue(options.customActions) + XCTAssertEqual(options.preferredBackend, SnapshotBackendKind.privateAX.rawValue) + XCTAssertTrue( + Self.snapshotXCTestChannelTreatedAsPenalized( + penalized: false, preferredBackend: options.preferredBackend)) + + // An explicit pin is never overwritten by the implied one. + let pinned = try JSONDecoder().decode( + Command.self, + from: Data(#"{"command":"snapshot","customActions":true,"preferredBackend":"tree"}"#.utf8)) + XCTAssertEqual(Self.presentationOptions(from: pinned).preferredBackend, "tree") + + // And the default capture neither asks nor pins. + let bare = try JSONDecoder().decode(Command.self, from: Data(#"{"command":"snapshot"}"#.utf8)) + XCTAssertFalse(Self.presentationOptions(from: bare).customActions) + XCTAssertNil(Self.presentationOptions(from: bare).preferredBackend) + } + + /// A request-pinned backend degraded nothing, so its verdict must not claim + /// slow accessibility work — that reason drives a user-facing warning. + func testRequestPinnedBackendReportsItsOwnReason() { + let requested = Self.xcTestChannelStateFirstFailure( + .deferredToIndependentBackend, requestPinnedBackend: true) + XCTAssertEqual(requested?.code, "requested-backend") + XCTAssertFalse(requested?.reason.contains("slow accessibility work") ?? true) + + // The circuit breaker's own deferral keeps its established code and wording. + let breaker = Self.xcTestChannelStateFirstFailure(.deferredToIndependentBackend) + XCTAssertEqual(breaker?.code, "deferred") + + // The bounded probe and the healthy plan are untouched by the new flag. + XCTAssertEqual( + Self.xcTestChannelStateFirstFailure(.boundedXCTestProbe, requestPinnedBackend: true)?.code, + "budget") + XCTAssertNil(Self.xcTestChannelStateFirstFailure(.normal, requestPinnedBackend: true)) + } + + /// The disclosure only exists if the counts survive the bridge boundary, and + /// "did not ask" must stay distinguishable from "read none". + func testCustomActionCoverageParsesOnlyCompletePairs() { + let complete: [String: Any] = [ + RunnerAXSnapshotCustomActionsReadKey: 12, + RunnerAXSnapshotCustomActionsCandidatesKey: 19, + RunnerAXSnapshotCustomActionsTruncatedKey: 2, + RunnerAXSnapshotCustomActionsBlockedKey: true, + ] + let coverage = Self.privateAXCustomActionCoverage(complete) + XCTAssertEqual(coverage?.read, 12) + XCTAssertEqual(coverage?.candidates, 19) + XCTAssertEqual(coverage?.truncated, 2) + XCTAssertEqual(coverage?.blocked, true) + + // Absent key = the capture never asked; it must not read as (0, 0), which + // would warn "0 of 0" on every default capture. + XCTAssertNil(Self.privateAXCustomActionCoverage(nil)) + // The bridge in this target always writes all four keys, so a partial + // dictionary is malformed and is dropped whole. + for key in complete.keys { + var partial = complete + partial.removeValue(forKey: key) + XCTAssertNil(Self.privateAXCustomActionCoverage(partial), "missing \(key)") + } + } + + /// The AX call cannot be cancelled once issued, so the read deadline frees + /// only the caller — the call keeps running. Without containment, repeating + /// `snapshot --actions` against a wedged element would stack orphaned reads, + /// all sharing one XCAXClient. This pins the containment: one serial queue and + /// a single-flight refusal that adds no work while a read is outstanding. + func testHungCustomActionReadIsContainedAndRecovers() { + let hung = HungAXClientForTesting() + let element = NSObject() + let dispatchesBefore = RunnerAXSnapshotBridge.customActionReadDispatchCount() + let blockedBefore = RunnerAXSnapshotBridge.customActionReadBlockedCount() + defer { hung.release() } + + // 1. First read wedges. The caller is freed by the deadline, but the call is + // still out there, so it stays counted in flight. + var completed = ObjCBool(true) + let firstStarted = Date() + let first = RunnerAXSnapshotBridge.customActionNames( + forElement: element, axClient: hung, completed: &completed) + XCTAssertNil(first) + XCTAssertFalse(completed.boolValue) + XCTAssertGreaterThanOrEqual(-firstStarted.timeIntervalSinceNow, 0.9) + XCTAssertEqual(RunnerAXSnapshotBridge.customActionReadsInFlight(), 1) + XCTAssertEqual( + RunnerAXSnapshotBridge.customActionReadDispatchCount(), dispatchesBefore + 1) + + // 2. Repeats do NOT accumulate: no new dispatch, still exactly one in + // flight, and every repeat is refused by single-flight admission. + for _ in 0..<5 { + XCTAssertNil( + RunnerAXSnapshotBridge.customActionNames( + forElement: element, axClient: hung, completed: &completed)) + XCTAssertFalse(completed.boolValue) + } + XCTAssertEqual(RunnerAXSnapshotBridge.customActionReadsInFlight(), 1) + XCTAssertEqual( + RunnerAXSnapshotBridge.customActionReadDispatchCount(), dispatchesBefore + 1) + XCTAssertEqual(RunnerAXSnapshotBridge.customActionReadBlockedCount(), blockedBefore + 5) + + // 3. A capture in that state discloses the skip rather than presenting the + // unread elements as action-free — and spends no read budget doing it. + let leaf = RunnerAXSnapshotFrontier() + leaf.snapshot = FrontierSnapshotWithElementForTesting() + leaf.node = NSMutableDictionary(dictionary: ["label": "feedItem", "children": []]) + let coverage = RunnerAXSnapshotBridge.annotateCustomActions( + onMergedLeaves: [leaf], axClient: hung, limit: 12, rootFrame: .zero, deadline: nil) + XCTAssertEqual(coverage[RunnerAXSnapshotCustomActionsBlockedKey] as? Bool, true) + XCTAssertEqual(coverage[RunnerAXSnapshotCustomActionsReadKey] as? Int, 0) + XCTAssertEqual(coverage[RunnerAXSnapshotCustomActionsCandidatesKey] as? Int, 1) + XCTAssertEqual( + RunnerAXSnapshotBridge.customActionReadDispatchCount(), dispatchesBefore + 1) + XCTAssertEqual( + Self.privateAXCustomActionCoverage(coverage), + SnapshotCustomActionCoverage(read: 0, candidates: 1, truncated: 0, blocked: true)) + + // 4. Recovery: once the wedged call returns, reads resume by themselves. + hung.release() + let recovered = expectation(description: "in-flight drains") + DispatchQueue.global().async { + while RunnerAXSnapshotBridge.customActionReadsInFlight() > 0 { + usleep(20_000) + } + recovered.fulfill() + } + wait(for: [recovered], timeout: 5) + + completed = ObjCBool(false) + XCTAssertNil( + RunnerAXSnapshotBridge.customActionNames( + forElement: element, axClient: hung, completed: &completed)) + // Completed (the fake answers nil actions), which is the point: the pass is + // live again rather than latched off. + XCTAssertTrue(completed.boolValue) + XCTAssertEqual( + RunnerAXSnapshotBridge.customActionReadDispatchCount(), dispatchesBefore + 2) + } + + /// The element budget bounds how many elements we read; these caps bound what + /// any ONE element can put in the response. Clipping must be reported, since + /// a clipped list looks exactly like a complete one. + func testActionNamesAreCappedPerElementAndReported() { + var truncated = ObjCBool(true) + + // Under both caps: untouched, nothing to report. + let small = ["Reply", "Repost"] + XCTAssertEqual( + RunnerAXSnapshotBridge.cappedActionNames(small, truncated: &truncated), small) + XCTAssertFalse(truncated.boolValue) + + // More actions than the per-element cap: clipped to the first 8, reported. + let many = (1...20).map { "Action \($0)" } + let cappedMany = RunnerAXSnapshotBridge.cappedActionNames(many, truncated: &truncated) + XCTAssertEqual(cappedMany.count, 8) + XCTAssertEqual(cappedMany.first, "Action 1") + XCTAssertTrue(truncated.boolValue) + + // A single very long name is shortened, reported, and stays one string. + let long = String(repeating: "a", count: 500) + let cappedLong = RunnerAXSnapshotBridge.cappedActionNames([long], truncated: &truncated) + XCTAssertEqual(cappedLong.count, 1) + XCTAssertTrue(truncated.boolValue) + XCTAssertLessThan(cappedLong[0].count, long.count) + XCTAssertTrue(cappedLong[0].hasSuffix("…")) + + // Empty input is not "truncated". + XCTAssertEqual(RunnerAXSnapshotBridge.cappedActionNames([], truncated: &truncated), []) + XCTAssertFalse(truncated.boolValue) + } + + /// Action names annotated by the bridge must survive into the emitted node — + /// the whole point of the capture is that the merged card names its hidden + /// affordances. + func testPrivateAXNodesCarryAnnotatedCustomActions() { + let tree: [String: Any] = [ + "type": Int(XCUIElement.ElementType.application.rawValue), + "label": "Blue Sky", + "frame": ["x": 0, "y": 0, "width": 390, "height": 844], + "children": [ + [ + "type": Int(XCUIElement.ElementType.link.rawValue), + "label": "feedItem-by-whiskers.test", + "frame": ["x": 0, "y": 100, "width": 390, "height": 200], + "actions": ["Reply", "Repost", "Open post options menu"], + "children": [], + ], + [ + "type": Int(XCUIElement.ElementType.button.rawValue), + "label": "Compose", + "frame": ["x": 300, "y": 700, "width": 60, "height": 60], + "children": [], + ], + ], + ] + let nodes = privateAXAcquisition( + rawRoot: tree, + hint: CaptureHint( + projection: .regular, depth: nil, regularPresentedDepth: nil, + interactiveOnly: false, customActions: false) + ) + + let card = nodes.first { $0.label == "feedItem-by-whiskers.test" } + XCTAssertEqual(card?.actions, ["Reply", "Repost", "Open post options menu"]) + // A node the bridge did not annotate stays absent, not empty. + XCTAssertNil(nodes.first { $0.label == "Compose" }?.actions) + } + + func testPrivateAXAcquisitionDoesNotInterpretScope() { + let tree: [String: Any] = [ + "type": 1, "label": "App", + "children": [ + [ + "type": 9, "identifier": "homeScreen", + "children": [ + ["type": 48, "label": "Post body without the scope text", "children": []] + ], + ], + ["type": 9, "label": "unrelated sibling", "children": []], + ], + ] + // Scope never reaches acquisition: the hint derived for a scoped request carries no scope, + // and the backend has no way to interpret one. + let nodes = privateAXAcquisition( + rawRoot: tree, + hint: SnapshotPresentation.captureHint( + for: PresentationOptions( + interactiveOnly: false, + depth: nil, + scope: "homeScreen", + raw: false + ) + ) + ) + + let labels = nodes.compactMap { $0.label ?? $0.identifier } + XCTAssertTrue(labels.contains("homeScreen")) + // Descendants of the matched scope are included even when they do not contain the text. + XCTAssertTrue(labels.contains("Post body without the scope text")) + XCTAssertTrue(labels.contains("unrelated sibling")) + } + + func testPrivateAXInteractiveFiltersLoginLikeHiddenDrawer() throws { + let tree: [String: Any] = [ + "type": Int(XCUIElement.ElementType.application.rawValue), + "label": "Blue Sky", + "frame": ["x": 0, "y": 0, "width": 390, "height": 844], + "children": [ + [ + "type": Int(XCUIElement.ElementType.scrollView.rawValue), + "frame": ["x": 0, "y": 0, "width": 390, "height": 844], + "children": [ + [ + "type": Int(XCUIElement.ElementType.image.rawValue), + "label": "Callstack", + "frame": ["x": 145, "y": 104, "width": 100, "height": 100], + "children": [], + ], + [ + "type": Int(XCUIElement.ElementType.staticText.rawValue), + "label": "Welcome back", + "frame": ["x": 32, "y": 260, "width": 326, "height": 32], + "children": [], + ], + [ + "type": Int(XCUIElement.ElementType.textField.rawValue), + "label": "Email", + "identifier": "login.email", + "frame": ["x": 32, "y": 348, "width": 326, "height": 48], + "children": [], + ], + [ + "type": Int(XCUIElement.ElementType.secureTextField.rawValue), + "label": "Password", + "identifier": "login.password", + "frame": ["x": 32, "y": 412, "width": 326, "height": 48], + "children": [], + ], + [ + "type": Int(XCUIElement.ElementType.button.rawValue), + "label": "Sign in", + "identifier": "login.submit", + "frame": ["x": 32, "y": 492, "width": 326, "height": 52], + "children": [], + ], + [ + "type": Int(XCUIElement.ElementType.link.rawValue), + "label": "Forgot password?", + "frame": ["x": 128, "y": 568, "width": 134, "height": 32], + "children": [], + ], + [ + "type": Int(XCUIElement.ElementType.button.rawValue), + "label": "Admin settings", + "frame": ["x": -260, "y": 184, "width": 220, "height": 44], + "children": [], + ], + [ + "type": Int(XCUIElement.ElementType.other.rawValue), + "frame": ["x": 16, "y": 184, "width": 220, "height": 44], + "children": [], + ], + ], + ] + ], + ] + let viewport = CGRect(x: 0, y: 0, width: 390, height: 844) + let hint = CaptureHint( + projection: .regular, depth: nil, regularPresentedDepth: nil, + interactiveOnly: true, customActions: false) + let acquired = SnapshotGeometrySpace.normalized( + nodes: privateAXAcquisition(rawRoot: tree, hint: hint), + viewport: .reported(box: viewport, interfaceOrientation: RunnerInterfaceOrientation.portrait) + ) + // Acquisition serializes the drawer too; the shared fold is what hides it (#1797). + XCTAssertTrue(acquired.compactMap(\.label).contains("Admin settings")) + + let capture = try SnapshotPresentation.presentRegular( + SnapshotAcquisition( + hint: hint, nodes: acquired, truncated: false, effectiveDepth: nil, viewport: .reported(box: viewport)), + options: PresentationOptions(interactiveOnly: true, depth: nil, scope: nil, raw: false), + policy: .cursorProjected + ) + let labels = capture.nodes.compactMap { $0.label } + XCTAssertEqual( + labels, + ["Blue Sky", "Callstack", "Welcome back", "Email", "Password", "Sign in", "Forgot password?"] + ) + XCTAssertFalse(labels.contains("Admin settings")) + } +} + +/// Stands in for an AX client whose `attributesForElement:` never returns — +/// the wedged-server case the containment exists for. `release()` lets the +/// hung call finish so recovery is observable. +private final class HungAXClientForTesting: NSObject { + private let gate = DispatchSemaphore(value: 0) + private let releasedOnce = NSLock() + private var released = false + + @objc(attributesForElement:attributes:error:) + func attributes(forElement element: Any, attributes: Any, error: NSErrorPointer) -> Any? { + releasedOnce.lock() + let alreadyReleased = released + releasedOnce.unlock() + // Once the wedge clears, the server answers normally again — that is what + // makes the recovery leg a recovery rather than a second hang. + if alreadyReleased { + return nil + } + gate.wait() + return nil + } + + func release() { + releasedOnce.lock() + defer { releasedOnce.unlock() } + guard !released else { return } + released = true + gate.signal() + } +} + +/// Minimal snapshot stand-in whose accessibilityElement resolves (so the +/// extension proceeds to the request) while the paired fake client cannot +/// serve it — the failed-re-root miss path. +private final class FrontierSnapshotWithElementForTesting: NSObject { + @objc let accessibilityElement = NSObject() +} + +/// The vanished-element case: KVC resolves the property and gets nil. +private final class FrontierSnapshotWithoutElementForTesting: NSObject { + @objc let accessibilityElement: NSObject? = nil +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertDispatchTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertDispatchTests.swift index 7cbca88ed1..0fe213c883 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertDispatchTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertDispatchTests.swift @@ -6,11 +6,8 @@ extension RunnerTests { final class ResultBox { var routingProbeCount = 0 var resolutionCount = 0 - var response: Response? - var error: Error? } let box = ResultBox() - let finished = expectation(description: "alert dispatch finished") currentApp = springboard currentBundleId = Self.springboardBundleId systemModalProbeOverrideForTesting = { _ in @@ -30,31 +27,19 @@ extension RunnerTests { let command = try runnerCommandFixture( #"{"command":"alert","commandId":"alert-routing-once","appBundleId":"com.apple.springboard","action":"get","timeoutMs":1000}"# ) - DispatchQueue(label: "agent-device.runner.tests.alert-routing").async { - do { - box.response = try self.execute(command: command) - } catch { - box.error = error - } - finished.fulfill() - } - wait(for: [finished], timeout: 2) - XCTAssertNil(box.error) - XCTAssertEqual(box.response?.error?.code, "ALERT_NOT_FOUND") + let response = try execute(command: command) + XCTAssertEqual(response.error?.code, "ALERT_NOT_FOUND") XCTAssertEqual(box.resolutionCount, 1) XCTAssertEqual(box.routingProbeCount, 0) } func testAlertResolutionCannotBypassRequestedDeadline() throws { final class ResultBox { - var error: Error? var observedDeadline: Date? - var commandStartedAt: Date? } let box = ResultBox() let releaseResolution = DispatchSemaphore(value: 0) let resolutionExited = expectation(description: "bounded alert resolution exited") - let commandFinished = expectation(description: "alert command respected its deadline") let command = try runnerCommandFixture( #"{"command":"alert","commandId":"alert-deadline","appBundleId":"com.apple.springboard","action":"get","timeoutMs":500}"# ) @@ -73,26 +58,16 @@ extension RunnerTests { currentBundleId = nil } - DispatchQueue(label: "agent-device.runner.tests.alert-deadline").async { - box.commandStartedAt = Date() - do { - _ = try self.execute(command: command) - } catch { - box.error = error - } - commandFinished.fulfill() - } - - wait(for: [commandFinished], timeout: 1) - let error = box.error as NSError? - XCTAssertEqual(error?.domain, RunnerErrorDomain.general) - XCTAssertEqual(error?.code, RunnerErrorCode.mainThreadExecutionTimedOut) - XCTAssertNotNil(box.observedDeadline) - if let observedDeadline = box.observedDeadline, - let commandStartedAt = box.commandStartedAt - { - XCTAssertEqual(observedDeadline.timeIntervalSince(commandStartedAt), 0.5, accuracy: 0.05) + let commandStartedAt = Date() + // The resolution outlives the 500 ms request, so only the deadline-bounded dispatch can throw + // the main-thread timeout; a bypassed deadline answers ALERT_NOT_FOUND once it returns. + XCTAssertThrowsError(try execute(command: command)) { error in + let error = error as NSError + XCTAssertEqual(error.domain, RunnerErrorDomain.general) + XCTAssertEqual(error.code, RunnerErrorCode.mainThreadExecutionTimedOut) } + let observedDeadline = try XCTUnwrap(box.observedDeadline) + XCTAssertEqual(observedDeadline.timeIntervalSince(commandStartedAt), 0.5, accuracy: 0.05) releaseResolution.signal() wait(for: [resolutionExited], timeout: 1) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertObservationTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertObservationTests.swift index 508ed18691..b57b0ed27c 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertObservationTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertObservationTests.swift @@ -18,23 +18,6 @@ extension RunnerTests { ) } - func testAlertReplacementCommandDeadlineScalesWithMeasuredLatency() { - // A quiet host still asks for the production default: 24 reads at 10 ms is a quarter of a second. - XCTAssertEqual(RunnerTests.alertReplacementCommandTimeoutMs(measuredRoundTrip: 0.001), 10_000) - XCTAssertEqual(RunnerTests.alertReplacementCommandTimeoutMs(measuredRoundTrip: 0.01), 10_000) - // The linear region is where a contended host lands: one second per read buys the 24 reads - // twice over, which is what the red nightly nights needed and 10 s did not cover. - XCTAssertEqual(RunnerTests.alertReplacementCommandTimeoutMs(measuredRoundTrip: 1), 48_000) - // Past the cap the fixture gives up rather than dominate the lane, whatever the host costs. - XCTAssertEqual(RunnerTests.alertReplacementCommandTimeoutMs(measuredRoundTrip: 2.5), 60_000) - XCTAssertEqual(RunnerTests.alertReplacementCommandTimeoutMs(measuredRoundTrip: 300), 60_000) - XCTAssertEqual( - RunnerTests.alertResolutionReadsBeforeActivation, - 24, - "the decomposition below is the traced command path; re-trace it before changing a count" - ) - } - func testAlertDeadlineBeforeActivationLeavesTheOriginalUntouched() throws { app.launchArguments = ["--agent-device-alert-replacement-regression"] app.launch() @@ -43,7 +26,7 @@ extension RunnerTests { app.terminate() } XCTAssertTrue(app.alerts.firstMatch.waitForExistence(timeout: appExistenceTimeout)) - let alert = try XCTUnwrap(resolveAlert(app: app, deadline: Date().addingTimeInterval(10))) + let alert = try resolveAlertBeforeTheCommand() let response = handleAlert(alert, action: "accept", deadline: .distantPast) XCTAssertFalse(response.ok) XCTAssertEqual(response.error?.code, "ALERT_DEADLINE_EXCEEDED") @@ -60,7 +43,7 @@ extension RunnerTests { app.terminate() } XCTAssertTrue(app.alerts.firstMatch.waitForExistence(timeout: appExistenceTimeout)) - let alert = try XCTUnwrap(resolveAlert(app: app, deadline: Date().addingTimeInterval(10))) + let alert = try resolveAlertBeforeTheCommand() alertButtonHittabilityProbeOverrideForTesting = { probeDeadline in while Date() < probeDeadline { Thread.sleep(forTimeInterval: min(0.02, max(0, probeDeadline.timeIntervalSinceNow))) @@ -77,7 +60,8 @@ extension RunnerTests { func testAlertActivationIgnoresAnAppThatNeverSettlesBeforeTheDeadline() throws { app.launchArguments = [ "--agent-device-alert-replacement-regression", - "--agent-device-alert-activation-busy" + "--agent-device-alert-activation-busy", + String(RunnerTests.alertResolutionAllowance + RunnerTests.alertActivationDeadline) ] app.launch() defer { @@ -85,105 +69,101 @@ extension RunnerTests { app.terminate() } XCTAssertTrue(app.alerts.firstMatch.waitForExistence(timeout: appExistenceTimeout)) - let alert = try XCTUnwrap(resolveAlert(app: app, deadline: Date().addingTimeInterval(10))) + let alert = try resolveAlertBeforeTheCommand() - let deadline = Date().addingTimeInterval(6) - let startedAt = Date() - let response = handleAlert(alert, action: "accept", deadline: deadline) - let elapsed = Date().timeIntervalSince(startedAt) + let response = handleAlert(alert, action: "accept", deadline: Date().addingTimeInterval(RunnerTests.alertActivationDeadline)) - // The fixture keeps an animation in flight, which is what XCTest waits out before it synthesises - // an event, so this comes back early only because activation refused to wait for an app that has - // no intention of settling (#2546). - XCTAssertLessThan(elapsed, 9, "activation waited \(elapsed)s for a busy app to idle") + // The fixture keeps an animation in flight until a button is answered or its backstop, which + // outlasts resolution plus activation, stops it, and an in-flight animation is what XCTest waits + // out before it synthesises an event. An answer that arrives while the app is still busy is one + // activation did not wait to idle (#2546). XCTAssertTrue(response.ok, String(describing: response.error)) - let recordedActions = app.staticTexts["agent-device-alert-actions"].label - if response.ok { - XCTAssertEqual(recordedActions, "First actions: 1; replacement actions: 0") - } else { - XCTAssertEqual( - recordedActions, - "First actions: 0; replacement actions: 0", - "a caller told about an expired deadline must not have a button activated behind it" - ) - } + XCTAssertEqual(app.staticTexts["agent-device-alert-actions"].label, "First actions: 1; replacement actions: 0") + XCTAssertEqual(app.staticTexts["agent-device-alert-busy-answer"].label, "Answered while busy") } - /// The accessibility round trips `resolveAlert` spends before a button is chosen, decomposed from - /// the implementation rather than counted off one trace: the blocking-modal probe scans - /// SpringBoard's alert and sheet lists and re-reads the candidate it settles on; the app's own - /// alert list is resolved and then read for existence and frame; `actionableElements` issues one - /// fetch per member of `actionableTypes`; and every candidate then pays `exists`, `isHittable`, - /// `elementType`, `frame`, `label` and `isEnabled` of its own. - static let alertResolutionModalProbeReads = 3 - static let alertResolutionAlertRootReads = 3 - static let alertResolutionActionableTypeReads = 6 - static let alertResolutionButtonCandidates = 2 - static let alertResolutionCandidateReads = 6 - - static let alertResolutionReadsBeforeActivation = - alertResolutionModalProbeReads + alertResolutionAlertRootReads + alertResolutionActionableTypeReads - + alertResolutionButtonCandidates * alertResolutionCandidateReads - - /// The reads that follow resolution — the hittability wait and the first verification observation - /// run on the same channel — plus the allowance for a host that gets slower mid-command. - private static let alertCommandSafetyFactor: TimeInterval = 2 - - /// No higher than the slowest test this lane already runs (61 s on a green nightly), so a badly - /// starved host cannot make these fixtures the lane's worst contributor. The cap covers the worst - /// night traced so far, which needed 40.4 s of deadline. - private static let alertCommandTimeoutCap: TimeInterval = 60 - - /// The alert command's deadline for the replacement fixtures. - /// - /// `timeoutMs` bounds the whole command, so it is only meaningful in units of what one - /// accessibility read costs this host right now: `RunnerTests+Alert.swift` starts the deadline at - /// dispatch and the resolution above spends all of it reading before anything is activated. The - /// hosted lane runs a read at about 10 ms on a healthy night and about 1.7 s on a red one, which - /// is why the fixed 10 s these fixtures asked for passed on green nights and expired mid-resolution - /// on red ones (runs 35, 36 and 37 needed 31.2 s, 30.6 s and 40.4 s). Deriving the budget from a - /// measured read is `docs/agents/testing.md`'s preferred answer for a timeout that only fails on a - /// contended host; production still defaults to `defaultAlertCommandTimeout` for real callers. - static func alertReplacementCommandTimeoutMs(measuredRoundTrip: TimeInterval) -> Int { - let derived = measuredRoundTrip * Double(alertResolutionReadsBeforeActivation) * alertCommandSafetyFactor - return Int((min(max(derived, defaultAlertCommandTimeout), alertCommandTimeoutCap) * 1000).rounded()) + func testAlertActivationDoesNotWaitOutANotificationBanner() throws { + app.launchArguments = ["--agent-device-alert-replacement-regression", "--agent-device-alert-banner"] + app.launch() + let banner = XCUIApplication(bundleIdentifier: "com.apple.springboard") + .descendants(matching: .any)["NotificationShortLookView"] + var consultedInterruptions: [String] = [] + let monitor = addUIInterruptionMonitor(withDescription: "alert activation banner") { element in + consultedInterruptions.append(element.identifier) + return false + } + defer { + removeUIInterruptionMonitor(monitor) + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + _ = banner.waitForNonExistence(timeout: 15) + } + acceptNotificationAuthorizationUntilAlertAppears() + XCTAssertTrue(banner.waitForExistence(timeout: appExistenceTimeout), "the fixture keeps a banner up") + let alert = try resolveAlertBeforeTheCommand() + + let response = handleAlert(alert, action: "accept", deadline: Date().addingTimeInterval(RunnerTests.alertActivationDeadline)) + + XCTAssertEqual(consultedInterruptions, [], "alert activation waited on XCTest's interruption handling") + XCTAssertTrue(response.ok, String(describing: response.error)) + XCTAssertEqual(app.staticTexts["agent-device-alert-actions"].label, "First actions: 1; replacement actions: 0") } - /// Samples the read the command is about to pay 24 times, once the alert is up, so the sample comes - /// from the window the command runs in. Two samples and the slower one wins: a single lucky read - /// must not size the budget low, and this is a worst-of measurement, not a mean. - private func measuredAccessibilityRoundTrip() -> TimeInterval { - let recordedActions = app.staticTexts["agent-device-alert-actions"] - var roundTrip = TimeInterval(0.001) - for _ in 0..<2 { - let startedAt = Date() - _ = recordedActions.label - roundTrip = max(roundTrip, Date().timeIntervalSince(startedAt)) + /// The banner fixture presents its alert only once this app may post notifications; a fresh + /// simulator asks first, through SpringBoard. + private func acceptNotificationAuthorizationUntilAlertAppears() { + let allow = XCUIApplication(bundleIdentifier: "com.apple.springboard").alerts.buttons["Allow"] + let fixtureAlert = app.alerts.firstMatch + let deadline = Date().addingTimeInterval(appExistenceTimeout) + while Date() < deadline, !fixtureAlert.exists { + if allow.exists { + allow.tap() + } else { + Thread.sleep(forTimeInterval: 0.25) + } } - return roundTrip + XCTAssertTrue(fixtureAlert.exists, "the banner fixture needs notification authorization before it presents its alert") + } + + /// Resolution reads the alert, its owner and every candidate button before anything is activated, + /// and a contended hosted simulator has spent 40 s of reads there. These fixtures prove what + /// activation and verification do, so they resolve first under this allowance, which only a failed + /// resolution ever spends. + static let alertResolutionAllowance: TimeInterval = 60 + + /// The deadline activation and verification run under once the alert is resolved: a confirmed + /// answer returns as soon as verification sees the replacement, so only a fixture whose answer is + /// the deadline itself pays it in full. It buys the dozen reads after resolution 2.5 s each, above the + /// 1.7 s a read cost on the worst hosted nights traced (#2708). + static let alertActivationDeadline: TimeInterval = 30 + + private func resolveAlertBeforeTheCommand() throws -> RunnerAlert { + try XCTUnwrap(resolveAlert(app: app, deadline: Date().addingTimeInterval(RunnerTests.alertResolutionAllowance))) } private func assertReplacementAlertUntouched(action: String, arguments: [String], confirmed: Bool) throws { app.launchArguments = ["--agent-device-alert-replacement-regression"] + arguments app.launch() defer { + alertResolutionOverrideForTesting = nil invalidateCachedTarget(reason: "unit_test_cleanup") app.terminate() } XCTAssertTrue(app.alerts.firstMatch.waitForExistence(timeout: appExistenceTimeout)) - let measuredRoundTrip = measuredAccessibilityRoundTrip() - let timeoutMs = RunnerTests.alertReplacementCommandTimeoutMs(measuredRoundTrip: measuredRoundTrip) + let original = try resolveAlertBeforeTheCommand() + alertResolutionOverrideForTesting = { _ in original } + let timeoutMs = Int(RunnerTests.alertActivationDeadline * 1000) let command = try runnerCommandFixture( #"{"command":"alert","commandId":"alert-replacement","action":"\#(action)","timeoutMs":\#(timeoutMs)}"# ) let response = try executeOnMainPrepared(command: command, activeApp: app) - let budget = "timeoutMs \(timeoutMs) from a \(String(format: "%.3f", measuredRoundTrip))s read" - XCTAssertEqual(response.ok, confirmed, "\(budget): \(String(describing: response.error))") + alertResolutionOverrideForTesting = nil + XCTAssertEqual(response.ok, confirmed, String(describing: response.error)) if !confirmed { XCTAssertEqual(response.error?.code, "ALERT_DEADLINE_EXCEEDED") } XCTAssertTrue(app.alerts.firstMatch.exists, "the replacement must remain visible") XCTAssertEqual(app.staticTexts["agent-device-alert-actions"].label, "First actions: 1; replacement actions: 0") - let current = try XCTUnwrap(resolveAlert(app: app, deadline: Date().addingTimeInterval(10))) - let inspection = handleAlert(current, action: "get", deadline: Date().addingTimeInterval(10)) + let current = try resolveAlertBeforeTheCommand() + let inspection = handleAlert(current, action: "get", deadline: Date().addingTimeInterval(RunnerTests.alertActivationDeadline)) XCTAssertTrue(inspection.ok) XCTAssertEqual(inspection.data?.items?.sorted(), ["Cancel", "OK"]) XCTAssertEqual(app.staticTexts["agent-device-alert-actions"].label, "First actions: 1; replacement actions: 0") diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertTests.swift new file mode 100644 index 0000000000..c5c18f6140 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertTests.swift @@ -0,0 +1,59 @@ +import XCTest + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +extension RunnerTests { + func testAlertAcceptTreatsOpenAsAffirmative() { + XCTAssertTrue(isAcceptButton("Open")) + } +} +#endif + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS) +extension RunnerTests { + func testAlertResolutionWithoutAnAlertDoesNotReadEveryElementOfTheScreen() throws { + launchCrowdedScreen(extraArguments: []) + defer { terminateCrowdedScreen() } + + let startedAt = Date() + XCTAssertNil(resolveAlert(app: app, deadline: startedAt.addingTimeInterval(RunnerTests.defaultAlertCommandTimeout))) + // Half the command's own budget: an absent alert answers ALERT_NOT_FOUND well inside it on a + // contended host, while a read per element spends it many times over on this screen. + XCTAssertLessThan(Date().timeIntervalSince(startedAt), RunnerTests.defaultAlertCommandTimeout / 2) + } + + func testAlertResolutionFindsADismissPopupMarkerOnACrowdedScreen() throws { + launchCrowdedScreen(extraArguments: ["--agent-device-dismiss-popup"]) + defer { terminateCrowdedScreen() } + + let alert = try XCTUnwrap( + resolveAlert(app: app, deadline: Date().addingTimeInterval(RunnerTests.defaultAlertCommandTimeout)) + ) + XCTAssertEqual(alert.source, .dismissPopup) + XCTAssertEqual(alert.root.elementType, .window) + XCTAssertTrue(alert.buttons.contains { $0.identifier == " Dismiss Popup " }) + } + + func testAlertResolutionFindsAWindowThatIsItselfTheDismissPopupMarker() throws { + launchCrowdedScreen(extraArguments: ["--agent-device-dismiss-popup-window"]) + defer { terminateCrowdedScreen() } + + let alert = try XCTUnwrap( + resolveAlert(app: app, deadline: Date().addingTimeInterval(RunnerTests.defaultAlertCommandTimeout)) + ) + XCTAssertEqual(alert.source, .dismissPopup) + XCTAssertEqual(alert.root.identifier, "Dismiss popup") + XCTAssertTrue(alert.buttons.contains { $0.identifier == "agent-device-close-popover" }) + } + + private func launchCrowdedScreen(extraArguments: [String]) { + app.launchArguments = ["--agent-device-crowded-screen"] + extraArguments + app.launch() + XCTAssertTrue(app.staticTexts["agent-device-crowded-row-499"].waitForExistence(timeout: appExistenceTimeout)) + } + + private func terminateCrowdedScreen() { + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AppScreenCaptureTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AppScreenCaptureTests.swift new file mode 100644 index 0000000000..74b65986a2 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AppScreenCaptureTests.swift @@ -0,0 +1,159 @@ +import XCTest + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +/// One read of contracts/fixtures/screen-capture-metadata.json. The capture decodes through the +/// production `ScreenshotMetadataPayload`, so the table cannot describe a shape this encoder does +/// not write, and this file's re-encode cannot write a shape the table does not describe. The table +/// names no field of its own: this struct and the host's reader in +/// `packages/contracts/src/screen-capture-contract.ts` are the only declarations of the shape, and +/// `packages/contracts/src/screen-capture-contract.test.ts` is the vitest twin (#2728). +private struct ScreenCaptureMetadataTable: Decodable { + struct Capture: Decodable { + let name: String + let metadata: ScreenshotMetadataPayload + } + + let key: String + let captures: [Capture] +} + +private func loadScreenCaptureMetadataTable() throws -> ScreenCaptureMetadataTable { + let fixtureURL = URL(fileURLWithPath: #filePath) + .deletingLastPathComponent() // UnitTests + .deletingLastPathComponent() // AgentDeviceRunnerUITests + .deletingLastPathComponent() // AgentDeviceRunner + .deletingLastPathComponent() // runner + .deletingLastPathComponent() // apple + .deletingLastPathComponent() // repo root + .appendingPathComponent("contracts/fixtures/screen-capture-metadata.json") + return try JSONDecoder().decode( + ScreenCaptureMetadataTable.self, + from: Data(contentsOf: fixtureURL) + ) +} + +extension RunnerTests { + func testScreenCaptureFailureCarriesTheBridgeReasonAsItsOwnCode() { + XCTAssertEqual( + RunnerAppScreenCaptureFailure(.unresolvedWindow).rawValue, + "APP_SCREEN_WINDOW_UNRESOLVED" + ) + XCTAssertEqual( + RunnerAppScreenCaptureFailure(.unresolvedScreen).rawValue, + "APP_SCREEN_UNRESOLVED" + ) + // The display resolved and the image did not: a distinct code, because an operator chasing a + // panel problem is chasing the wrong thing here. This one has no bridge counterpart, so it is + // named on the Swift reason rather than mapped from one. + XCTAssertEqual( + RunnerAppScreenCaptureFailure.unrenderableImage.rawValue, + "APP_SCREEN_CAPTURE_UNRENDERABLE" + ) + // A success-shaped reason cannot become a success: the only way here is a caller that forgot to + // branch, and the capture it describes did not happen. + XCTAssertEqual( + RunnerAppScreenCaptureFailure(RunnerApplicationScreenFailure.none).rawValue, + "APP_SCREEN_UNRESOLVED" + ) + } + + func testScreenshotDisplayFactsReEncodeEveryGoldenTableRowUnderTheTablesKey() throws { + let table = try loadScreenCaptureMetadataTable() + XCTAssertFalse(table.captures.isEmpty, "the table must record at least one measured capture") + for capture in table.captures { + let payload = DataPayload( + message: "tmp/screenshot-1.png", + screenshotMetadata: capture.metadata + ) + let encoded = try JSONSerialization.jsonObject( + with: JSONEncoder().encode(payload) + ) as? [String: Any] + XCTAssertEqual(encoded?["message"] as? String, "tmp/screenshot-1.png", capture.name) + // The host reads these facts under the key the table spells, which neither language restates. + guard let metadata = encoded?[table.key] as? [String: Any] else { + return XCTFail("\(capture.name): no display facts under the table's key '\(table.key)'") + } + XCTAssertEqual(metadata["displayID"] as? UInt, capture.metadata.displayID, capture.name) + XCTAssertEqual(metadata["pixelWidth"] as? Int, capture.metadata.pixelWidth, capture.name) + XCTAssertEqual(metadata["pixelHeight"] as? Int, capture.metadata.pixelHeight, capture.name) + XCTAssertEqual( + metadata["pixelsPerPoint"] as? Double, + capture.metadata.pixelsPerPoint, + capture.name + ) + } + } + + func testScreenshotResultCarriesNoDisplayFactsWhenNothingResolved() throws { + let key = try loadScreenCaptureMetadataTable().key + let encoded = try JSONSerialization.jsonObject( + with: JSONEncoder().encode(DataPayload(message: "tmp/screenshot-1.png")) + ) as? [String: Any] + XCTAssertNil(encoded?[key], "a capture that resolved nothing owes no facts either") + } +} +#endif + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS) + extension RunnerTests { + /// The no-session-app and not-running-app cases both reach the resolver as an unresolved window, + /// and the capture is still owed a display: the home screen owns one (#2728). + func testObservedScreenCaptureAsksTheSystemSurfaceWhenTheSessionWindowIsUnresolved() { + var askedSystemSurface = false + let outcome = selectObservedScreenCapture( + resolving: { .failure(.unresolvedWindow) }, + fallingBack: { + askedSystemSurface = true + return .failure(.unrenderableImage) + } + ) + XCTAssertTrue(askedSystemSurface, "an unresolved window has to move the question on") + guard case .failure(let systemAnswer) = outcome else { + return XCTFail("the system surface refused, so a refusal is what the caller must get") + } + XCTAssertEqual( + systemAnswer.rawValue, + "APP_SCREEN_CAPTURE_UNRENDERABLE", + "the system surface's answer is the answer, not the session app's" + ) + } + + /// A window that resolved and then refused to name its display is the failure the host has to + /// see; asking a second process would replace a real refusal with an unrelated capture. + func testObservedScreenCaptureReportsADisplayRefusalWithoutAskingTheSystemSurface() { + var askedSystemSurface = false + let outcome = selectObservedScreenCapture( + resolving: { .failure(.unresolvedScreen) }, + fallingBack: { + askedSystemSurface = true + return .failure(.unrenderableImage) + } + ) + XCTAssertFalse(askedSystemSurface, "only an unresolved window may reach the system surface") + guard case .failure(let refusal) = outcome else { + return XCTFail("a display that refused must not become a capture") + } + XCTAssertEqual(refusal.rawValue, "APP_SCREEN_UNRESOLVED") + } + + func testObservedScreenCaptureKeepsTheSessionAppCaptureItResolved() { + var askedSystemSurface = false + let captured = CapturedAppScreen( + image: UIImage(), + displayID: 3, + pixelWidth: 2852, + pixelHeight: 2006, + pixelsPerPoint: 3 + ) + let outcome = selectObservedScreenCapture( + resolving: { .success(captured) }, + fallingBack: { + askedSystemSurface = true + return .failure(.unresolvedWindow) + } + ) + XCTAssertFalse(askedSystemSurface, "a resolved app is never second-guessed") + XCTAssertEqual(try? outcome.get().displayID, 3) + } + } +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ApplicationStateRawValueTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ApplicationStateRawValueTests.swift index 02375b2a99..84c8f2b472 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ApplicationStateRawValueTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ApplicationStateRawValueTests.swift @@ -7,8 +7,8 @@ extension RunnerTests { /// against a second handwritten table: the decoder maps the integer the runner stamps, and #2726 /// shipped `runningBackground` and `runningBackgroundSuspended` reversed. /// - /// The host lane runs this on every PR, and `.github/workflows/ios.yml` lists it for the simulator - /// lane so the suspended case — compiled out of the macOS build below — is pinned on every PR too. + /// The host lane runs this on every PR; the simulator lane derives it from the platform branch + /// below so the suspended case, compiled out of the macOS build, is pinned on every PR too. /// `packages/platform-apple/src/runner/__tests__/target-activation.test.ts` reads these calls back /// and compares them with its decode table, so a table that drifts from them fails on any host /// instead of waiting for a lane to report a mislabelled repair. diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+BlockingSystemModalResolutionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+BlockingSystemModalResolutionTests.swift new file mode 100644 index 0000000000..8a124bb669 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+BlockingSystemModalResolutionTests.swift @@ -0,0 +1,37 @@ +import XCTest + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +extension RunnerTests { + func testRemoteHostProbeRunsOnlyWhenSpringboardModalHasNoActions() { + XCTAssertTrue(RemoteHostedSystemModalPolicy.shouldProbeRemoteHost(springboardActionCount: 0)) + XCTAssertFalse(RemoteHostedSystemModalPolicy.shouldProbeRemoteHost(springboardActionCount: 1)) + XCTAssertFalse(RemoteHostedSystemModalPolicy.shouldProbeRemoteHost(springboardActionCount: 3)) + } + + func testRemoteHostStateGateFailsClosedToForeground() { + XCTAssertTrue(RemoteHostedSystemModalPolicy.isEligibleHostState(.runningForeground)) + XCTAssertFalse(RemoteHostedSystemModalPolicy.isEligibleHostState(.runningBackground)) + XCTAssertFalse(RemoteHostedSystemModalPolicy.isEligibleHostState(.notRunning)) + XCTAssertFalse(RemoteHostedSystemModalPolicy.isEligibleHostState(.unknown)) + } + + // No SpringBoard host (`hasSpringBoardSystemModalHost`) means modal resolution must return + // `.absent` without probing com.apple.springboard (#1351). Written for tvOS, where no lane + // ever executed it; `resolveBlockingSystemModal` takes that decision at RUNTIME off the same + // flag on macOS, so the host lane runs the real branch on every PR. + // + // Its former sibling `testBlockingSystemAlertSnapshotIsNilOnTvOS` is deleted rather than + // widened: `blockingSystemAlertSnapshot` is `#if os(macOS) return nil`, so on the only lane + // that could run it the assertion would pin a compile-time literal — a green that no change + // to the runner could turn red. The runtime gate it meant to cover is this test's subject, + // and the nil it returns on macOS is the compiler's business, not a test's. + #if os(tvOS) || os(macOS) + func testResolveBlockingSystemModalIsAbsentWithoutSpringBoardHost() { + guard case .absent = resolveBlockingSystemModal(deadline: .distantFuture) else { + XCTFail("blocking system-modal resolution must be .absent without a SpringBoard host") + return + } + } + #endif +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandDispatchTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandDispatchTests.swift new file mode 100644 index 0000000000..e9b6c6a1b2 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandDispatchTests.swift @@ -0,0 +1,482 @@ +import XCTest +import AgentDeviceSnapshotPresentation + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +extension RunnerTests { + func testInjectedTapRecordedFailureGateIsTapOnlyAndCountGated() { + // The seam's recording side cannot run in-bundle (a real XCTIssue would + // fail this very test run — same constraint the record(_:) suppression + // tests document); the live daemon proof covers it. This pins the gate. + XCTAssertFalse(RunnerTests.shouldInjectTapRecordedFailure(command: .tap, remaining: 0)) + XCTAssertTrue(RunnerTests.shouldInjectTapRecordedFailure(command: .tap, remaining: 1)) + XCTAssertFalse(RunnerTests.shouldInjectTapRecordedFailure(command: .type, remaining: 1)) + XCTAssertFalse(RunnerTests.shouldInjectTapRecordedFailure(command: .snapshot, remaining: 1)) + } + + func testXCTestRecordedFailureResponseFailsMutatingSuccesses() throws { + let command = try runnerCommandFixture(#"{"command":"tap","commandId":"tap-1"}"#) + let response = Response(ok: true, data: DataPayload(message: "tapped")) + + let failureResponse = xctestRecordedFailureResponse(command: command, response: response) + + XCTAssertEqual(failureResponse?.ok, false) + XCTAssertEqual(failureResponse?.error?.code, "XCTEST_RECORDED_FAILURE") + XCTAssertEqual( + failureResponse?.error?.message, + "XCTest recorded a failure while executing tap; the action may not have been performed." + ) + } + + func testXCTestRecordedFailureResponseFailsActionButtonSuccess() throws { + // The Action Button press carries no settle and no post-action observation, so this conversion is + // the only evidence the press landed. That is why the press is not classified runner-lifecycle: + // `isLifecycle` would silence the conversion here (#2699, #2702 review). + let command = try runnerCommandFixture(#"{"command":"actionButton","commandId":"action-button-1"}"#) + let response = Response(ok: true, data: DataPayload(message: "actionButton")) + + let failureResponse = xctestRecordedFailureResponse(command: command, response: response) + + XCTAssertEqual(failureResponse?.ok, false) + XCTAssertEqual(failureResponse?.error?.code, "XCTEST_RECORDED_FAILURE") + XCTAssertEqual( + failureResponse?.error?.message, + "XCTest recorded a failure while executing actionButton; the action may not have been performed." + ) + } + + func testXCTestRecordedFailureResponseDoesNotWrapReadOnlyOrRunnerFatalResponses() throws { + let snapshotCommand = try runnerCommandFixture(#"{"command":"snapshot","commandId":"snapshot-1"}"#) + let tapCommand = try runnerCommandFixture(#"{"command":"tap","commandId":"tap-1"}"#) + let runnerFatalResponse = Response( + ok: true, + data: DataPayload(runnerFatal: true, runnerFatalReason: "ax_snapshot_unavailable") + ) + + XCTAssertNil( + xctestRecordedFailureResponse( + command: snapshotCommand, + response: Response(ok: true, data: DataPayload(nodes: [], truncated: false)) + ) + ) + XCTAssertNil(xctestRecordedFailureResponse(command: tapCommand, response: runnerFatalResponse)) + } + + // Simulator-only from here to the matching #endif: these launch the host app, route through + // SpringBoard, or assert the iOS-only alert/system-modal branches. Tests outside the + // `os(iOS)` regions in this file are pure runner decisions and also run on the macOS host + // lane (ci.yml) — see the classification convention in RunnerTests.swift. +#if os(iOS) + func testMissingBundleCommandInvalidatesCompleteCachedTargetState() throws { + app.launch() + currentApp = app + currentBundleId = "com.example.stale-target" + currentAppProcessIdentifier = 42 + snapshotXCTestPenaltyWarmupExemption.isPending = true + defer { + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } + let command = try runnerCommandFixture( + #"{"command":"snapshot","commandId":"snapshot-without-bundle"}"# + ) + + _ = prepareActiveCommandContext(command: command) + + XCTAssertNil(currentApp) + XCTAssertNil(currentBundleId) + XCTAssertNil(currentAppProcessIdentifier) + XCTAssertFalse(snapshotXCTestPenaltyWarmupExemption.isPending) + } + + func testSkipAppActivationPreflightIncludesForegroundCachedCoordinateOnlyTaps() throws { + app.launch() + currentApp = app + currentBundleId = nil + defer { + currentApp = nil + currentBundleId = nil + app.terminate() + } + let tap = try runnerCommandFixture( + #"{"command":"tap","commandId":"tap-1","x":10,"y":20}"# + ) + + XCTAssertTrue(shouldSkipAppActivationPreflight(tap)) + } + + func testSkipAppActivationPreflightRejectsMissingChangedAndBackgroundTargets() throws { + let coordinateTap = try runnerCommandFixture( + #"{"command":"tap","commandId":"tap-1","x":10,"y":20}"# + ) + currentApp = nil + currentBundleId = nil + XCTAssertFalse(shouldSkipAppActivationPreflight(coordinateTap)) + + app.launch() + currentApp = app + currentBundleId = "com.example.current" + defer { + currentApp = nil + currentBundleId = nil + app.terminate() + } + let changedBundleTap = try runnerCommandFixture( + #"{"command":"tap","commandId":"tap-2","appBundleId":"com.example.other","x":10,"y":20}"# + ) + + XCTAssertFalse(shouldSkipAppActivationPreflight(changedBundleTap)) + + app.terminate() + currentApp = app + currentBundleId = nil + + XCTAssertFalse(shouldSkipAppActivationPreflight(coordinateTap)) + } + + func testActionButtonPressSkipsAppActivationPreflightWithoutBeingRunnerLifecycle() throws { + currentApp = nil + currentBundleId = nil + let press = try runnerCommandFixture(#"{"command":"actionButton","commandId":"action-button-1"}"#) + + // The skip is its own decision, reached without the lifecycle flag that would also drop the + // recorded-failure conversion; no cached target and no foreground app is required for it. + XCTAssertFalse(isRunnerLifecycleCommand(.actionButton)) + XCTAssertTrue(shouldSkipAppActivationPreflight(press)) + } + + func testPrepareActiveCommandContextRoutesBlockingSystemModalToSpringboard() throws { + blockingSystemModalPresenceOverrideForTesting = true + currentApp = nil + currentBundleId = nil + defer { + blockingSystemModalPresenceOverrideForTesting = nil + currentApp = nil + currentBundleId = nil + } + let tap = try runnerCommandFixture( + #"{"command":"tap","commandId":"tap-1","x":10,"y":20}"# + ) + + let preparation = prepareActiveCommandContext( + command: tap, + routeToSpringboard: shouldRouteToSpringboardBlockingSystemModal(tap) + ) + + guard case .context(let context) = preparation else { + XCTFail("expected command context") + return + } + XCTAssertTrue(context.app === springboard) + } + + func testExecuteDispatchedReturnsBusyBeforeBlockingSystemModalProbeDrains() throws { + app.launch() + currentApp = app + currentBundleId = nil + defer { + currentApp = nil + currentBundleId = nil + systemModalProbeOverrideForTesting = nil + clearSnapshotXCTestChannelPenalty(reason: "test-cleanup") + app.terminate() + } + + final class ResultBox { + var response: Response? + var error: Error? + var commandRecoveredBeforeRelease = false + var wasBusyBeforeRelease = false + var hadAbandonedProbeBeforeRelease = false + var drained = false + } + let box = ResultBox() + let probeStarted = expectation(description: "system-modal routing probe started") + let verificationFinished = expectation(description: "command recovery and modal probe drain verified") + let probeReleaseGate = DispatchSemaphore(value: 0) + let commandFinishedGate = DispatchSemaphore(value: 0) + systemModalProbeOverrideForTesting = { _ in + probeStarted.fulfill() + _ = probeReleaseGate.wait(timeout: .now() + 15) + return DataPayload(message: "late system modal") + } + + let command = try runnerCommandFixture( + #"{"command":"tap","commandId":"bounded-modal-routing","x":10,"y":20}"# + ) + DispatchQueue(label: "agent-device.runner.tests.modal-routing-probe").async { + do { + box.response = try self.executeDispatched(command: command) + } catch { + box.error = error + } + commandFinishedGate.signal() + } + DispatchQueue(label: "agent-device.runner.tests.modal-routing-probe-verifier").async { + let commandWait = commandFinishedGate.wait( + timeout: .now() + self.systemModalProbeBudget + 3 + ) + box.commandRecoveredBeforeRelease = commandWait == .success + && box.error == nil + && box.response?.error?.code == "RUNNER_BUSY" + if case .busy = self.currentMainThreadBusyState() { + box.wasBusyBeforeRelease = true + } + box.hadAbandonedProbeBeforeRelease = self.hasAbandonedMainThreadWork() + + // The XCTest main thread is blocked inside the injected probe, so this verifier owns the + // ordered release after recording the command result and abandoned-work state above. + probeReleaseGate.signal() + let deadline = Date().addingTimeInterval(5) + while self.hasAbandonedMainThreadWork(), Date() < deadline { + self.sleepFor(0.002) + } + box.drained = !self.hasAbandonedMainThreadWork() + verificationFinished.fulfill() + } + + wait(for: [probeStarted, verificationFinished], timeout: 15) + XCTAssertTrue( + box.commandRecoveredBeforeRelease, + "the public coordinate tap must return RUNNER_BUSY before the blocked modal probe drains" + ) + XCTAssertTrue(box.wasBusyBeforeRelease) + XCTAssertTrue(box.hadAbandonedProbeBeforeRelease) + XCTAssertTrue(box.drained) + guard case .idle = currentMainThreadBusyState() else { + return XCTFail("expected the runner to become idle after the routing probe drained") + } + XCTAssertFalse(hasAbandonedMainThreadWork()) + } + + /// A coordinate tap resolves its system-modal routing on the command queue while main may still be + /// clearing or rebinding the cached target. Target identity belongs to main, so an abandoned + /// routing probe must arm its penalty with the identity main settled on — never with the identity + /// the command queue read while that write was still pending (#2781). + func testCoordinateTapRoutingPenalizesTheIdentityMainSettledOnWhileTheWriteWasPending() throws { + app.launch() + XCTAssertTrue(app.wait(for: .runningForeground, timeout: 10)) + let pendingBundleId = "com.example.routing-pending-stale" + let settledBundleId = "com.example.routing-pending-settled" + currentApp = app + currentBundleId = pendingBundleId + snapshotXCTestPenaltyWarmupExemption.isPending = false + clearSnapshotXCTestChannelPenalty(reason: "test-setup") + defer { + systemModalProbeOverrideForTesting = nil + clearSnapshotXCTestChannelPenalty(reason: "test-cleanup") + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } + + // Occupy main and rebind the target inside that block: every identity read that arrives while it + // is queued sees a target that main is on its way to replacing. + let mainRelease = DispatchSemaphore(value: 0) + DispatchQueue.main.async { + _ = mainRelease.wait(timeout: .now() + 0.5) + self.currentBundleId = settledBundleId + } + + let probeStarted = expectation(description: "system-modal routing probe started") + let probeReleaseGate = DispatchSemaphore(value: 0) + systemModalProbeOverrideForTesting = { _ in + probeStarted.fulfill() + // Outlives the probe's own slice, so the abandonment hook is what arms the penalty. + _ = probeReleaseGate.wait(timeout: .now() + 15) + return nil + } + let tap = try runnerCommandFixture( + #"{"command":"tap","commandId":"tap-pending-target-write","x":10,"y":20}"# + ) + + final class ResultBox { + var response: Response? + var penalizedPendingIdentity = false + var penalizedSettledIdentity = false + } + let box = ResultBox() + let tapFinished = expectation(description: "off-main tap returned") + DispatchQueue(label: "agent-device.runner.tests.tap-pending-target-write").async { + box.response = try? self.executeDispatched(command: tap) + box.penalizedPendingIdentity = self.isSnapshotXCTestChannelPenalized(bundleId: pendingBundleId) + box.penalizedSettledIdentity = self.isSnapshotXCTestChannelPenalized(bundleId: settledBundleId) + probeReleaseGate.signal() + tapFinished.fulfill() + } + + wait(for: [probeStarted, tapFinished], timeout: 40) + mainRelease.signal() + let drainDeadline = Date().addingTimeInterval(5) + while hasAbandonedMainThreadWork(), Date() < drainDeadline { + sleepFor(0.002) + } + + XCTAssertTrue( + box.penalizedSettledIdentity, + "the abandoned routing probe must penalize the target main settled on" + ) + XCTAssertFalse( + box.penalizedPendingIdentity, + "the command queue may not key a penalty with an identity whose write was still pending on main" + ) + XCTAssertFalse(hasAbandonedMainThreadWork()) + } + + func testSkipAppActivationPreflightRejectsSelectorAndMixedSequenceGestures() throws { + app.launch() + currentApp = app + currentBundleId = nil + defer { + currentApp = nil + currentBundleId = nil + app.terminate() + } + let selectorTap = try runnerCommandFixture( + #"{"command":"tap","commandId":"tap-1","selectorKey":"label","selectorValue":"Search","synthesized":true}"# + ) + let standardDrag = try runnerCommandFixture( + #"{"command":"drag","commandId":"drag-1","x":10,"y":20,"x2":30,"y2":40}"# + ) + let mixedSequence = try runnerCommandFixture( + """ + {"command":"sequence","commandId":"seq-1","steps":[ + {"kind":"tap","x":10,"y":20,"synthesized":true}, + {"kind":"doubleTap","x":30,"y":40} + ]} + """ + ) + + XCTAssertFalse(shouldSkipAppActivationPreflight(selectorTap)) + XCTAssertFalse(shouldSkipAppActivationPreflight(standardDrag)) + XCTAssertFalse(shouldSkipAppActivationPreflight(mixedSequence)) + } + + // Launches nothing, but still simulator-only: `shouldSkipAppActivationPreflight` is + // `#if os(iOS) …guards… #else return false #endif`, so on macOS this asserts a compile-time + // literal and no edit to the iOS body could make it red. Its five siblings above and below + // are gated for the same reason. + func testSkipAppActivationPreflightRequiresCachedForegroundTarget() throws { + currentApp = nil + currentBundleId = nil + let scroll = try runnerCommandFixture( + #"{"command":"scroll","commandId":"scroll-1","direction":"down","pixels":400}"# + ) + + XCTAssertFalse(shouldSkipAppActivationPreflight(scroll)) + } + + func testSkipAppActivationPreflightKeepsDragScrollAndSequenceOnForegroundGuard() throws { + app.launch() + currentApp = app + currentBundleId = nil + defer { + currentApp = nil + currentBundleId = nil + app.terminate() + } + let drag = try runnerCommandFixture( + #"{"command":"drag","commandId":"drag-1","x":10,"y":20,"x2":30,"y2":40}"# + ) + let scroll = try runnerCommandFixture( + #"{"command":"scroll","commandId":"scroll-1","direction":"down","pixels":400}"# + ) + let sequence = try runnerCommandFixture( + """ + {"command":"sequence","commandId":"seq-1","steps":[ + {"kind":"tap","x":10,"y":20,"synthesized":true}, + {"kind":"longPress","x":10,"y":200,"durationMs":300} + ]} + """ + ) + + XCTAssertFalse(shouldSkipAppActivationPreflight(drag)) + XCTAssertFalse(shouldSkipAppActivationPreflight(scroll)) + XCTAssertFalse(shouldSkipAppActivationPreflight(sequence)) + } + + func testSkipAppActivationPreflightIncludesAlertCommands() throws { + let alert = try runnerCommandFixture( + #"{"command":"alert","commandId":"alert-1","action":"get"}"# + ) + + XCTAssertTrue(shouldSkipAppActivationPreflight(alert)) + } +#endif + + func testDispatchReturnsBusyBeforeQueueingMainThreadWork() throws { + let command = try runnerCommandFixture(#"{"command":"snapshot","commandId":"snapshot-busy"}"#) + abandonedMainThreadWorkCount = 1 + abandonedMainThreadWorkSince = Date(timeIntervalSinceNow: -2) + defer { + abandonedMainThreadWorkCount = 0 + abandonedMainThreadWorkSince = nil + } + + let response = try execute(command: command) + + XCTAssertFalse(response.ok) + XCTAssertEqual(response.error?.code, "RUNNER_BUSY") + XCTAssertTrue(response.error?.message.contains("previous command") == true) + } + + func testDispatchReturnsWedgedBeforeQueueingMainThreadWork() throws { + let command = try runnerCommandFixture(#"{"command":"snapshot","commandId":"snapshot-wedged"}"#) + abandonedMainThreadWorkCount = 1 + abandonedMainThreadWorkSince = Date(timeIntervalSinceNow: -(mainThreadWedgeThreshold + 1)) + defer { + abandonedMainThreadWorkCount = 0 + abandonedMainThreadWorkSince = nil + } + + let response = try execute(command: command) + + XCTAssertFalse(response.ok) + XCTAssertEqual(response.error?.code, "RUNNER_WEDGED") + XCTAssertTrue(response.error?.hint?.contains("runner session will be restarted") == true) + } + + func testDispatchRecoverySkipsBookkeepingWhileXCTestChannelOccupied() { + // The #1244 recovery shape: the modal probe abandoned an XCTest query that is still grinding on + // main, the capture recovered independently, and its response is ready. The recovery loop must + // return it without re-entering the main queue for recorded-failure/retry bookkeeping (that hop + // would block behind the abandoned query and re-stall the command), and a later command must + // still see the runner busy until the abandoned work drains. Removing the guard regresses this. + let command = try! JSONDecoder().decode( + Command.self, + from: Data(#"{"command":"snapshot","commandId":"recovery-guard"}"#.utf8) + ) + let recovered = Response(ok: false, error: .targetAppUnavailable(bundleId: nil)) + + setAbandonedMainThreadWork(1) + defer { setAbandonedMainThreadWork(0) } + guard case .busy = currentMainThreadBusyState() else { + return XCTFail("expected RUNNER_BUSY while abandoned XCTest work is outstanding") + } + + var occupiedCalls = 0 + let occupied = try! executeDispatchedWithRecovery(command: command) { + occupiedCalls += 1 + return recovered + } + XCTAssertEqual(occupiedCalls, 1, "recovered response must not retry behind abandoned XCTest work") + XCTAssertEqual(occupied.ok, false) + + setAbandonedMainThreadWork(0) + guard case .idle = currentMainThreadBusyState() else { + return XCTFail("runner should be idle once the abandoned work drained") + } + var drainedCalls = 0 + _ = try! executeDispatchedWithRecovery(command: command) { + drainedCalls += 1 + return recovered + } + XCTAssertEqual(drainedCalls, 2, "with the channel free the read-only retry runs once") + } + + private func setAbandonedMainThreadWork(_ count: Int) { + mainThreadWorkLock.lock() + abandonedMainThreadWorkCount = count + abandonedMainThreadWorkSince = count > 0 ? Date(timeIntervalSinceNow: -1) : nil + mainThreadWorkLock.unlock() + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandExecutionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandExecutionTests.swift new file mode 100644 index 0000000000..af9a99939e --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandExecutionTests.swift @@ -0,0 +1,109 @@ +import XCTest +import AgentDeviceSnapshotPresentation + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS) +import ObjectiveC.runtime + +private final class RunnerSynthesizedTapFailureStub: NSObject { + @objc(synthesizeTapWithApplication:resolvedWindow:x:y:) + class func synthesizeTap(application: XCUIApplication, resolvedWindow: Any?, x: Double, y: Double) -> String? { + "forced private synthesis failure" + } +} + +extension RunnerTests { + /// Makes private tap synthesis fail until the returned closure restores it. + func forceSynthesizedTapFailure() throws -> () -> Void { + let selector = NSSelectorFromString("synthesizeTapWithApplication:resolvedWindow:x:y:") + let synthesizedTapMethod = try XCTUnwrap(class_getClassMethod(RunnerSynthesizedGesture.self, selector)) + let failureStubMethod = try XCTUnwrap(class_getClassMethod(RunnerSynthesizedTapFailureStub.self, selector)) + let originalImplementation = method_getImplementation(synthesizedTapMethod) + method_setImplementation(synthesizedTapMethod, method_getImplementation(failureStubMethod)) + return { method_setImplementation(synthesizedTapMethod, originalImplementation) } + } +} +#endif + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +extension RunnerTests { +#if os(iOS) + func testSelectorTapFallsBackToXCTestCoordinateWhenPrivateSynthesisFails() throws { + let restoreSynthesizedTap = try forceSynthesizedTapFailure() + app.launch() + currentApp = app + runnerAccessibilityHealth = .healthy + defer { + restoreSynthesizedTap() + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } + let command = try runnerCommandFixture( + #"{"command":"tap","commandId":"selector-tap-fallback","selectorKey":"label","selectorValue":"Agent Device Runner","synthesized":true}"# + ) + + let response = try executeOnMainPrepared(command: command, activeApp: app) + + XCTAssertTrue(response.ok) + XCTAssertEqual(response.data?.message, "tapped") + XCTAssertEqual(response.data?.gestureFallback, "xctest-coordinate-tap") + XCTAssertEqual(response.data?.gestureFallbackMessage, "forced private synthesis failure") + XCTAssertEqual( + response.data?.gestureFallbackHint, + "Falling back to XCTest coordinate tap may be slower and can still need a healthy accessibility tree." + ) + } +#endif + +#if os(iOS) + // `waitForTextEntryReadiness`'s hardware-keyboard fallback returns early only on confirmed + // focus (#1874), and `keyboardFocusConfirmed` reads that from the app-wide focus predicate this + // bundle otherwise refuses to trust. Two XCTest facts it rests on, neither a repository + // invariant: the predicate reports a responder that shows NO software keyboard at all, and it + // names the element well enough to tell the tapped field from another one. The fixture field is + // the exact shape the fallback exists for — a real responder with an empty `inputView` — so this + // is where both are observable. If either regressed, readiness would silently stop taking the + // fallback and spend the full readinessTimeout on every hardware-keyboard field, which no other + // assertion would notice. + func testHardwareKeyboardResponderConfirmsItsOwnKeyboardFocus() throws { + app.launchArguments = ["--agent-device-text-entry-regression"] + app.launch() + defer { + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } + XCTAssertTrue(app.waitForExistence(timeout: appExistenceTimeout)) + + let textField = app.textFields["agent-device-hardware-keyboard-input"] + XCTAssertTrue(textField.waitForExistence(timeout: appExistenceTimeout)) + let otherElement = app.staticTexts["Agent Device Runner"] + XCTAssertTrue(otherElement.waitForExistence(timeout: appExistenceTimeout)) + XCTAssertFalse( + keyboardFocusConfirmed(app: app, element: textField), + "an untapped field must not confirm focus, or the fallback would fire immediately" + ) + + let tapCommand = try runnerCommandFixture( + #"{"command":"tap","commandId":"tap-focus-confirmation","selectorKey":"id","selectorValue":"agent-device-hardware-keyboard-input"}"# + ) + let tapResponse = try executeOnMainPrepared(command: tapCommand, activeApp: app) + XCTAssertTrue(tapResponse.ok, String(describing: tapResponse.error)) + try XCTSkipIf( + isKeyboardVisible(app: app), + "software keyboard is up: this simulator cannot exercise the hidden-keyboard responder path" + ) + + let deadline = Date().addingTimeInterval(TextEntryTiming.readinessTimeout) + var confirmed = keyboardFocusConfirmed(app: app, element: textField) + while !confirmed && Date() < deadline { + sleepFor(TextEntryTiming.pollInterval) + confirmed = keyboardFocusConfirmed(app: app, element: textField) + } + XCTAssertTrue(confirmed, "a tapped responder must confirm its own keyboard focus") + XCTAssertFalse( + keyboardFocusConfirmed(app: app, element: otherElement), + "focus held by another element must read as a refusal, never as this element's focus" + ) + } +#endif +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandJournalTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandJournalTests.swift new file mode 100644 index 0000000000..44555b0b76 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandJournalTests.swift @@ -0,0 +1,340 @@ +import Foundation +import XCTest +import AgentDeviceSnapshotPresentation + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +extension RunnerTests { + func testUptimeBypassesCommandJournal() throws { + let command = runnerJournalCommand("uptime", id: "uptime-probe") + + let response = try execute(command: command) + let status = commandJournal.status(normalizedCommandId: "uptime-probe") + + XCTAssertEqual(response.ok, true) + XCTAssertNotNil(response.data?.currentUptimeMs) + XCTAssertEqual(status.lifecycleState, RunnerCommandLifecycleState.notAccepted.rawValue) + } + + func testStampingCurrentUptimePreservesPayload() { + let stamped = Response(ok: true, data: DataPayload(message: "recording started")) + .stampingCurrentUptimeMs(123.5) + + XCTAssertEqual(stamped.ok, true) + XCTAssertEqual(stamped.data?.message, "recording started") + XCTAssertEqual(stamped.data?.currentUptimeMs, 123.5) + } + + func testStampingCurrentUptimeCreatesPayloadWhenNil() { + let stamped = Response(ok: true).stampingCurrentUptimeMs(456.0) + + XCTAssertEqual(stamped.ok, true) + XCTAssertEqual(stamped.data?.currentUptimeMs, 456.0) + } + + func testStampingCurrentUptimeSkipsErrorResponses() { + let response = Response(ok: false, error: ErrorPayload(message: "boom")) + let stamped = response.stampingCurrentUptimeMs(789.0) + + XCTAssertEqual(stamped.ok, false) + XCTAssertNil(stamped.data) + XCTAssertEqual(stamped.error?.message, "boom") + } + + func testStampingCurrentMainThreadBusyPreservesPayload() { + let stamped = Response(ok: true, data: DataPayload(nodes: [], truncated: false)) + .stampingCurrentMainThreadBusy(true) + + XCTAssertEqual(stamped.ok, true) + XCTAssertEqual(stamped.data?.runnerMainThreadBusy, true) + } + + func testStampingCurrentMainThreadBusySkipsErrorResponses() { + let response = Response(ok: false, error: ErrorPayload(code: "RUNNER_BUSY", message: "busy")) + let stamped = response.stampingCurrentMainThreadBusy(true) + + XCTAssertEqual(stamped.ok, false) + XCTAssertNil(stamped.data) + XCTAssertEqual(stamped.error?.code, "RUNNER_BUSY") + } + + func testMainThreadBusyStateReportsOccupancy() { + XCTAssertFalse(MainThreadBusyState.idle.reportsMainThreadBusy) + XCTAssertTrue(MainThreadBusyState.busy(abandonedForSeconds: 5).reportsMainThreadBusy) + XCTAssertTrue(MainThreadBusyState.wedged(abandonedForSeconds: 200).reportsMainThreadBusy) + XCTAssertEqual( + Response(ok: true).stampingCurrentMainThreadBusy(false).data?.runnerMainThreadBusy, false) + } + + func testCommandFailedResponseTagsMainThreadTimeoutWithTypedCode() { + let timeout = NSError( + domain: RunnerErrorDomain.general, + code: RunnerErrorCode.mainThreadExecutionTimedOut, + userInfo: [NSLocalizedDescriptionKey: "main thread execution timed out"] + ) + + let response = commandFailedResponse(from: timeout) + + XCTAssertEqual(response.ok, false) + XCTAssertEqual(response.error?.code, RunnerWireErrorCode.mainThreadTimeout) + } + + func testCommandFailedResponseKeepsGenericCodeForOtherErrors() { + let other = NSError(domain: "SomeOtherDomain", code: 99, userInfo: nil) + + let response = commandFailedResponse(from: other) + + XCTAssertEqual(response.error?.code, "COMMAND_FAILED") + } + + func testJournalStoredResponseStaysUnstamped() throws { + let journal = RunnerCommandJournal() + let recordStart = runnerJournalCommand("recordStart", id: "record-start-anchor") + + journal.accept(command: recordStart) + journal.finish( + command: recordStart, + response: Response(ok: true, data: DataPayload(message: "recording started")) + ) + + let status = journal.status(normalizedCommandId: "record-start-anchor") + let responseJson = try XCTUnwrap(status.lifecycleResponseJson) + XCTAssertFalse(responseJson.contains("currentUptimeMs")) + } + + func testCommandJournalRetentionPolicy() throws { + let journal = RunnerCommandJournal() + + let uptime = runnerJournalCommand("uptime", id: "small-scalar") + journal.accept(command: uptime) + journal.finish( + command: uptime, + response: Response(ok: true, data: DataPayload(currentUptimeMs: 12.5)) + ) + + let scalarStatus = journal.status(normalizedCommandId: "small-scalar") + XCTAssertEqual(scalarStatus.lifecycleState, RunnerCommandLifecycleState.completed.rawValue) + XCTAssertEqual(scalarStatus.lifecycleResponseOk, true) + XCTAssertNotNil(scalarStatus.lifecycleResponseJson) + let scalarResponse = try decodeRunnerJournalResponse(scalarStatus.lifecycleResponseJson) + XCTAssertEqual(scalarResponse.data?.currentUptimeMs, 12.5) + + let querySelector = runnerJournalCommand("querySelector", id: "small-object") + journal.accept(command: querySelector) + journal.finish( + command: querySelector, + response: Response(ok: true, data: DataPayload(found: true, nodes: [runnerJournalNode()])) + ) + + let objectStatus = journal.status(normalizedCommandId: "small-object") + XCTAssertNotNil(objectStatus.lifecycleResponseJson) + let objectResponse = try decodeRunnerJournalResponse(objectStatus.lifecycleResponseJson) + XCTAssertEqual(objectResponse.data?.found, true) + XCTAssertEqual(objectResponse.data?.nodes?.count, 1) + + let snapshot = runnerJournalCommand("snapshot", id: "snapshot-tree") + journal.accept(command: snapshot) + journal.finish( + command: snapshot, + response: Response(ok: true, data: DataPayload(nodes: [runnerJournalNode()], truncated: false)) + ) + + let snapshotStatus = journal.status(normalizedCommandId: "snapshot-tree") + XCTAssertEqual(snapshotStatus.lifecycleState, RunnerCommandLifecycleState.completed.rawValue) + XCTAssertEqual(snapshotStatus.lifecycleResponseOk, true) + XCTAssertNil(snapshotStatus.lifecycleResponseJson) + + let screenshot = runnerJournalCommand("screenshot", id: "screenshot-artifact") + journal.accept(command: screenshot) + journal.finish( + command: screenshot, + response: Response(ok: true, data: DataPayload(message: "tmp/screenshot-1.png")) + ) + + let screenshotStatus = journal.status(normalizedCommandId: "screenshot-artifact") + XCTAssertEqual(screenshotStatus.lifecycleState, RunnerCommandLifecycleState.completed.rawValue) + XCTAssertEqual(screenshotStatus.lifecycleResponseOk, true) + XCTAssertNil(screenshotStatus.lifecycleResponseJson) + + let scroll = runnerJournalCommand("scroll", id: "scroll-drag") + journal.accept(command: scroll) + journal.finish( + command: scroll, + response: Response( + ok: true, + data: DataPayload( + message: "scrolled", + gestureStartUptimeMs: 1, + gestureEndUptimeMs: 2, + x: 155, + y: 420, + x2: 155, + y2: 301, + referenceWidth: 300, + referenceHeight: 600 + ) + ) + ) + + let scrollStatus = journal.status(normalizedCommandId: "scroll-drag") + XCTAssertEqual(scrollStatus.lifecycleState, RunnerCommandLifecycleState.completed.rawValue) + XCTAssertEqual(scrollStatus.lifecycleResponseOk, true) + XCTAssertNotNil(scrollStatus.lifecycleResponseJson) + let scrollResponse = try decodeRunnerJournalResponse(scrollStatus.lifecycleResponseJson) + XCTAssertEqual(scrollResponse.data?.x, 155) + XCTAssertEqual(scrollResponse.data?.y, 420) + XCTAssertEqual(scrollResponse.data?.x2, 155) + XCTAssertEqual(scrollResponse.data?.y2, 301) + XCTAssertEqual(scrollResponse.data?.referenceWidth, 300) + XCTAssertEqual(scrollResponse.data?.referenceHeight, 600) + + let largeRead = runnerJournalCommand("readText", id: "large-read") + journal.accept(command: largeRead) + journal.finish( + command: largeRead, + response: Response(ok: true, data: DataPayload(text: String(repeating: "x", count: 17 * 1024))) + ) + + let largeReadStatus = journal.status(normalizedCommandId: "large-read") + XCTAssertEqual(largeReadStatus.lifecycleState, RunnerCommandLifecycleState.completed.rawValue) + XCTAssertEqual(largeReadStatus.lifecycleResponseOk, true) + XCTAssertNil(largeReadStatus.lifecycleResponseJson) + } + + func testCommandJournalKeepsErrorMetadataWhenResponseJsonIsDropped() { + let journal = RunnerCommandJournal() + let snapshot = runnerJournalCommand("snapshot", id: "snapshot-error") + let hint = "Try a smaller read such as snapshot -s -d 8." + + journal.accept(command: snapshot) + journal.finish( + command: snapshot, + response: Response( + ok: false, + error: ErrorPayload( + code: "IOS_AX_SNAPSHOT_FAILED", + message: "iOS XCTest snapshot failed while serializing the accessibility tree.", + hint: hint + ) + ) + ) + + let status = journal.status(normalizedCommandId: "snapshot-error") + XCTAssertEqual(status.lifecycleState, RunnerCommandLifecycleState.failed.rawValue) + XCTAssertEqual(status.lifecycleResponseOk, false) + XCTAssertNil(status.lifecycleResponseJson) + XCTAssertEqual(status.lifecycleErrorCode, "IOS_AX_SNAPSHOT_FAILED") + XCTAssertEqual( + status.lifecycleErrorMessage, + "iOS XCTest snapshot failed while serializing the accessibility tree." + ) + XCTAssertEqual(status.lifecycleErrorHint, hint) + } + + func testCommandJournalRetainsCompletedSequenceResults() throws { + let journal = RunnerCommandJournal() + let sequence = runnerJournalCommand("sequence", id: "sequence-completed") + let results = (0..<20).map { _ in + SequenceStepResult( + ok: true, + kind: "tap", + errorCode: nil, + errorMessage: nil, + gestureStartUptimeMs: 100, + gestureEndUptimeMs: 120 + ) + } + + journal.accept(command: sequence) + journal.finish( + command: sequence, + response: Response( + ok: true, + data: DataPayload( + message: "sequence", + completedSteps: 20, + failedStepIndex: nil, + sequenceResults: results + ) + ) + ) + + let status = journal.status(normalizedCommandId: "sequence-completed") + XCTAssertEqual(status.lifecycleState, RunnerCommandLifecycleState.completed.rawValue) + XCTAssertEqual(status.lifecycleResponseOk, true) + let json = try XCTUnwrap(status.lifecycleResponseJson) + // Worst-case 20-step response must stay under the 16KB journal retention cap. + XCTAssertLessThan(json.utf8.count, 16 * 1024) + let decoded = try decodeRunnerJournalResponse(status.lifecycleResponseJson) + XCTAssertEqual(decoded.data?.completedSteps, 20) + XCTAssertEqual(decoded.data?.sequenceResults?.count, 20) + } + + func testCommandJournalRetainsFailedSequenceResults() throws { + let journal = RunnerCommandJournal() + let sequence = runnerJournalCommand("sequence", id: "sequence-failed") + let longError = String(repeating: "z", count: 200) + let results: [SequenceStepResult] = [ + SequenceStepResult(ok: true, kind: "tap", errorCode: nil, errorMessage: nil, + gestureStartUptimeMs: 100, gestureEndUptimeMs: 120), + SequenceStepResult(ok: true, kind: "tap", errorCode: nil, errorMessage: nil, + gestureStartUptimeMs: 130, gestureEndUptimeMs: 150), + SequenceStepResult(ok: false, kind: "longPress", errorCode: "UNSUPPORTED_OPERATION", + errorMessage: longError, gestureStartUptimeMs: 160, gestureEndUptimeMs: 180), + ] + + journal.accept(command: sequence) + journal.finish( + command: sequence, + response: Response( + ok: true, + data: DataPayload( + message: "sequence", + completedSteps: 2, + failedStepIndex: 2, + sequenceResults: results + ) + ) + ) + + let status = journal.status(normalizedCommandId: "sequence-failed") + XCTAssertEqual(status.lifecycleState, RunnerCommandLifecycleState.completed.rawValue) + let decoded = try decodeRunnerJournalResponse(status.lifecycleResponseJson) + XCTAssertEqual(decoded.data?.completedSteps, 2) + XCTAssertEqual(decoded.data?.failedStepIndex, 2) + XCTAssertEqual(decoded.data?.sequenceResults?.count, 3) + XCTAssertEqual(decoded.data?.sequenceResults?[2].ok, false) + XCTAssertEqual(decoded.data?.sequenceResults?[2].errorCode, "UNSUPPORTED_OPERATION") + } + + private func runnerJournalCommand(_ command: String, id: String) -> Command { + let json = #"{"command":"\#(command)","commandId":"\#(id)"}"# + return try! JSONDecoder().decode(Command.self, from: Data(json.utf8)) + } + + private func runnerJournalNode() -> PresentedNode { + SnapshotPresentation.singleElementRead( + RawAXNode( + index: 0, + type: "button", + label: "Continue", + identifier: "continue", + value: nil, + rect: SnapshotRect(x: 10, y: 20, width: 100, height: 44), + enabled: true, + focused: nil, + selected: nil, + hittable: true, + depth: 0, + parentIndex: nil, + hiddenContentAbove: nil, + hiddenContentBelow: nil + ) + ) + } + + private func decodeRunnerJournalResponse(_ responseJson: String?) throws -> Response { + let responseJson = try XCTUnwrap(responseJson) + return try JSONDecoder().decode(Response.self, from: Data(responseJson.utf8)) + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift index ed2b0001ca..e7bd7961c5 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift @@ -42,7 +42,7 @@ extension RunnerTests { let failureCountBefore = currentXCTestFailureCount() let typeCommand = try runnerCommandFixture( - #"{"command":"type","commandId":"type-after-penalized-non-text-target","text":"must-not-type"}"# + #"{"command":"type","commandId":"type-after-penalized-non-text-target","text":"must-not-type","textEntryMode":"append"}"# ) let typeResponse = executeTypeCommand(activeApp: app, command: typeCommand) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+GestureExecutionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+GestureExecutionTests.swift new file mode 100644 index 0000000000..d5c517cf45 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+GestureExecutionTests.swift @@ -0,0 +1,73 @@ +import XCTest +import AgentDeviceSnapshotPresentation + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +extension RunnerTests { + func testGestureResponseIncludesSynthesizedTapFallbackDiagnostics() { + let response = gestureResponse( + message: "tapped", + timing: (gestureStartUptimeMs: 1, gestureEndUptimeMs: 2), + fallback: GestureFallback( + strategy: "xctest-coordinate-tap", + message: "Runner synthesized coordinate tap is unavailable", + hint: "Using XCTest coordinate tap fallback." + ) + ) + + XCTAssertEqual(response.ok, true) + XCTAssertEqual(response.data?.gestureFallback, "xctest-coordinate-tap") + XCTAssertEqual( + response.data?.gestureFallbackMessage, + "Runner synthesized coordinate tap is unavailable" + ) + XCTAssertEqual(response.data?.gestureFallbackHint, "Using XCTest coordinate tap fallback.") + } + + func testGestureResponseIncludesMaestroNonHittableFallbackUsage() { + let response = gestureResponse( + message: "tapped via non-hittable coordinate fallback", + timing: (gestureStartUptimeMs: 1, gestureEndUptimeMs: 2), + frame: .touch(nil), + maestroNonHittableCoordinateFallbackUsed: true + ) + + XCTAssertEqual(response.data?.maestroNonHittableCoordinateFallbackUsed, true) + } + + func testCanonicalPlannedGestureResponseOmitsDragFrameAndPreservesDiagnostics() { + let response = gestureResponse( + message: "fling", + timing: (gestureStartUptimeMs: 1, gestureEndUptimeMs: 2), + frame: .drag( + DragVisualizationFrame( + x: 160, + y: 150, + x2: 40, + y2: 150, + referenceWidth: 200, + referenceHeight: 300 + ) + ), + fallback: GestureFallback( + strategy: "xctest-coordinate-drag", + message: "Private synthesis unavailable", + hint: "Using XCTest coordinate fallback." + ) + ) + + let canonical = canonicalPlannedGestureResponse(response) + + XCTAssertEqual(canonical.data?.gestureStartUptimeMs, 1) + XCTAssertEqual(canonical.data?.gestureEndUptimeMs, 2) + XCTAssertEqual(canonical.data?.gestureFallback, "xctest-coordinate-drag") + XCTAssertEqual(canonical.data?.gestureFallbackMessage, "Private synthesis unavailable") + XCTAssertEqual(canonical.data?.gestureFallbackHint, "Using XCTest coordinate fallback.") + XCTAssertNil(canonical.data?.x) + XCTAssertNil(canonical.data?.y) + XCTAssertNil(canonical.data?.x2) + XCTAssertNil(canonical.data?.y2) + XCTAssertNil(canonical.data?.referenceWidth) + XCTAssertNil(canonical.data?.referenceHeight) + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+InteractionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+InteractionTests.swift new file mode 100644 index 0000000000..659dbd9c8b --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+InteractionTests.swift @@ -0,0 +1,27 @@ +import XCTest +import AgentDeviceSnapshotPresentation + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +extension RunnerTests { + func testDesktopScrollWheelDeltasMapDirections() { + XCTAssertEqual(desktopScrollWheelDeltas(direction: .up, pixels: 120).vertical, 120) + XCTAssertEqual(desktopScrollWheelDeltas(direction: .down, pixels: 120).vertical, -120) + XCTAssertEqual(desktopScrollWheelDeltas(direction: .left, pixels: 120).horizontal, 120) + XCTAssertEqual(desktopScrollWheelDeltas(direction: .right, pixels: 120).horizontal, -120) + } + + func testDesktopScrollWheelDeltaEventsHonorDurationAndPreservePixels() { + let events = desktopScrollWheelDeltaEvents(direction: .down, pixels: 200, durationMs: 50) + XCTAssertEqual(events.count, 4) + XCTAssertEqual(events.map(\.vertical).reduce(0, +), -200) + XCTAssertEqual(events.map(\.horizontal).reduce(0, +), 0) + XCTAssertEqual(desktopScrollEventIntervalSeconds(durationMs: 50, eventCount: events.count), 0.05 / 3.0) + } + + func testDesktopScrollWheelDeltaEventsKeepInstantScrollSingleEvent() { + let events = desktopScrollWheelDeltaEvents(direction: .down, pixels: 200, durationMs: 0) + XCTAssertEqual(events.count, 1) + XCTAssertEqual(events.first?.vertical, -200) + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+KeyboardTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+KeyboardTests.swift new file mode 100644 index 0000000000..8264903617 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+KeyboardTests.swift @@ -0,0 +1,144 @@ +import XCTest +import AgentDeviceSnapshotPresentation + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +extension RunnerTests { + func testKeyboardBandFactReadFailureIsStatedAsUnmeasurableNotAbsence() { + // Absence would let a consumer claim the screen is clear of a keyboard on the strength of a read + // that never answered. + let fact = runnerKeyboardBandFact( + readSucceeded: false, + exists: false, + frame: CGRect(x: 0, y: 198, width: 402, height: 204) + ) + XCTAssertEqual(fact, .unmeasurable(RunnerKeyboardBandReason.queryFailed)) + XCTAssertEqual(fact.payload.kind, "unmeasurable") + XCTAssertEqual(fact.payload.reason, RunnerKeyboardBandReason.queryFailed) + XCTAssertNil(fact.payload.frame) + } + + func testKeyboardBandFactPublishesAbsenceWhenTheQueryFindsNoKeyboard() { + let fact = runnerKeyboardBandFact(readSucceeded: true, exists: false, frame: .zero) + XCTAssertEqual(fact, .absent) + XCTAssertEqual(fact.payload.kind, "absent") + XCTAssertNil(fact.payload.frame) + XCTAssertNil(fact.payload.reason) + } + + func testKeyboardBandFactPublishesTheMeasuredBandInAppOrientationSpace() { + // The landscape band measured on iPhone 17 Pro (iOS 26.2) after #2653: full width across the + // bottom of a 402 pt-tall app, which is what the tree reported as a strip down the left edge. + let frame = CGRect(x: 0, y: 198, width: 874, height: 204) + let fact = runnerKeyboardBandFact(readSucceeded: true, exists: true, frame: frame) + XCTAssertEqual(fact, .visible(frame)) + let payload = fact.payload + XCTAssertEqual(payload.kind, "visible") + XCTAssertEqual(payload.frame, SnapshotRect(x: 0, y: 198, width: 874, height: 204)) + XCTAssertNil(payload.reason) + } + + func testKeyboardBandFactRefusesUnusableGeometryInsteadOfClaimingAbsence() { + let nan = CGFloat(Double.nan) + let infinite = CGFloat.infinity + let unusable: [CGRect] = [ + .zero, + CGRect(x: 0, y: 198, width: 0, height: 204), + CGRect(x: 0, y: 198, width: 874, height: -1), + CGRect(x: 0, y: 198, width: -874, height: 204), + CGRect(x: 0, y: nan, width: 874, height: 204), + CGRect(x: 0, y: 198, width: nan, height: 204), + CGRect(x: infinite, y: 198, width: 874, height: 204) + ] + for frame in unusable { + let fact = runnerKeyboardBandFact(readSucceeded: true, exists: true, frame: frame) + XCTAssertEqual( + fact, + .unmeasurable(RunnerKeyboardBandReason.unusableFrame), + "expected \(frame) to be refused as a band" + ) + } + } + + func testKeyboardBandFactPayloadRoundTripsThroughTheWireShape() throws { + let cases: [RunnerKeyboardBandFact] = [ + .visible(CGRect(x: 0, y: 583, width: 402, height: 291)), + .absent, + .unmeasurable(RunnerKeyboardBandReason.queryTimeout) + ] + for fact in cases { + let data = try JSONEncoder().encode(fact.payload) + // `encodeIfPresent` for the two optional fields: a fact carries its own evidence and nothing + // else, so the daemon never has to distinguish a null from an absent key. + let object = try XCTUnwrap(JSONSerialization.jsonObject(with: data) as? [String: Any]) + switch fact { + case .visible: + XCTAssertNil(object["reason"]) + XCTAssertNotNil(object["frame"]) + case .absent: + XCTAssertNil(object["frame"]) + XCTAssertNil(object["reason"]) + case .unmeasurable: + XCTAssertNil(object["frame"]) + XCTAssertNotNil(object["reason"]) + } + XCTAssertEqual(try JSONDecoder().decode(KeyboardBandFactPayload.self, from: data), fact.payload) + } + } + + func testRunnerScreenshotStabilitySettledNeedsEnoughSamples() { + XCTAssertFalse(runnerScreenshotStabilitySettled([], requiredConsecutiveMatches: 3)) + XCTAssertFalse(runnerScreenshotStabilitySettled([Data([1])], requiredConsecutiveMatches: 3)) + let frame = Data([1, 2, 3]) + XCTAssertFalse( + runnerScreenshotStabilitySettled([frame, frame], requiredConsecutiveMatches: 3) + ) + } + + func testRunnerScreenshotStabilitySettledTrueWhenWindowMatches() { + let frame = Data([1, 2, 3]) + XCTAssertTrue( + runnerScreenshotStabilitySettled([Data([9]), frame, frame, frame], requiredConsecutiveMatches: 3) + ) + } + + func testRunnerScreenshotStabilitySettledFalseOnMidWindowMismatch() { + // A momentary pause (two matching samples) followed by resumed movement + // must not read as settled: the 3-sample window still spans the mismatch. + let frame = Data([1, 2, 3]) + let moved = Data([4, 5, 6]) + XCTAssertFalse( + runnerScreenshotStabilitySettled([frame, frame, moved], requiredConsecutiveMatches: 3) + ) + } + + func testRunnerScreenshotStabilitySettledFalseOnFailedCapture() { + // A nil sample (failed screenshot) never counts as a match, even against + // other nils — an unverifiable run must not look "stable". + XCTAssertFalse(runnerScreenshotStabilitySettled([nil, nil, nil], requiredConsecutiveMatches: 3)) + let frame = Data([1, 2, 3]) + XCTAssertFalse( + runnerScreenshotStabilitySettled([frame, frame, nil], requiredConsecutiveMatches: 3) + ) + } + + func testRunnerScreenshotStabilitySettledOnlyLooksAtTheTrailingWindow() { + // An older mismatch before the trailing window must not block settlement + // once the required run of most-recent samples agrees. + let frame = Data([9]) + XCTAssertTrue( + runnerScreenshotStabilitySettled( + [Data([1]), Data([2]), frame, frame, frame], + requiredConsecutiveMatches: 3 + ) + ) + } + + func testRunnerScreenshotStabilitySettledRejectsDegenerateRequirement() { + // Fewer than 2 required matches would make any single sample "settled" — + // guard against a misconfigured caller rather than silently no-op the wait. + XCTAssertFalse( + runnerScreenshotStabilitySettled([Data([1])], requiredConsecutiveMatches: 1) + ) + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LifecycleCacheTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LifecycleCacheTests.swift index 1cf54a6b29..3c1aa5a21c 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LifecycleCacheTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LifecycleCacheTests.swift @@ -101,10 +101,10 @@ extension RunnerTests { } func testSnapshotPenaltyWarmupExemptionIsConsumedOnce() { - snapshotXCTestPenaltyWarmupExemptionPending = true + snapshotXCTestPenaltyWarmupExemption.isPending = true - XCTAssertTrue(consumeSnapshotXCTestPenaltyWarmupExemption()) - XCTAssertFalse(consumeSnapshotXCTestPenaltyWarmupExemption()) + XCTAssertTrue(snapshotXCTestPenaltyWarmupExemption.consume()) + XCTAssertFalse(snapshotXCTestPenaltyWarmupExemption.consume()) } func testSnapshotPenaltyCanBeClearedAcrossTargetProcessReplacement() { @@ -120,14 +120,14 @@ extension RunnerTests { currentApp = app currentBundleId = "com.example.app" currentAppProcessIdentifier = 42 - snapshotXCTestPenaltyWarmupExemptionPending = true + snapshotXCTestPenaltyWarmupExemption.isPending = true invalidateCachedTarget(reason: "unit_test") XCTAssertNil(currentApp) XCTAssertNil(currentBundleId) XCTAssertNil(currentAppProcessIdentifier) - XCTAssertFalse(snapshotXCTestPenaltyWarmupExemptionPending) + XCTAssertFalse(snapshotXCTestPenaltyWarmupExemption.isPending) } func testTextEntryTapWitnessIsBoundToTargetIdentity() { @@ -146,7 +146,7 @@ extension RunnerTests { currentApp = app currentBundleId = "com.example.app" currentAppProcessIdentifier = 42 - snapshotXCTestPenaltyWarmupExemptionPending = true + snapshotXCTestPenaltyWarmupExemption.isPending = true firstInteractionReadyUptime = nil penalizeSnapshotXCTestChannel(bundleId: "com.example.app", reason: "test") XCTAssertTrue(isSnapshotXCTestChannelPenalized(bundleId: "com.example.app")) @@ -157,7 +157,7 @@ extension RunnerTests { XCTAssertNil(currentApp) XCTAssertNil(currentBundleId) XCTAssertNil(currentAppProcessIdentifier) - XCTAssertFalse(snapshotXCTestPenaltyWarmupExemptionPending) + XCTAssertFalse(snapshotXCTestPenaltyWarmupExemption.isPending) XCTAssertFalse(isSnapshotXCTestChannelPenalized(bundleId: "com.example.app")) XCTAssertNotNil(firstInteractionReadyUptime) } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LifecycleTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LifecycleTests.swift index 281e6ad43d..352cdbce1c 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LifecycleTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LifecycleTests.swift @@ -27,5 +27,151 @@ extension RunnerTests { XCTAssertEqual(isReadOnlyCommand(fixture.command), fixture.readOnly, fixture.name) } } + + func testTargetAppUnavailableResponseIsRetried() { + for bundleId in ["com.example.app", nil] as [String?] { + let response = Response(ok: false, error: .targetAppUnavailable(bundleId: bundleId)) + XCTAssertTrue(shouldRetryResponse(response), String(describing: bundleId)) + } + let reworded = ErrorPayload(message: "target vanished", retryableFailure: .targetAppUnavailable) + XCTAssertTrue(shouldRetryResponse(Response(ok: false, error: reworded))) + } + + func testUntypedUnavailableMessageIsNotRetried() { + for message in ["app 'com.example.app' is not available", "runner app is not available"] { + let response = Response(ok: false, error: ErrorPayload(message: message)) + XCTAssertFalse(shouldRetryResponse(response), message) + } + let succeeded = Response(ok: true, error: .targetAppUnavailable(bundleId: nil)) + XCTAssertFalse(shouldRetryResponse(succeeded)) + } + + func testExceptionRetryIsLimitedToTheAxServerNotFoundReadOnlyCase() throws { + let readText = try runnerCommandFixture(#"{"command":"readText"}"#) + let snapshot = try runnerCommandFixture(#"{"command":"snapshot"}"#) + let tap = try runnerCommandFixture(#"{"command":"tap"}"#) + let axServerNotFound = "NSException: Error kAXErrorServerNotFound" + XCTAssertTrue(shouldRetryException(readText, message: axServerNotFound)) + XCTAssertFalse(shouldRetryException(tap, message: axServerNotFound)) + XCTAssertFalse( + shouldRetryException(readText, message: "NSException: main thread execution timed out") + ) + XCTAssertFalse(shouldRetryException(snapshot, message: "NSException: query timed out")) + } + + func testInlineScreenshotResponseKeepsDisplayFactsBesideTheImage() throws { + let pngData = Data([0x89, 0x50, 0x4E, 0x47]) + let response = screenshotResponse( + pngData: pngData, + inlineScreenshot: true, + metadata: ScreenshotMetadataPayload( + displayID: 3, + pixelWidth: 2852, + pixelHeight: 2006, + pixelsPerPoint: 3 + ) + ) + XCTAssertTrue(response.ok) + XCTAssertEqual(response.data?.imageBase64, pngData.base64EncodedString()) + XCTAssertEqual(response.data?.screenshotMetadata?.displayID, 3) + XCTAssertNil(response.data?.message, "an inline answer must not also name a file") + } + + func testFileScreenshotResponseWritesTheImageAndStillReportsItsDisplay() throws { + let pngData = Data([0x89, 0x50, 0x4E, 0x47]) + let response = screenshotResponse( + pngData: pngData, + inlineScreenshot: false, + metadata: ScreenshotMetadataPayload( + displayID: 1, + pixelWidth: 1398, + pixelHeight: 2034, + pixelsPerPoint: 3 + ) + ) + XCTAssertTrue(response.ok) + XCTAssertNil(response.data?.imageBase64, "a file answer must not also carry bytes") + let message = try XCTUnwrap(response.data?.message) + // The answer names the file rather than carrying bytes, and the name is a host-resolvable form: + // absolute on macOS, container-relative `tmp/…` on iOS. Both name the one file just written + // into the runner's temporary directory. + let fileName = URL(fileURLWithPath: message).lastPathComponent + XCTAssertTrue(message.hasSuffix(fileName), message) +#if os(iOS) + XCTAssertTrue(message.hasPrefix("tmp/"), message) +#elseif os(macOS) + XCTAssertTrue(message.hasPrefix("/"), message) +#endif + let storedPath = (NSTemporaryDirectory() as NSString).appendingPathComponent(fileName) + XCTAssertEqual(try Data(contentsOf: URL(fileURLWithPath: storedPath)), pngData) + XCTAssertEqual(response.data?.screenshotMetadata?.pixelsPerPoint, 3) + try? FileManager.default.removeItem(atPath: storedPath) + } +} +#endif + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +extension RunnerTests { + func testResettingTargetBoundStateForgetsTheLastWrittenMarkers() { + defer { invalidateCachedTarget(reason: "unit_test_cleanup") } + lastLoggedFastAppGuardLine = "AGENT_DEVICE_RUNNER_FAST_APP_GUARD bundle=app state=4" + lastLoggedGesturePolicyLines[.scroll] = "AGENT_DEVICE_RUNNER_SYNTHESIZED_GESTURE_POLICY kind=scroll" + resetTargetBoundState() + XCTAssertNil(lastLoggedFastAppGuardLine, "a rebind must state the guard once more") + XCTAssertTrue(lastLoggedGesturePolicyLines.isEmpty, "a rebind must state the policy once more") + } + + func testFastAppGuardMarkerWritesOnceUntilTheFactChanges() { + var written: [String] = [] + runnerMarkerWriter = { written.append($0) } + defer { + runnerMarkerWriter = { NSLog("%@", $0) } + invalidateCachedTarget(reason: "unit_test_cleanup") + } + writeFastAppGuardMarker(bundleId: "com.example.app", state: .runningForeground) + writeFastAppGuardMarker(bundleId: "com.example.app", state: .runningForeground) + XCTAssertEqual(written.count, 1, "a repeated fact writes no second line") + writeFastAppGuardMarker(bundleId: "com.example.other", state: .runningForeground) + XCTAssertEqual(written.count, 2, "a changed fact writes a new line") + resetTargetBoundState() + writeFastAppGuardMarker(bundleId: "com.example.other", state: .runningForeground) + XCTAssertEqual(written.count, 3, "a rebind states the same fact once more") + } +} +#endif + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS) +extension RunnerTests { + /// Installed on every Simulator runtime and cheap to leave terminated. + private static let notRunningTargetBundleId = "com.apple.Preferences" + + private func executeOnTerminatedTarget(_ json: String) throws -> (Response, XCUIApplication) { + let target = XCUIApplication(bundleIdentifier: Self.notRunningTargetBundleId) + target.terminate() + invalidateCachedTarget(reason: "unit_test_setup") + defer { invalidateCachedTarget(reason: "unit_test_cleanup") } + return (try execute(command: try runnerCommandFixture(json)), target) + } + + /// Covers a user-level read and a mutation's leading read (a gesture's `gestureViewport`). + func testReadRefusesToLaunchANotRunningSessionApp() throws { + for command in ["snapshot", "gestureViewport"] { + let (response, target) = try executeOnTerminatedTarget( + #"{"command":"\#(command)","commandId":"read","appBundleId":"\#(Self.notRunningTargetBundleId)"}"# + ) + XCTAssertEqual(response.error?.code, RunnerWireErrorCode.appNotRunning, command) + XCTAssertEqual(target.state, .notRunning, "\(command) must not launch the session app") + target.terminate() + } + } + + func testNonReadCommandStillLaunchesANotRunningSessionApp() throws { + let (response, target) = try executeOnTerminatedTarget( + #"{"command":"activate","commandId":"repair","appBundleId":"\#(Self.notRunningTargetBundleId)"}"# + ) + defer { target.terminate() } + XCTAssertTrue(response.ok) + XCTAssertNotEqual(target.state, .notRunning) + } } #endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+MainThreadWorkTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+MainThreadWorkTests.swift new file mode 100644 index 0000000000..3a292bff9e --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+MainThreadWorkTests.swift @@ -0,0 +1,298 @@ +import XCTest + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +extension RunnerTests { + func testRunMainThreadWorkExecutesOffMainCallerOnMainThread() { + final class ResultBox { + var observedMainThread: Bool? + var error: Error? + } + let box = ResultBox() + let finished = expectation(description: "off-main caller finished") + + DispatchQueue(label: "agent-device.runner.tests.off-main").async { + do { + box.observedMainThread = try self.runMainThreadWork( + "command_execution", + timeout: 1, + timeoutError: self.mainThreadExecutionTimeoutError + ) { + Thread.isMainThread + } + } catch { + box.error = error + } + finished.fulfill() + } + + wait(for: [finished], timeout: 2) + XCTAssertNil(box.error) + XCTAssertEqual(box.observedMainThread, true) + } + + func testRunMainThreadWorkTimeoutMarksAbandonedUntilDrained() { + final class ResultBox { + var error: Error? + var abandonedCount: Int? + var abandonedSinceSet: Bool? + var busyWhileAbandoned = false + } + let box = ResultBox() + let releaseWork = DispatchSemaphore(value: 0) + let observedAbandoned = DispatchSemaphore(value: 0) + let timedOut = expectation(description: "off-main caller timed out") + + DispatchQueue(label: "agent-device.runner.tests.timeout").async { + do { + _ = try self.runMainThreadWork( + "command_execution", + timeout: 0, + timeoutError: self.mainThreadExecutionTimeoutError + ) { + _ = releaseWork.wait(timeout: .now() + 2) + return true + } + } catch { + box.error = error + } + self.mainThreadWorkLock.lock() + box.abandonedCount = self.abandonedMainThreadWorkCount + box.abandonedSinceSet = self.abandonedMainThreadWorkSince != nil + self.mainThreadWorkLock.unlock() + if case .busy = self.currentMainThreadBusyState() { + box.busyWhileAbandoned = true + } + observedAbandoned.signal() + timedOut.fulfill() + } + DispatchQueue(label: "agent-device.runner.tests.release-timeout").async { + _ = observedAbandoned.wait(timeout: .now() + 2) + releaseWork.signal() + } + + wait(for: [timedOut], timeout: 3) + let drainDeadline = Date().addingTimeInterval(2) + while hasAbandonedMainThreadWork(), Date() < drainDeadline { + sleepFor(0.005) + } + + XCTAssertEqual((box.error as NSError?)?.code, RunnerErrorCode.mainThreadExecutionTimedOut) + XCTAssertEqual(box.abandonedCount, 1) + XCTAssertEqual(box.abandonedSinceSet, true) + XCTAssertTrue(box.busyWhileAbandoned) + XCTAssertFalse(hasAbandonedMainThreadWork(), "drained work must release the main thread") + mainThreadWorkLock.lock() + let sinceCleared = abandonedMainThreadWorkSince == nil + mainThreadWorkLock.unlock() + XCTAssertTrue(sinceCleared) + guard case .idle = currentMainThreadBusyState() else { + return XCTFail("expected the runner idle once the abandoned work drained") + } + } + + func testRunMainThreadWorkReturnsWorkThatFinishedAtTheTimeoutBoundary() { + let outcome = runMainThreadWorkFinishingAtTheTimeoutBoundary { 42 } + + XCTAssertNil(outcome.error, "work that finished before the lock must not read as a timeout") + XCTAssertEqual(outcome.value, 42) + XCTAssertEqual(outcome.abandonedCount, 0) + XCTAssertEqual(outcome.onAbandonedCalls, 0) + } + + func testRunMainThreadWorkRethrowsWorkThatFailedAtTheTimeoutBoundary() { + let outcome = runMainThreadWorkFinishingAtTheTimeoutBoundary { () throws -> Int in + throw NSError(domain: "agent-device.runner.tests", code: 7) + } + + XCTAssertNil(outcome.value) + XCTAssertEqual((outcome.error as NSError?)?.domain, "agent-device.runner.tests") + XCTAssertEqual((outcome.error as NSError?)?.code, 7) + XCTAssertEqual(outcome.abandonedCount, 0) + XCTAssertEqual(outcome.onAbandonedCalls, 0) + } + + func testRunMainThreadWorkIfIdleDeclinesWhileOtherWorkIsInFlight() { + final class Outcome { + var offeredWhileInFlight = false + var whileInFlight: Bool? + var ranWhileInFlight = false + var whenIdle: Bool? + var error: Error? + } + let outcome = Outcome() + let releaseCommand = DispatchSemaphore(value: 0) + let commandEntered = DispatchSemaphore(value: 0) + let finished = expectation(description: "optional work was offered during and after the command") + + DispatchQueue(label: "agent-device.runner.tests.in-flight-command").async { + _ = try? self.runMainThreadWork( + "command_execution", + timeout: 5, + timeoutError: self.mainThreadExecutionTimeoutError + ) { + commandEntered.signal() + _ = releaseCommand.wait(timeout: .now() + 3) + } + } + DispatchQueue(label: "agent-device.runner.tests.optional-work").async { + defer { finished.fulfill() } + guard commandEntered.wait(timeout: .now() + 3) == .success else { return } + do { + outcome.whileInFlight = try self.runMainThreadWorkIfIdle( + "recording_frame", + timeout: 5, + timeoutError: self.mainThreadExecutionTimeoutError + ) { () -> Bool in + outcome.ranWhileInFlight = true + return true + } + outcome.offeredWhileInFlight = true + } catch { + outcome.error = error + } + releaseCommand.signal() + let idleDeadline = Date().addingTimeInterval(3) + while Date() < idleDeadline { + self.mainThreadWorkLock.lock() + let inFlight = self.mainThreadWorkInFlightCount + self.mainThreadWorkLock.unlock() + if inFlight == 0 { break } + usleep(2_000) + } + do { + outcome.whenIdle = try self.runMainThreadWorkIfIdle( + "recording_frame", + timeout: 5, + timeoutError: self.mainThreadExecutionTimeoutError + ) { + Thread.isMainThread + } + } catch { + outcome.error = error + } + } + + wait(for: [finished], timeout: 10) + XCTAssertNil(outcome.error) + XCTAssertTrue(outcome.offeredWhileInFlight) + XCTAssertNil(outcome.whileInFlight, "optional work declines while a hop is in flight") + XCTAssertFalse(outcome.ranWhileInFlight, "declined work is never dispatched") + XCTAssertEqual(outcome.whenIdle, true, "optional work runs on main once main is idle") + XCTAssertFalse(hasAbandonedMainThreadWork()) + } + + func testRunMainThreadWorkIfIdleDeclinesAbandonedWorkUntilItDrains() { + final class Outcome { + var abandonedCount = 0 + var inFlightCount = 0 + var offeredWhileAbandoned = false + var whileAbandoned: Bool? + var ranWhileAbandoned = false + var afterDrain: Bool? + var error: Error? + } + let outcome = Outcome() + let releaseWork = DispatchSemaphore(value: 0) + let finished = expectation(description: "optional work was offered while abandoned and after drain") + + DispatchQueue(label: "agent-device.runner.tests.abandoned-then-drained").async { + defer { finished.fulfill() } + _ = try? self.runMainThreadWork( + "command_execution", + timeout: 0, + timeoutError: self.mainThreadExecutionTimeoutError + ) { + _ = releaseWork.wait(timeout: .now() + 3) + } + self.mainThreadWorkLock.lock() + outcome.abandonedCount = self.abandonedMainThreadWorkCount + outcome.inFlightCount = self.mainThreadWorkInFlightCount + self.mainThreadWorkLock.unlock() + do { + outcome.whileAbandoned = try self.runMainThreadWorkIfIdle( + "recording_frame", + timeout: 5, + timeoutError: self.mainThreadExecutionTimeoutError + ) { () -> Bool in + outcome.ranWhileAbandoned = true + return true + } + outcome.offeredWhileAbandoned = true + } catch { + outcome.error = error + } + releaseWork.signal() + let drainDeadline = Date().addingTimeInterval(3) + while self.hasAbandonedMainThreadWork(), Date() < drainDeadline { + usleep(2_000) + } + do { + outcome.afterDrain = try self.runMainThreadWorkIfIdle( + "recording_frame", + timeout: 5, + timeoutError: self.mainThreadExecutionTimeoutError + ) { + Thread.isMainThread + } + } catch { + outcome.error = error + } + } + + wait(for: [finished], timeout: 10) + XCTAssertNil(outcome.error) + XCTAssertEqual(outcome.abandonedCount, 1) + XCTAssertEqual(outcome.inFlightCount, 1, "abandoned work stays counted in flight until it returns") + XCTAssertTrue(outcome.offeredWhileAbandoned) + XCTAssertNil(outcome.whileAbandoned, "optional work declines while abandoned work holds main") + XCTAssertFalse(outcome.ranWhileAbandoned, "declined work is never dispatched") + XCTAssertEqual(outcome.afterDrain, true, "optional work runs on main once abandoned work drained") + XCTAssertFalse(hasAbandonedMainThreadWork()) + } + + private final class BoundaryOutcome { + var value: Int? + var error: Error? + var abandonedCount: Int? + var onAbandonedCalls = 0 + } + + /// Times the wait out immediately, then lets the work finish before the watchdog takes the lock: + /// the seam releases the blocked work and waits for the serial main queue to run past it. + private func runMainThreadWorkFinishingAtTheTimeoutBoundary( + _ produce: @escaping () throws -> Int + ) -> BoundaryOutcome { + let outcome = BoundaryOutcome() + let releaseWork = DispatchSemaphore(value: 0) + let finished = expectation(description: "off-main caller finished") + mainThreadWorkTimedOutForTesting = { + releaseWork.signal() + DispatchQueue.main.sync {} + } + defer { mainThreadWorkTimedOutForTesting = nil } + + DispatchQueue(label: "agent-device.runner.tests.timeout-boundary").async { + do { + outcome.value = try self.runMainThreadWork( + "command_execution", + timeout: 0, + timeoutError: self.mainThreadExecutionTimeoutError, + onAbandoned: { outcome.onAbandonedCalls += 1 } + ) { + _ = releaseWork.wait(timeout: .now() + 2) + return try produce() + } + } catch { + outcome.error = error + } + self.mainThreadWorkLock.lock() + outcome.abandonedCount = self.abandonedMainThreadWorkCount + self.mainThreadWorkLock.unlock() + finished.fulfill() + } + + wait(for: [finished], timeout: 3) + return outcome + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ModelsTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ModelsTests.swift new file mode 100644 index 0000000000..1931498727 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ModelsTests.swift @@ -0,0 +1,146 @@ +import Foundation +import XCTest + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +extension RunnerTests { + func testErrorPayloadEncodesEveryFieldButTheRunnerInternalRetryableFailure() throws { + let payload = ErrorPayload( + code: "CODE", + message: "message", + hint: "hint", + retryableFailure: .targetAppUnavailable + ) + let encoded = try JSONEncoder().encode(payload) + let object = try XCTUnwrap(JSONSerialization.jsonObject(with: encoded) as? [String: Any]) + let storedFields = Set(Mirror(reflecting: payload).children.compactMap(\.label)) + XCTAssertEqual(Set(object.keys), storedFields.subtracting(["retryableFailure"])) + } + + func testTargetAppUnavailableErrorKeepsItsWireShape() throws { + let encoded = try JSONEncoder().encode( + ErrorPayload.targetAppUnavailable(bundleId: "com.example.app") + ) + let object = try XCTUnwrap(JSONSerialization.jsonObject(with: encoded) as? [String: Any]) + XCTAssertEqual(Array(object.keys), ["message"]) + XCTAssertEqual(object["message"] as? String, "app 'com.example.app' is not available") + XCTAssertEqual( + ErrorPayload.targetAppUnavailable(bundleId: nil).message, + "runner app is not available" + ) + } + + func runnerCommandFixture(_ json: String) throws -> Command { + try JSONDecoder().decode(Command.self, from: Data(json.utf8)) + } + + func testProductionRunnerRequestsDecodeWithoutDroppingAKey() throws { + for (name, request) in try productionRunnerRequests() { + let command = try decodeProductionRunnerRequest(request, name) + XCTAssertEqual(command.command.rawValue, request["command"] as? String, name) + let reencoded = try JSONSerialization.jsonObject(with: JSONEncoder().encode(command)) + XCTAssertEqual(runnerRequestKeyPaths(reencoded), runnerRequestKeyPaths(request), name) + } + } + + func testEveryRunnerCommandTypeHasAProductionRequest() throws { + let produced = Set(try productionRunnerRequests().compactMap { $0.request["command"] as? String }) + let orphaned = CommandType.allCases.map(\.rawValue).filter { !produced.contains($0) } + XCTAssertEqual(orphaned, [], "CommandType cases with no production request") + } + + func testEveryRunnerRequestFieldHasAProductionRequest() throws { + let entries = try productionRunnerRequests() + let requests = entries.map(\.request) + let steps = requests.flatMap { $0["steps"] as? [[String: Any]] ?? [] } + let plans = requests.compactMap { $0["gesturePlan"] as? [String: Any] } + let pointers = plans.flatMap { $0["pointers"] as? [[String: Any]] ?? [] } + let samples = pointers.flatMap { $0["samples"] as? [[String: Any]] ?? [] } + let commands = try entries.map { try decodeProductionRunnerRequest($0.request, $0.name) } + let command = try XCTUnwrap(commands.first, "no production request") + let plan = try XCTUnwrap( + commands.compactMap(\.gesturePlan).first, + "no production request carries a gesturePlan" + ) + let step = try JSONDecoder().decode(SequenceStep.self, from: Data(#"{"kind":"tap"}"#.utf8)) + let sample = try XCTUnwrap(plan.pointers.first?.samples.first) + assertEveryStoredField(of: command, appearsIn: requests, "Command") + assertEveryStoredField(of: step, appearsIn: steps, "SequenceStep") + assertEveryStoredField(of: plan, appearsIn: plans, "RunnerGesturePlan") + assertEveryStoredField( + of: plan.viewport, + appearsIn: plans.compactMap { $0["viewport"] as? [String: Any] }, + "RunnerGestureViewport" + ) + assertEveryStoredField(of: plan.pointers[0], appearsIn: pointers, "RunnerGesturePointer") + assertEveryStoredField(of: sample, appearsIn: samples, "RunnerGestureSample") + assertEveryStoredField( + of: sample.point, + appearsIn: samples.compactMap { $0["point"] as? [String: Any] }, + "RunnerGesturePoint" + ) + } + + private func productionRunnerRequests() throws -> [(name: String, request: [String: Any])] { + let fixtureURL = URL(fileURLWithPath: #filePath) + .deletingLastPathComponent() + .deletingLastPathComponent() + .deletingLastPathComponent() + .deletingLastPathComponent() + .deletingLastPathComponent() + .deletingLastPathComponent() + .appendingPathComponent("contracts/fixtures/runner-requests.json") + let entries = try XCTUnwrap( + JSONSerialization.jsonObject(with: Data(contentsOf: fixtureURL)) as? [[String: Any]] + ) + return try entries.map { entry in + ( + name: try XCTUnwrap(entry["name"] as? String), + request: try XCTUnwrap(entry["request"] as? [String: Any], "\(entry["name"] ?? "?")") + ) + } + } + + private func decodeProductionRunnerRequest(_ request: [String: Any], _ name: String) throws + -> Command + { + do { + return try JSONDecoder().decode( + Command.self, + from: JSONSerialization.data(withJSONObject: request) + ) + } catch { + XCTFail("\(name) does not decode as Command: \(error)") + throw error + } + } + + private func runnerRequestKeyPaths(_ value: Any, _ prefix: String = "") -> Set { + if let object = value as? [String: Any] { + return object.reduce(into: Set()) { paths, field in + paths.insert(prefix + field.key) + paths.formUnion(runnerRequestKeyPaths(field.value, "\(prefix)\(field.key).")) + } + } + if let array = value as? [Any] { + return array.reduce(into: Set()) { paths, element in + paths.formUnion(runnerRequestKeyPaths(element, "\(prefix)[].")) + } + } + return [] + } + + private func assertEveryStoredField( + of value: Any, + appearsIn objects: [[String: Any]], + _ level: String + ) { + let fields = Set(Mirror(reflecting: value).children.compactMap(\.label)) + let produced = Set(objects.flatMap(\.keys)) + XCTAssertEqual( + fields.subtracting(produced).sorted(), + [], + "\(level) fields with no production request" + ) + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift new file mode 100644 index 0000000000..e18e91241f --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift @@ -0,0 +1,169 @@ +import XCTest + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +extension RunnerTests { + func testTopLeadingNavigationFallbackPointTargetsHeaderControlBand() throws { + let point = try XCTUnwrap( + Self.topLeadingNavigationFallbackPoint( + in: CGRect(x: 0, y: 0, width: 430, height: 932) + ) + ) + + XCTAssertEqual(point.x, 34.4, accuracy: 0.01) + XCTAssertEqual(point.y, 132, accuracy: 0.01) + } + + func testTopLeadingNavigationFallbackPointRejectsInvalidFrame() { + XCTAssertNil(Self.topLeadingNavigationFallbackPoint(in: .infinite)) + XCTAssertNil(Self.topLeadingNavigationFallbackPoint(in: .zero)) + } + + func testNavigationBackControlRankPrefersBackThenCloseThenCancel() { + XCTAssertEqual(Self.navigationBackControlRank(label: "Back", identifier: ""), 0) + XCTAssertEqual(Self.navigationBackControlRank(label: "Close", identifier: ""), 1) + XCTAssertEqual(Self.navigationBackControlRank(label: "Cancel search", identifier: ""), 2) + XCTAssertNil(Self.navigationBackControlRank(label: "Search for more feeds", identifier: "")) + } + + func testNavigationBackPredicateUsesTheSharedKeywordTable() { + let predicate = Self.navigationBackPredicate() + + XCTAssertTrue(predicate.evaluate(with: ["label": "Back", "identifier": ""])) + XCTAssertTrue(predicate.evaluate(with: ["label": "", "identifier": "close-button"])) + XCTAssertFalse(predicate.evaluate(with: ["label": "Search for more feeds", "identifier": ""])) + } + + func testTopNavigationControlFrameAcceptsOnlyHeaderBand() { + let window = CGRect(x: 0, y: 0, width: 430, height: 932) + + XCTAssertTrue( + Self.isTopNavigationControlFrame( + CGRect(x: 340, y: 84, width: 72, height: 44), + in: window + ) + ) + XCTAssertFalse( + Self.isTopNavigationControlFrame( + CGRect(x: 20, y: 760, width: 72, height: 44), + in: window + ) + ) + XCTAssertFalse(Self.isTopNavigationControlFrame(.infinite, in: window)) + } + + func testNavigationVisualVerificationSeparatesNoChangeFromNoSample() { + XCTAssertEqual( + Self.navigationVisualObservation(before: Data([1, 2, 3]), after: Data([1, 2, 4])), + .changed + ) + XCTAssertEqual( + Self.navigationVisualObservation(before: Data([1, 2, 3]), after: Data([1, 2, 3])), + .unchanged + ) + // A missing sample is neither a change nor a no-change; treating it as "unchanged" would let a + // capture that refused become the reason the `back` command claims no control exists (#2728). + XCTAssertEqual(Self.navigationVisualObservation(before: nil, after: Data([1])), .unobserved) + XCTAssertEqual(Self.navigationVisualObservation(before: Data([1]), after: nil), .unobserved) + XCTAssertEqual(Self.navigationVisualObservation(before: nil, after: nil), .unobserved) + } + + func testNavigationFallbackReportsTheRefusalItHitNotADefaultCode() { + // The refusal from the most recent sample wins, so the code names what the fallback last looked at + // before giving up; an earlier refusal is reported only when the later sample carried none (#2728). + let after = NavigationVisualSample( + data: nil, + refusalCode: "APP_SCREEN_WINDOW_UNRESOLVED", + refusalHint: "after hint" + ) + let before = NavigationVisualSample( + data: nil, + refusalCode: "APP_SCREEN_UNRESOLVED", + refusalHint: "before hint" + ) + let laterWins = Self.navigationFallbackErrorPayload(after: after, before: before) + XCTAssertEqual(laterWins.code, "APP_SCREEN_WINDOW_UNRESOLVED") + XCTAssertEqual(laterWins.hint, "after hint") + + let onlyBefore = Self.navigationFallbackErrorPayload( + after: NavigationVisualSample(data: nil), + before: before + ) + XCTAssertEqual(onlyBefore.code, "APP_SCREEN_UNRESOLVED") + XCTAssertEqual(onlyBefore.hint, "before hint") + + // Neither side named a reason (unreachable on iOS): a real capture code, never a bare failure. + let unnamed = Self.navigationFallbackErrorPayload( + after: NavigationVisualSample(data: nil), + before: NavigationVisualSample(data: nil) + ) + XCTAssertEqual(unnamed.code, "APP_SCREEN_UNRESOLVED") + XCTAssertTrue(unnamed.message.contains("unknown outcome")) + } + + func testVerifyNavigationFallbackOutcomeReportsUnresolvedWindowWithoutSystemSurface() { + // The in-app `back` fallback ran its tap but the app resolved no window. It must name that refusal + // as an unknown outcome AND must not have sampled the system surface: capturing SpringBoard's home + // screen twice reads as "unchanged" and launders a wrong-process frame into "no back control + // exists" (#2728). This drives the SAME `navigationFallbackSample` production calls, handing it a + // system surface that fails the test if consulted — so reverting the fallback to sample SpringBoard + // (or dropping the refusal code) turns this red, which an inline `.never` re-creation could not. + var askedSystemSurface = false + let sample = Self.navigationFallbackSample( + resolvingApp: { .failure(.unresolvedWindow) }, + systemSurface: { + askedSystemSurface = true + return .failure(.unresolvedWindow) + }, + encoding: { _ in Data([1, 2, 3]) } + ) + XCTAssertFalse(askedSystemSurface, "the in-app fallback samples the app only, never SpringBoard") + + XCTAssertNil(sample.data) + XCTAssertEqual(sample.refusalCode, "APP_SCREEN_WINDOW_UNRESOLVED") + + let observation = Self.navigationVisualObservation(before: sample.data, after: sample.data) + XCTAssertEqual(observation, .unobserved) + + switch Self.inAppBackOutcome(observation: observation, before: sample, after: sample) { + case .unverified(let payload): + XCTAssertEqual(payload.code, "APP_SCREEN_WINDOW_UNRESOLVED") + case .performed, .unavailable: + XCTFail("a refused capture must report an unknown outcome, not 'no back control'") + } + } + + func testNavigationVisualSampleDistinguishesEncodedFrameFromRefusal() { + // The same capture entry point yields three different samples, and only the refusal ones may carry + // a code: an encoded frame is evidence, a resolved-but-unencodable image and a refusal are not + // (#2728). Reverting the mapping to a plain no-sample loses the reason a host keys on. + let captured = CapturedAppScreen( + image: RunnerImage(), + displayID: 3, + pixelWidth: 12, + pixelHeight: 24, + pixelsPerPoint: 3 + ) + + let encoded = Self.navigationVisualSample( + from: .success(captured), + encoding: { _ in Data([7, 7]) } + ) + XCTAssertEqual(encoded.data, Data([7, 7])) + XCTAssertNil(encoded.refusalCode) + + let unencodable = Self.navigationVisualSample( + from: .success(captured), + encoding: { _ in nil } + ) + XCTAssertNil(unencodable.data) + XCTAssertEqual(unencodable.refusalCode, "APP_SCREEN_CAPTURE_UNRENDERABLE") + + let refused = Self.navigationVisualSample( + from: .failure(.unresolvedScreen), + encoding: { _ in Data([7, 7]) } + ) + XCTAssertNil(refused.data) + XCTAssertEqual(refused.refusalCode, "APP_SCREEN_UNRESOLVED") + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+PrivateAXPresentationTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+PrivateAXPresentationTests.swift new file mode 100644 index 0000000000..3dcd2908dd --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+PrivateAXPresentationTests.swift @@ -0,0 +1,213 @@ +import XCTest +import AgentDeviceSnapshotPresentation + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +extension RunnerTests { + fileprivate static func privateAXFrame( + _ x: Double, _ y: Double, _ width: Double, _ height: Double + ) -> [String: Any] { + ["x": x, "y": y, "width": width, "height": height] + } + + /// A scroll container whose second row is scrolled out of the viewport, plus an unlabeled + /// decoration: the shapes the regular projection folds away and the raw projection must keep. + fileprivate static var privateAXScrolledFixture: [String: Any] { + let frame = privateAXFrame + return ["type": Int(XCUIElement.ElementType.application.rawValue), + "label": "Element", "frame": frame(0, 0, 402, 874), "children": [[ + "type": Int(XCUIElement.ElementType.scrollView.rawValue), "frame": frame(0, 96, 402, 700), + "actions": ["Scroll down"], + "children": [ + ["type": Int(XCUIElement.ElementType.button.rawValue), "label": "Profile picture", + "frame": frame(16, 120, 44, 44), + "children": [["type": Int(XCUIElement.ElementType.image.rawValue), + "frame": frame(16, 120, 1, 1)]]], + ["type": Int(XCUIElement.ElementType.button.rawValue), "label": "Theme", + "frame": frame(16, 900, 360, 44)]]]]] + } + + /// Acquire with the private-AX serializer, run the one normalization pass the production capture + /// plan runs (`captureWithBackend`), then present through the shared regular fold -- the production + /// route for this backend since the fold moved into presentation (#1797, #2661). + fileprivate func privateAXRegularPresentation( + rawRoot: [String: Any], + viewport: CGRect, + interactiveOnly: Bool = false + ) throws -> [PresentedNode] { + let hint = CaptureHint( + projection: .regular, depth: nil, regularPresentedDepth: nil, + interactiveOnly: interactiveOnly, customActions: false) + let nodes = privateAXNormalizedAcquisition( + rawRoot: rawRoot, hint: hint, viewport: viewport, + interfaceOrientation: RunnerInterfaceOrientation.portrait) + return try SnapshotPresentation.presentRegular( + SnapshotAcquisition( + hint: hint, nodes: nodes, truncated: false, effectiveDepth: nil, + viewport: .reported( + box: viewport, + interfaceOrientation: RunnerInterfaceOrientation.portrait + )), + options: PresentationOptions( + interactiveOnly: interactiveOnly, depth: nil, scope: nil, raw: false), + policy: .cursorProjected + ).nodes + } + + /// Acquire, then normalize once -- the exact pair `captureWithBackend` runs for this backend. + fileprivate func privateAXNormalizedAcquisition( + rawRoot: [String: Any], + hint: CaptureHint, + viewport: CGRect, + interfaceOrientation: Int + ) -> [RawAXNode] { + SnapshotGeometrySpace.normalized( + nodes: privateAXAcquisition(rawRoot: rawRoot, hint: hint), + viewport: .reported(box: viewport, interfaceOrientation: interfaceOrientation) + ) + } + + /// The one normalization pass has to reach the node it publishes: this asserts the rotated rect of + /// a key under a turned surface host, and an untouched sibling under the app's own window, so a + /// pass that drops the space a subtree declared fails here. + func testPrivateAXAcquisitionPublishesATurnedSurfaceHostInAppOrientationSpace() { + let frame = Self.privateAXFrame + let appWindow: [String: Any] = [ + "type": Int(XCUIElement.ElementType.window.rawValue), + "frame": frame(0, 0, 874, 402), + "children": [ + ["type": Int(XCUIElement.ElementType.button.rawValue), "label": "Home", + "frame": frame(204, 323, 91, 55), "children": []] + ] + ] + let keyboardWindow: [String: Any] = [ + "type": Int(XCUIElement.ElementType.window.rawValue), + "frame": frame(0, 0, 874, 402), + "children": [ + ["type": Int(XCUIElement.ElementType.other.rawValue), + "frame": frame(0, 0, 402, 874), + "children": [ + ["type": Int(XCUIElement.ElementType.key.rawValue), "label": "q", + "frame": frame(154, 77, 45, 72), "children": []] + ]] + ] + ] + let hint = CaptureHint( + projection: .raw, depth: nil, regularPresentedDepth: nil, + interactiveOnly: false, customActions: false) + let nodes = privateAXNormalizedAcquisition( + rawRoot: [ + "type": Int(XCUIElement.ElementType.application.rawValue), + "label": "Element", "frame": frame(0, 0, 874, 402), + "children": [appWindow, keyboardWindow] + ], + hint: hint, + viewport: CGRect(x: 0, y: 0, width: 874, height: 402), + interfaceOrientation: RunnerInterfaceOrientation.landscapeRight + ) + + // Measured on iPhone 17 Pro (26.2): the key plane's left column arrives 154 pt along the device's + // long axis and comes back 203 pt down the app's short one. + XCTAssertEqual( + nodes.first { $0.label == "q" }?.rect, + SnapshotRect(x: 77, y: 203, width: 72, height: 45) + ) + XCTAssertEqual( + nodes.first { $0.label == "Home" }?.rect, + SnapshotRect(x: 204, y: 323, width: 91, height: 55) + ) + } + + func testPrivateAXRegularPresentationProjectsToViewportAndKeepsScrollHint() throws { + let nodes = try privateAXRegularPresentation( + rawRoot: Self.privateAXScrolledFixture, + viewport: CGRect(x: 0, y: 0, width: 402, height: 874)) + XCTAssertEqual(nodes.compactMap(\.label), ["Element", "Profile picture"]) + let scrollView = nodes.first { $0.type == "ScrollView" } + XCTAssertEqual(scrollView?.hiddenContentBelow, true) + XCTAssertEqual(scrollView?.actions, ["Scroll down"]) + } + + /// #1797 D4: the raw projection is the acquired tree. The offscreen row and the sub-pixel + /// decoration the regular projection folds away are both present, at traversal depth, and every + /// regular node still appears -- `regular ⊆ raw` on the same capture. + func testPrivateAXRawProjectionKeepsEveryAcquiredNode() throws { + let viewport = CGRect(x: 0, y: 0, width: 402, height: 874) + let root = Self.privateAXScrolledFixture + let regular = try privateAXRegularPresentation(rawRoot: root, viewport: viewport, + interactiveOnly: true) + let raw = privateAXNormalizedAcquisition(rawRoot: root, + hint: CaptureHint( + projection: .raw, depth: nil, regularPresentedDepth: nil, + interactiveOnly: false, customActions: false), + viewport: viewport, + interfaceOrientation: RunnerInterfaceOrientation.portrait) + + XCTAssertEqual(raw.map(\.type), ["Application", "ScrollView", "Button", "Image", "Button"]) + XCTAssertEqual(raw.map(\.depth), [0, 1, 2, 3, 2]) + XCTAssertEqual(raw.map(\.parentIndex), [nil, 0, 1, 2, 1]) + XCTAssertEqual(raw.compactMap(\.label), ["Element", "Profile picture", "Theme"]) + // The offscreen row is a reported fact in raw, and reported facts do not become hittable + // just because the projection kept them. + XCTAssertEqual(raw.last?.hittable, false) + XCTAssertTrue(raw.allSatisfy { $0.hiddenContentAbove == nil && $0.hiddenContentBelow == nil }) + + let rawKeys = Set(raw.map { "\($0.type)-\($0.label ?? "")-\($0.rect.y)" }) + for node in regular { + XCTAssertTrue( + rawKeys.contains("\(node.type)-\(node.label ?? "")-\(node.rect.y)"), + "regular node \(node.type)/\(node.label ?? "") is missing from the raw projection" + ) + } + XCTAssertGreaterThan(raw.count, regular.count) + } + + /// Raw depth is traversal depth, so a raw `--depth` request is the one narrowing this backend + /// can prove complete. + func testPrivateAXRawProjectionAppliesRequestedTraversalDepth() { + let raw = privateAXNormalizedAcquisition(rawRoot: Self.privateAXScrolledFixture, + hint: CaptureHint( + projection: .raw, depth: 2, regularPresentedDepth: nil, + interactiveOnly: false, customActions: false), + viewport: CGRect(x: 0, y: 0, width: 402, height: 874), + interfaceOrientation: RunnerInterfaceOrientation.portrait) + XCTAssertEqual(raw.map(\.type), ["Application", "ScrollView", "Button", "Button"]) + XCTAssertEqual(raw.map(\.depth), [0, 1, 2, 2]) + } + + func testPrivateAXPresentationKeepsOffscreenSubtreeExcludedWhenChildFramesAreClamped() throws { + let frame = Self.privateAXFrame + let root: [String: Any] = ["type": Int(XCUIElement.ElementType.application.rawValue), + "label": "Element", "frame": frame(0, 0, 402, 874), "children": [[ + "type": Int(XCUIElement.ElementType.table.rawValue), "frame": frame(0, 96, 402, 700), + "children": [["type": Int(XCUIElement.ElementType.cell.rawValue), + "label": "Theme", "frame": frame(0, 900, 402, 44), "children": [[ + "type": Int(XCUIElement.ElementType.staticText.rawValue), + "label": "Theme", "frame": frame(16, 96, 120, 44)], + ["type": Int(XCUIElement.ElementType.switch.rawValue), + "label": "Theme", "frame": frame(340, 96, 46, 44)]]]]]]] + + let nodes = try privateAXRegularPresentation( + rawRoot: root, viewport: CGRect(x: 0, y: 0, width: 402, height: 874)) + + XCTAssertEqual(nodes.compactMap(\.label), ["Element"]) + XCTAssertEqual(nodes.first { $0.type == "Table" }?.hiddenContentBelow, true) + } + + func testPrivateAXGeometrylessSemanticsAreNeverActionableOrScrollContexts() throws { + let zero = ["x": 0, "y": 0, "width": 0, "height": 0] + let root: [String: Any] = ["type": Int(XCUIElement.ElementType.application.rawValue), + "label": "Element", "frame": ["x": 0, "y": 0, "width": 402, "height": 874], + "children": [["type": Int(XCUIElement.ElementType.scrollView.rawValue), + "label": "Settings semantics", "frame": zero, "children": [[ + "type": Int(XCUIElement.ElementType.button.rawValue), "label": "Theme", "frame": zero], + ["type": Int(XCUIElement.ElementType.other.rawValue), "frame": zero]]]]] + let nodes = try privateAXRegularPresentation( + rawRoot: root, viewport: CGRect(x: 0, y: 0, width: 402, height: 874), + interactiveOnly: true) + XCTAssertEqual(nodes.compactMap(\.label), ["Element", "Settings semantics", "Theme"]) + XCTAssertEqual(nodes.filter { $0.index != 0 }.map(\.hittable), [false, false]) + XCTAssertFalse(nodes.contains { $0.type == "Other" }) + XCTAssertTrue(nodes.allSatisfy { $0.hiddenContentAbove == nil && $0.hiddenContentBelow == nil }) + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+RecordingTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+RecordingTests.swift index 8875716a6e..0c97926342 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+RecordingTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+RecordingTests.swift @@ -1,4 +1,9 @@ import XCTest +#if canImport(UIKit) +import UIKit +#elseif canImport(AppKit) +import AppKit +#endif extension RunnerTests { #if AGENT_DEVICE_RUNNER_UNIT_TESTS @@ -23,5 +28,501 @@ extension RunnerTests { // The helper records each returned value as an accepted frame. XCTAssertEqual(recorder.allocateTimestampForTesting(0), 101) } + + func testRecordingBootstrapErrorKeepsTheTypedRefusalAndFallsBackGenerically() { + // A capture that refused is the honest bootstrap error; nothing refusing keeps the pre-panel + // untyped record error, so the iOS typed path and the macOS generic path are both pinned (#2728). + let typed = RunnerTests.recordingBootstrapError(from: .unresolvedWindow) + XCTAssertEqual( + (typed as? RunnerAppScreenCaptureFailure)?.rawValue, + "APP_SCREEN_WINDOW_UNRESOLVED" + ) + + let generic = RunnerTests.recordingBootstrapError(from: nil) + XCTAssertNil(generic as? RunnerAppScreenCaptureFailure) + XCTAssertEqual((generic as NSError).code, 1) + } + + func testRecordingStartSurfacesACaptureRefusalAsATypedCode() { + // The bootstrap frame is required, so a no-window refusal must travel as its own code rather than + // the generic record error it used to collapse into; a real writer failure keeps its message (#2728). + let refusal = RunnerTests.recordingStartErrorPayload( + for: RunnerAppScreenCaptureFailure.unresolvedWindow + ) + XCTAssertEqual(refusal.code, "APP_SCREEN_WINDOW_UNRESOLVED") + XCTAssertNotNil(refusal.hint) + + let writerFailure = NSError( + domain: "AgentDeviceRunner.Record", + code: 5, + userInfo: [NSLocalizedDescriptionKey: "failed to append frame"] + ) + let generic = RunnerTests.recordingStartErrorPayload(for: writerFailure) + XCTAssertNil(generic.code) + XCTAssertTrue(generic.message.contains("failed to start recording")) + XCTAssertTrue(generic.message.contains("failed to append frame")) + } #endif } + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS) +extension RunnerTests { + func testRecordStartThrowsTheCaptureRefusalItReceived() throws { + // The bootstrap frame is required, so `record start` must surface the exact refusal its capture + // saw rather than the generic "failed to capture initial frame" every refusal used to collapse + // into, and that refusal must reach the host as its own code. Driving `start` with an always- + // refusing capture and mapping the ACTUAL thrown error — not a re-typed literal — proves the + // bootstrap forwards its last refusal end to end; the pure mapping tests cannot catch that wiring. + let outputPath = (NSTemporaryDirectory() as NSString).appendingPathComponent( + "record-refusal-\(UUID().uuidString).mp4" + ) + let recorder = ScreenRecorder(outputPath: outputPath, fps: 30) + + var captureCalls = 0 + var thrown: Error? + XCTAssertThrowsError( + try recorder.start( + bootstrap: { + captureCalls += 1 + return .failure(.unresolvedWindow) + }, + frame: { nil } + ) + ) { error in + thrown = error + XCTAssertEqual( + (error as? RunnerAppScreenCaptureFailure)?.rawValue, + "APP_SCREEN_WINDOW_UNRESOLVED", + "the thrown bootstrap error is the refusal the capture returned" + ) + } + XCTAssertGreaterThan(captureCalls, 0, "the bootstrap must poll the injected capture") + + // The refusal the bootstrap actually threw maps to its own host code, not the generic record error. + let payload = RunnerTests.recordingStartErrorPayload(for: try XCTUnwrap(thrown)) + XCTAssertEqual(payload.code, "APP_SCREEN_WINDOW_UNRESOLVED") + } +} +#endif + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +/// Frames for a recording test. Every capture draws a new image, so an appended frame is identified +/// by instance, and holds the calling thread for `captureDelay` like a screenshot round trip. An +/// armed wedge makes the next capture block the main thread until released, and remembers the image +/// that capture will hand back late. +private final class RecordingFrameSource { + static let side = 128 + private let captureDelay: TimeInterval + private let lock = NSLock() + private var wedgeArmed = false + private var wedgedImage: RunnerImage? + let wedgeEntered = DispatchSemaphore(value: 0) + let releaseWedge = DispatchSemaphore(value: 0) + + init(captureDelay: TimeInterval = 0) { + self.captureDelay = captureDelay + } + + var lateImage: RunnerImage? { + lock.lock() + defer { lock.unlock() } + return wedgedImage + } + + func armWedge() { + lock.lock() + wedgeArmed = true + lock.unlock() + } + + func capture() -> Result { + if captureDelay > 0 { + Thread.sleep(forTimeInterval: captureDelay) + } + let image = Self.makeImage() + lock.lock() + let wedge = wedgeArmed + wedgeArmed = false + if wedge { + wedgedImage = image + } + lock.unlock() + if wedge { + wedgeEntered.signal() + _ = releaseWedge.wait(timeout: .now() + 10) + } + return .success( + CapturedAppScreen( + image: image, + displayID: 0, + pixelWidth: Self.side, + pixelHeight: Self.side, + pixelsPerPoint: 1 + ) + ) + } + + private static func makeImage() -> RunnerImage { + let context = CGContext( + data: nil, + width: side, + height: side, + bitsPerComponent: 8, + bytesPerRow: 0, + space: CGColorSpaceCreateDeviceRGB(), + bitmapInfo: CGImageAlphaInfo.noneSkipFirst.rawValue + )! + context.setFillColor(red: 0, green: 0.4, blue: 1, alpha: 1) + context.fill(CGRect(x: 0, y: 0, width: side, height: side)) + let cgImage = context.makeImage()! + #if canImport(UIKit) + return UIImage(cgImage: cgImage) + #else + return NSImage(cgImage: cgImage, size: NSSize(width: side, height: side)) + #endif + } +} + +/// Samples, off main, the readers that commands consult for main-thread occupancy until stopped. +private final class MainThreadOccupancySampler { + private let lock = NSLock() + private var stopped = false + private let done = DispatchSemaphore(value: 0) + private(set) var sampled = false + private(set) var sawBusy = false + private(set) var sawAbandoned = false + private(set) var sawXCTestTierSkip = false + + init(runner: RunnerTests) { + DispatchQueue(label: "agent-device.runner.tests.occupancy-sampler").async { + defer { self.done.signal() } + while !self.isStopped { + let busy = runner.currentMainThreadBusyState().reportsMainThreadBusy + let abandoned = runner.hasAbandonedMainThreadWork() + let tierSkip = runner.shouldSkipSnapshotBackendForAbandonedMainThreadWork(.recursiveTree) + self.lock.lock() + self.sampled = true + self.sawBusy = self.sawBusy || busy + self.sawAbandoned = self.sawAbandoned || abandoned + self.sawXCTestTierSkip = self.sawXCTestTierSkip || tierSkip + self.lock.unlock() + usleep(2_000) + } + } + } + + private var isStopped: Bool { + lock.lock() + defer { lock.unlock() } + return stopped + } + + func stop() { + lock.lock() + stopped = true + lock.unlock() + _ = done.wait(timeout: .now() + 2) + } +} + +extension RunnerTests { + func testRecordingFrameTimeoutDropsTheFrameAndResumesOnceTheWorkDrains() throws { + let source = RecordingFrameSource() + let recorder = ScreenRecorder(outputPath: recordingTestOutputPath(), fps: 10) + try startRecording(recorder, capture: source.capture) + defer { try? recorder.stop() } + XCTAssertTrue(pumpMainThread(until: { recorder.appendedFrameSnapshotForTesting().count >= 3 })) + + final class Observation { + var abandonedWhileWedged: Int? + var framesWhileWedged: Int? + var secondsToAbandon: TimeInterval? + } + let observation = Observation() + let observed = expectation(description: "the wedged frame was abandoned and released") + source.armWedge() + DispatchQueue(label: "agent-device.runner.tests.recording-timeout").async { + defer { + source.releaseWedge.signal() + observed.fulfill() + } + guard source.wedgeEntered.wait(timeout: .now() + 3) == .success else { return } + let enteredAt = Date() + guard self.waitOffMain(until: { self.hasAbandonedMainThreadWork() }) else { return } + observation.secondsToAbandon = Date().timeIntervalSince(enteredAt) + self.mainThreadWorkLock.lock() + observation.abandonedWhileWedged = self.abandonedMainThreadWorkCount + self.mainThreadWorkLock.unlock() + observation.framesWhileWedged = recorder.appendedFrameSnapshotForTesting().count + } + wait(for: [observed], timeout: 15) + + let framesAtRelease = observation.framesWhileWedged ?? 0 + XCTAssertTrue( + pumpMainThread(until: { + !self.hasAbandonedMainThreadWork() + && recorder.appendedFrameSnapshotForTesting().count >= framesAtRelease + 2 + }), + "the abandoned capture drains and the recorder appends frames again" + ) + XCTAssertEqual(observation.abandonedWhileWedged, 1, "the timed-out frame counts as abandoned") + XCTAssertLessThan( + observation.secondsToAbandon ?? .infinity, + recordingFrameCaptureTimeout + 1, + "a frame is bounded by the recording capture timeout, not the command watchdog" + ) + let lateImage = try XCTUnwrap(source.lateImage) + XCTAssertFalse( + recorder.appendedFrameSnapshotForTesting().contains { $0 === lateImage }, + "the late result of the abandoned capture is never appended" + ) + } + + func testRecordingPersistentWedgeKeepsOneCaptureQueuedOnMain() throws { + let source = RecordingFrameSource() + let recorder = ScreenRecorder(outputPath: recordingTestOutputPath(), fps: 20) + try startRecording(recorder, capture: source.capture) + defer { try? recorder.stop() } + XCTAssertTrue(pumpMainThread(until: { recorder.appendedFrameSnapshotForTesting().count >= 2 })) + + final class Observation { + var maxAbandoned = 0 + var maxInFlight = 0 + } + let observation = Observation() + let observed = expectation(description: "the wedge was held for many ticks") + source.armWedge() + DispatchQueue(label: "agent-device.runner.tests.recording-wedge").async { + defer { + source.releaseWedge.signal() + observed.fulfill() + } + guard source.wedgeEntered.wait(timeout: .now() + 3) == .success else { return } + let holdUntil = Date().addingTimeInterval(self.recordingFrameCaptureTimeout * 2.5) + while Date() < holdUntil { + self.mainThreadWorkLock.lock() + observation.maxAbandoned = max(observation.maxAbandoned, self.abandonedMainThreadWorkCount) + observation.maxInFlight = max(observation.maxInFlight, self.mainThreadWorkInFlightCount) + self.mainThreadWorkLock.unlock() + usleep(10_000) + } + } + wait(for: [observed], timeout: 15) + XCTAssertTrue(pumpMainThread(until: { !self.hasAbandonedMainThreadWork() })) + + XCTAssertEqual( + observation.maxAbandoned, + 1, + "every tick against a wedged main thread leaves one recorder capture pending, not one per tick" + ) + XCTAssertEqual(observation.maxInFlight, 1, "skipped ticks dispatch nothing to main") + } + + func testRecordingStopDuringATimedOutCaptureAppendsNoLateFrame() throws { + let source = RecordingFrameSource() + let recorder = ScreenRecorder(outputPath: recordingTestOutputPath(), fps: 10) + try startRecording(recorder, capture: source.capture) + XCTAssertTrue(pumpMainThread(until: { recorder.appendedFrameSnapshotForTesting().count >= 2 })) + + final class Observation { + var stopError: Error? + var stopReturnedWhileWedged = false + var framesAtStop: Int? + } + let observation = Observation() + let observed = expectation(description: "stop returned during the timed-out capture") + source.armWedge() + DispatchQueue(label: "agent-device.runner.tests.recording-stop").async { + defer { + source.releaseWedge.signal() + observed.fulfill() + } + guard source.wedgeEntered.wait(timeout: .now() + 3) == .success else { return } + guard self.waitOffMain(until: { self.hasAbandonedMainThreadWork() }) else { return } + do { + try recorder.stop() + } catch { + observation.stopError = error + } + observation.stopReturnedWhileWedged = self.hasAbandonedMainThreadWork() + observation.framesAtStop = recorder.appendedFrameSnapshotForTesting().count + } + wait(for: [observed], timeout: 20) + XCTAssertTrue(pumpMainThread(until: { !self.hasAbandonedMainThreadWork() })) + sleepFor(0.3) + + XCTAssertNil(observation.stopError) + XCTAssertTrue(observation.stopReturnedWhileWedged, "stop must not wait for the wedged capture") + XCTAssertEqual(recorder.appendedFrameSnapshotForTesting().count, observation.framesAtStop) + let lateImage = try XCTUnwrap(source.lateImage) + XCTAssertFalse(recorder.appendedFrameSnapshotForTesting().contains { $0 === lateImage }) + } + + func testRecordingStopRefusesAFrameThatFinishedAtTheTimeoutBoundary() throws { + let source = RecordingFrameSource() + let recorder = ScreenRecorder(outputPath: recordingTestOutputPath(), fps: 10) + try startRecording(recorder, capture: source.capture) + XCTAssertTrue(pumpMainThread(until: { recorder.appendedFrameSnapshotForTesting().count >= 2 })) + + final class Observation { + var framesAtStop: Int? + } + let observation = Observation() + mainThreadWorkTimedOutForTesting = { + try? recorder.stop() + observation.framesAtStop = recorder.appendedFrameSnapshotForTesting().count + source.releaseWedge.signal() + DispatchQueue.main.sync {} + } + defer { mainThreadWorkTimedOutForTesting = nil } + source.armWedge() + XCTAssertTrue(pumpMainThread(until: { observation.framesAtStop != nil })) + sleepFor(0.3) + + XCTAssertFalse(hasAbandonedMainThreadWork(), "a capture finished at the boundary is not abandoned") + let lateImage = try XCTUnwrap(source.lateImage) + XCTAssertFalse( + recorder.appendedFrameSnapshotForTesting().contains { $0 === lateImage }, + "a frame returned after stop is never appended" + ) + XCTAssertEqual(recorder.appendedFrameSnapshotForTesting().count, observation.framesAtStop) + } + + func testRecordingAfterAStopDuringATimedOutCaptureStartsClean() throws { + let first = RecordingFrameSource() + let firstRecorder = ScreenRecorder(outputPath: recordingTestOutputPath(), fps: 10) + try startRecording(firstRecorder, capture: first.capture) + XCTAssertTrue(pumpMainThread(until: { firstRecorder.appendedFrameSnapshotForTesting().count >= 2 })) + let stopped = expectation(description: "first recording stopped during its timed-out capture") + first.armWedge() + DispatchQueue(label: "agent-device.runner.tests.recording-restart").async { + defer { + first.releaseWedge.signal() + stopped.fulfill() + } + guard first.wedgeEntered.wait(timeout: .now() + 3) == .success else { return } + guard self.waitOffMain(until: { self.hasAbandonedMainThreadWork() }) else { return } + try? firstRecorder.stop() + } + wait(for: [stopped], timeout: 20) + XCTAssertTrue(pumpMainThread(until: { !self.hasAbandonedMainThreadWork() })) + let firstFrames = firstRecorder.appendedFrameSnapshotForTesting() + + let second = RecordingFrameSource() + let secondOutputPath = recordingTestOutputPath() + let secondRecorder = ScreenRecorder(outputPath: secondOutputPath, fps: 10) + try startRecording(secondRecorder, capture: second.capture) + XCTAssertTrue(pumpMainThread(until: { secondRecorder.appendedFrameSnapshotForTesting().count >= 3 })) + try secondRecorder.stop() + + let secondFrames = secondRecorder.appendedFrameSnapshotForTesting() + XCTAssertFalse(secondFrames.contains { frame in firstFrames.contains { $0 === frame } }) + let lateImage = try XCTUnwrap(first.lateImage) + XCTAssertFalse(secondFrames.contains { $0 === lateImage }) + XCTAssertFalse(firstRecorder.appendedFrameSnapshotForTesting().contains { $0 === lateImage }) + XCTAssertEqual(firstRecorder.appendedFrameSnapshotForTesting().count, firstFrames.count) + XCTAssertFalse(hasAbandonedMainThreadWork()) + let attributes = try FileManager.default.attributesOfItem(atPath: secondOutputPath) + XCTAssertGreaterThan((attributes[.size] as? NSNumber)?.intValue ?? 0, 0) + } + + func testRecordingFrameNeverQueuesBehindACommandsMainThreadWork() throws { + let source = RecordingFrameSource() + let recorder = ScreenRecorder(outputPath: recordingTestOutputPath(), fps: 60) + try startRecording(recorder, capture: source.capture) + defer { try? recorder.stop() } + XCTAssertTrue(pumpMainThread(until: { recorder.appendedFrameSnapshotForTesting().count >= 3 })) + + let occupancy = MainThreadOccupancySampler(runner: self) + let commandFinished = expectation(description: "the command's main-thread hops finished") + final class Outcome { + var error: Error? + } + let outcome = Outcome() + DispatchQueue(label: "agent-device.runner.tests.recording-command").async { + defer { + occupancy.stop() + commandFinished.fulfill() + } + do { + for _ in 0..<2 { + try self.runMainThreadWork( + "command_execution", + timeout: self.mainThreadExecutionTimeout, + timeoutError: self.mainThreadExecutionTimeoutError + ) { + Thread.sleep(forTimeInterval: self.recordingFrameCaptureTimeout + 0.3) + } + } + } catch { + outcome.error = error + } + } + wait(for: [commandFinished], timeout: 15) + let framesAfterCommand = recorder.appendedFrameSnapshotForTesting().count + XCTAssertTrue( + pumpMainThread(until: { recorder.appendedFrameSnapshotForTesting().count >= framesAfterCommand + 2 }), + "the recorder appends frames again once the command leaves main" + ) + + XCTAssertNil(outcome.error) + XCTAssertTrue(occupancy.sampled, "the sampler ran while the command held main") + XCTAssertFalse(occupancy.sawBusy, "a recorder tick must not make the busy gate or stamp see occupancy") + XCTAssertFalse(occupancy.sawAbandoned, "a recorder tick must not mark work abandoned") + XCTAssertFalse(occupancy.sawXCTestTierSkip, "a recorder tick must not skip XCTest snapshot tiers") + guard case .idle = currentMainThreadBusyState() else { + return XCTFail("expected the runner idle after the command") + } + } + + func testRecordingFrameSlowerThanTheIntervalStillYieldsFrames() throws { + let fps: Int32 = 20 + let interval = 1.0 / Double(fps) + let source = RecordingFrameSource(captureDelay: interval * 1.6) + let recorder = ScreenRecorder(outputPath: recordingTestOutputPath(), fps: fps) + try startRecording(recorder, capture: source.capture) + defer { try? recorder.stop() } + XCTAssertTrue(pumpMainThread(until: { recorder.appendedFrameSnapshotForTesting().count >= 2 })) + + let window: TimeInterval = 1.5 + let occupancy = MainThreadOccupancySampler(runner: self) + let framesBefore = recorder.appendedFrameSnapshotForTesting().count + sleepFor(window) + occupancy.stop() + let yielded = recorder.appendedFrameSnapshotForTesting().count - framesBefore + + XCTAssertGreaterThanOrEqual( + yielded, + Int(window / (interval * 1.6) / 3), + "a capture slower than the interval lowers the frame rate instead of dropping every frame" + ) + XCTAssertTrue(occupancy.sampled) + XCTAssertFalse(occupancy.sawAbandoned, "an ordinary slow capture is not abandoned work") + XCTAssertFalse(occupancy.sawBusy, "an ordinary slow capture keeps the runner available") + } + + private func recordingTestOutputPath() -> String { + (NSTemporaryDirectory() as NSString).appendingPathComponent( + "record-bounded-\(UUID().uuidString).mp4" + ) + } + + private func pumpMainThread(timeout: TimeInterval = 5, until condition: () -> Bool) -> Bool { + let deadline = Date().addingTimeInterval(timeout) + while !condition() { + if Date() >= deadline { return false } + sleepFor(0.01) + } + return true + } + + private func waitOffMain(timeout: TimeInterval = 5, until condition: () -> Bool) -> Bool { + let deadline = Date().addingTimeInterval(timeout) + while !condition() { + if Date() >= deadline { return false } + usleep(5_000) + } + return true + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScrollDragExecutionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScrollDragExecutionTests.swift new file mode 100644 index 0000000000..440f19fe90 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScrollDragExecutionTests.swift @@ -0,0 +1,72 @@ +import XCTest +import AgentDeviceSnapshotPresentation + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS) +import ObjectiveC.runtime + +private final class RunnerSynthesizedSwipeFailureStub: NSObject { + @objc(synthesizeSwipeWithApplication:resolvedWindow:x:y:x2:y2:durationMs:) + class func synthesizeSwipe( + application: XCUIApplication, + resolvedWindow: Any?, + x: Double, + y: Double, + x2: Double, + y2: Double, + durationMs: Double + ) -> String? { + "forced private synthesis failure" + } +} +#endif + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +extension RunnerTests { +#if os(iOS) + func testSinglePointerFlingFallsBackToXCTestCoordinateDragWhenPrivateSynthesisFails() throws { + let selector = NSSelectorFromString( + "synthesizeSwipeWithApplication:resolvedWindow:x:y:x2:y2:durationMs:" + ) + guard + let synthesizedSwipeMethod = class_getClassMethod(RunnerSynthesizedGesture.self, selector), + let failureStubMethod = class_getClassMethod(RunnerSynthesizedSwipeFailureStub.self, selector) + else { + XCTFail("unable to install synthesized swipe failure stub") + return + } + let originalImplementation = method_getImplementation(synthesizedSwipeMethod) + method_setImplementation( + synthesizedSwipeMethod, + method_getImplementation(failureStubMethod) + ) + app.launch() + runnerAccessibilityHealth = .healthy + defer { + method_setImplementation(synthesizedSwipeMethod, originalImplementation) + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } + let command = try runnerCommandFixture( + """ + {"command":"gesture","commandId":"gesture-fling-fallback","gesturePlan":{"topology":"single","intent":"fling","executionProfile":"endpoint-hold","durationMs":100,"viewport":{"x":0,"y":0,"width":200,"height":300},"pointers":[{"pointerId":0,"samples":[{"offsetMs":0,"point":{"x":160,"y":150}},{"offsetMs":100,"point":{"x":40,"y":150}}]}]}} + """ + ) + + let response = try executeOnMainPrepared(command: command, activeApp: app) + + XCTAssertTrue(response.ok) + XCTAssertEqual(response.data?.message, "fling") + XCTAssertEqual(response.data?.gestureFallback, "xctest-coordinate-drag") + XCTAssertEqual(response.data?.gestureFallbackMessage, "forced private synthesis failure") + XCTAssertEqual( + response.data?.gestureFallbackHint, + "Private XCTest event synthesis is required for AX-free coordinate drag on iOS; update Xcode if this persists." + ) + XCTAssertNil(response.data?.x) + XCTAssertNil(response.data?.y) + XCTAssertNil(response.data?.x2) + XCTAssertNil(response.data?.y2) + } +#endif +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScrollGestureTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScrollGestureTests.swift new file mode 100644 index 0000000000..9e9b9d26fc --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScrollGestureTests.swift @@ -0,0 +1,197 @@ +import XCTest + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +private struct ScrollGestureFixture: Decodable { + struct Constants: Decodable { + let defaultIosScrollAmount: Double + let defaultMobileScrollDurationMs: Double + let defaultIosScrollDurationMs: Double + let defaultScrollAmount: Double + let defaultEdgePaddingFraction: Double + let ordinaryScrollReleaseBehavior: String + let edgeScrollReleaseBehavior: String + } + struct Expected: Decodable { + let x1: Double + let y1: Double + let x2: Double + let y2: Double + let pixels: Double + } + struct Case: Decodable { + let name: String + let direction: String + let amount: Double? + let pixels: Double? + let referenceWidth: Double + let referenceHeight: Double + let expected: Expected + } + + let constants: Constants + let cases: [Case] +} + +extension RunnerTests { + // Cross-language parity table: every case in contracts/fixtures/scroll-gesture.json must agree + // with the vitest twin (packages/contracts/src/scroll-gesture.test.ts). Add vectors there, + // never fork the math. + private func loadScrollGestureFixture() throws -> ScrollGestureFixture { + let fixtureURL = URL(fileURLWithPath: #filePath) + .deletingLastPathComponent() // UnitTests + .deletingLastPathComponent() // AgentDeviceRunnerUITests + .deletingLastPathComponent() // AgentDeviceRunner + .deletingLastPathComponent() // runner + .deletingLastPathComponent() // apple + .deletingLastPathComponent() // repo root + .appendingPathComponent("contracts") + .appendingPathComponent("fixtures") + .appendingPathComponent("scroll-gesture.json") + return try JSONDecoder().decode(ScrollGestureFixture.self, from: Data(contentsOf: fixtureURL)) + } + + func testRunnerScrollGesturePlanMatchesParityTable() throws { + let fixture = try loadScrollGestureFixture() + XCTAssertFalse(fixture.cases.isEmpty, "parity table must not be empty") + for testCase in fixture.cases { + let plan = try XCTUnwrap( + runnerScrollGesturePlan( + direction: try XCTUnwrap(RunnerScrollDirection(rawValue: testCase.direction)), + amount: testCase.amount, + pixels: testCase.pixels, + referenceWidth: testCase.referenceWidth, + referenceHeight: testCase.referenceHeight + ), + testCase.name + ) + XCTAssertEqual(plan.x1, testCase.expected.x1, testCase.name) + XCTAssertEqual(plan.y1, testCase.expected.y1, testCase.name) + XCTAssertEqual(plan.x2, testCase.expected.x2, testCase.name) + XCTAssertEqual(plan.y2, testCase.expected.y2, testCase.name) + XCTAssertEqual(plan.travelPixels, testCase.expected.pixels, testCase.name) + } + } + + // The planner constants are private on both sides; the table pins them behaviourally on a + // 1000px axis where every rounding step is exact. + func testRunnerScrollGesturePlanUsesParityTableConstants() throws { + let constants = try loadScrollGestureFixture().constants + let defaultScroll = try JSONDecoder().decode( + Command.self, from: Data(#"{"command":"scroll"}"#.utf8)) + let defaults = runnerDragCommandDefaults(defaultScroll) + XCTAssertEqual(defaults.durationMs, constants.defaultIosScrollDurationMs) + XCTAssertEqual(defaults.scrollAmount, constants.defaultIosScrollAmount) + let defaulted = try XCTUnwrap( + runnerScrollGesturePlan( + direction: .down, amount: nil, pixels: nil, referenceWidth: 1000, referenceHeight: 1000 + ) + ) + XCTAssertEqual(defaulted.travelPixels, 1000 * constants.defaultScrollAmount) + let saturated = try XCTUnwrap( + runnerScrollGesturePlan( + direction: .down, amount: 10, pixels: nil, referenceWidth: 1000, referenceHeight: 1000 + ) + ) + XCTAssertEqual( + saturated.travelPixels, 1000 - 2 * 1000 * constants.defaultEdgePaddingFraction) + } + + func testRunnerScrollAndDragCommandDefaultsStayDistinct() throws { + func command(_ json: String) throws -> Command { + try JSONDecoder().decode(Command.self, from: Data(json.utf8)) + } + + let pixelScroll = runnerDragCommandDefaults( + try command(#"{"command":"scroll","pixels":120}"#)) + XCTAssertNil(pixelScroll.scrollAmount) + XCTAssertEqual(pixelScroll.durationMs, 400) + + let drag = runnerDragCommandDefaults(try command(#"{"command":"drag"}"#)) + XCTAssertNil(drag.scrollAmount) + XCTAssertEqual(drag.durationMs, 250) + + let explicitScroll = runnerDragCommandDefaults( + try command(#"{"command":"scroll","amount":0.5,"durationMs":125}"#)) + XCTAssertEqual(explicitScroll.scrollAmount, 0.5) + XCTAssertEqual(explicitScroll.durationMs, 125) + + let explicitDrag = runnerDragCommandDefaults( + try command(#"{"command":"drag","durationMs":125}"#)) + XCTAssertEqual(explicitDrag.durationMs, 125) + } + + func testRunnerScrollReleaseBehaviorSelectsTheDragProfile() throws { + let constants = try loadScrollGestureFixture().constants + let controlled = try XCTUnwrap(ScrollReleaseBehavior(rawValue: constants.ordinaryScrollReleaseBehavior)) + let inertial = try XCTUnwrap(ScrollReleaseBehavior(rawValue: constants.edgeScrollReleaseBehavior)) + XCTAssertEqual( + scrollDragProfile(releaseBehavior: nil), + .controlledScroll + ) + XCTAssertEqual( + scrollDragProfile(releaseBehavior: controlled), + .controlledScroll + ) + XCTAssertEqual( + scrollDragProfile(releaseBehavior: inertial), + .fastSwipe + ) + } + + func testControlledScrollProfileUsesReliableCadenceAndMonotonicDeceleration() { + XCTAssertEqual(RunnerControlledScrollFrameCount(350), 21) + XCTAssertEqual(RunnerControlledScrollFrameCount(400), 24) + XCTAssertEqual(RunnerControlledScrollFrameCount(500), 30) + XCTAssertEqual(RunnerControlledScrollFrameCount(1_000), 30) + XCTAssertEqual(RunnerControlledScrollFrameCount(10_000), 30) + + let frameCount = RunnerControlledScrollFrameCount(400) + let progress = (0...frameCount).map { + RunnerControlledScrollProgress(Double($0) / Double(frameCount)) + } + let deltas = zip(progress.dropFirst(), progress).map { $0.0 - $0.1 } + XCTAssertEqual(progress.first, 0) + XCTAssertEqual(progress.last, 1) + XCTAssertTrue(zip(deltas, deltas.dropFirst()).allSatisfy { $0.1 <= $0.0 }) + + let iPhoneViewportPoints = 874.0 + let defaultFingerTravel = iPhoneViewportPoints * 0.65 + let finalSampleTravel = (1 - progress[progress.count - 2]) * defaultFingerTravel + XCTAssertLessThan(finalSampleTravel, 0.1) + } + + func testRunnerScrollGesturePlanRejectsUnknownDirection() { + XCTAssertNil(RunnerScrollDirection(rawValue: "sideways")) + } + + func testRunnerScrollGesturePlanRejectsInvalidAmountAndPixels() { + XCTAssertNil( + runnerScrollGesturePlan( + direction: .down, + amount: 0, + pixels: nil, + referenceWidth: 300, + referenceHeight: 600 + ) + ) + XCTAssertNil( + runnerScrollGesturePlan( + direction: .down, + amount: nil, + pixels: -10, + referenceWidth: 300, + referenceHeight: 600 + ) + ) + XCTAssertNil( + runnerScrollGesturePlan( + direction: .down, + amount: .infinity, + pixels: nil, + referenceWidth: 300, + referenceHeight: 600 + ) + ) + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScrollViewportPolicyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScrollViewportPolicyTests.swift new file mode 100644 index 0000000000..208397cc95 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ScrollViewportPolicyTests.swift @@ -0,0 +1,165 @@ +import XCTest +import AgentDeviceSnapshotPresentation + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +private struct ScrollViewportPolicyFixture: Decodable { + struct Frame: Decodable { + let x: Double + let y: Double + let width: Double + let height: Double + + var cgRect: CGRect { + CGRect(x: x, y: y, width: width, height: height) + } + } + + struct Constants: Decodable { + let minVisibleFraction: Double + let accessoryAllowance: Double + } + + struct Expected: Decodable { + let kind: String + let viewport: Frame? + let keyboardMinY: Double? + let visibleHeight: Double? + } + + struct TestCase: Decodable { + let name: String + let viewport: Frame + let keyboard: Frame + let expected: Expected + } + + let constants: Constants + let cases: [TestCase] +} + +extension RunnerTests { + /// Golden parity table (#2500): every case in contracts/fixtures/scroll-keyboard-policy.json must + /// agree with the vitest twin. Add cases there, never fork the rule. + func testScrollViewportKeyboardClipMatchesGoldenParityTable() throws { + let fixture = try loadScrollViewportPolicyFixture() + XCTAssertFalse(fixture.cases.isEmpty, "parity table must not be empty") + for testCase in fixture.cases { + let clip = ScrollViewportPolicy.clip( + viewport: testCase.viewport.cgRect, + keyboard: testCase.keyboard.cgRect + ) + switch testCase.expected.kind { + case "unobstructed": + XCTAssertEqual(clip, .unobstructed, testCase.name) + case "avoided": + let expectedFrame = try XCTUnwrap(testCase.expected.viewport, testCase.name).cgRect + let expectedMinY = try XCTUnwrap(testCase.expected.keyboardMinY, testCase.name) + XCTAssertEqual( + clip, + .avoided(frame: expectedFrame, keyboardMinY: expectedMinY), + testCase.name + ) + case "occluded": + let expectedMinY = try XCTUnwrap(testCase.expected.keyboardMinY, testCase.name) + let expectedVisibleHeight = try XCTUnwrap(testCase.expected.visibleHeight, testCase.name) + XCTAssertEqual( + clip, + .occluded(keyboardMinY: expectedMinY, visibleHeight: expectedVisibleHeight), + testCase.name + ) + default: + XCTFail("unknown expected kind `\(testCase.expected.kind)` in \(testCase.name)") + } + } + } + + /// The thresholds are the table's, not RunnerScrollViewportPolicy.swift's. The refusal reason + /// and the runner code are each one side's own vocabulary: the reason is what the host publishes, + /// the code is what this runner answers with, and neither is a shared clip constant. + func testScrollViewportPolicyUsesParityTableConstants() throws { + let constants = try loadScrollViewportPolicyFixture().constants + XCTAssertEqual(constants.minVisibleFraction, ScrollViewportPolicy.minVisibleFraction) + XCTAssertEqual(constants.accessoryAllowance, ScrollViewportPolicy.accessoryAllowance) + } + + /// A clipped landscape band shortens the frame, and `CoordinateSpaceRotation.native(point:)` derives a + /// `landscapeRight` native x from the frame's HEIGHT. Rotating inside the band therefore moves the + /// dispatched path sideways by exactly what the keyboard took, off the lane the plan was built for, + /// so the plan band and the coordinate basis stay separate values through dispatch (#2500). + func testScrollViewportDispatchKeepsTheUnclippedFrameAsItsCoordinateRotationBasis() throws { + let viewport = CGRect(x: 0, y: 0, width: 1210, height: 834) + let keyboard = CGRect(x: 0, y: 588, width: 1210, height: 246) + let clip = ScrollViewportPolicy.clip(viewport: viewport, keyboard: keyboard) + guard case .avoided(let band, let keyboardMinY) = clip else { + return XCTFail("expected a landscape keyboard to be avoided, got \(clip)") + } + XCTAssertEqual(band.height, 576) + + guard case .gesture(let gesture) = ScrollViewportPolicy.frames( + referenceFrame: viewport, + clip: clip + ).gestureDispatch(direction: .up, amount: nil, pixels: nil) else { + return XCTFail("expected a gesture inside the clipped band") + } + XCTAssertEqual(gesture.planFrame, band) + XCTAssertEqual(gesture.keyboardMinY, keyboardMinY) + XCTAssertEqual(gesture.coordinateFrame, viewport, "the rotation basis must survive the clip") + XCTAssertLessThanOrEqual( + max(gesture.plan.y1, gesture.plan.y2), + keyboard.minY - ScrollViewportPolicy.accessoryAllowance, + "a landscape swipe must stay clear of the keys" + ) + + let reported = gesture.attachingEvidence( + to: Response( + ok: true, + data: DataPayload(referenceWidth: viewport.width, referenceHeight: viewport.height), + error: nil + ) + ) + XCTAssertEqual( + reported.data?.referenceHeight, + band.height, + "the payload names the band the plan ran inside, not the synthesis frame" + ) + XCTAssertEqual(reported.data?.referenceWidth, viewport.width) + XCTAssertEqual(reported.data?.keyboardMinY, keyboardMinY) + XCTAssertEqual(reported.data?.keyboardAvoided, true) + + let orientedStartY = gesture.planFrame.minY + gesture.plan.y1 + let dispatchedFromViewport = CoordinateSpaceRotation.native( + point: CGPoint(x: gesture.planFrame.minX + gesture.plan.x1, y: orientedStartY), + in: gesture.coordinateFrame, + interfaceOrientation: RunnerInterfaceOrientation.landscapeRight + ) + let dispatchedFromBand = CoordinateSpaceRotation.native( + point: CGPoint(x: gesture.planFrame.minX + gesture.plan.x1, y: orientedStartY), + in: gesture.planFrame, + interfaceOrientation: RunnerInterfaceOrientation.landscapeRight + ) + XCTAssertEqual( + dispatchedFromViewport.x - dispatchedFromBand.x, + viewport.height - band.height, + accuracy: 0.001, + "rotating inside the clipped band would shift native x by what the keyboard took" + ) + } + + private func loadScrollViewportPolicyFixture() throws -> ScrollViewportPolicyFixture { + let fixtureURL = URL(fileURLWithPath: #filePath) + .deletingLastPathComponent() // UnitTests + .deletingLastPathComponent() // AgentDeviceRunnerUITests + .deletingLastPathComponent() // AgentDeviceRunner + .deletingLastPathComponent() // runner + .deletingLastPathComponent() // apple + .deletingLastPathComponent() // repo root + .appendingPathComponent("contracts") + .appendingPathComponent("fixtures") + .appendingPathComponent("scroll-keyboard-policy.json") + return try JSONDecoder().decode( + ScrollViewportPolicyFixture.self, + from: Data(contentsOf: fixtureURL) + ) + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SequenceExecutionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SequenceExecutionTests.swift new file mode 100644 index 0000000000..8535c29ebb --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SequenceExecutionTests.swift @@ -0,0 +1,200 @@ +import XCTest + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +// MARK: - In-bundle unit tests (device-free) + +extension RunnerTests { + func testSequenceDecodesStepsFromWire() throws { + let json = """ + {"command":"sequence","commandId":"seq-1","steps":[ + {"kind":"tap","x":100,"y":200}, + {"kind":"doubleTap","x":101,"y":200}, + {"kind":"longPress","x":102,"y":200,"durationMs":300,"pauseMs":50} + ]} + """ + let command = try JSONDecoder().decode(Command.self, from: Data(json.utf8)) + XCTAssertEqual(command.command, .sequence) + XCTAssertEqual(command.steps?.count, 3) + XCTAssertEqual(command.steps?[0].kind, "tap") + XCTAssertEqual(command.steps?[1].kind, "doubleTap") + XCTAssertEqual(command.steps?[2].pauseMs, 50) + } + + func testSequenceAcceptsDoubleTapKind() { + // A doubleTap step missing coords must fail on the coords check, not the kind allowlist — + // proving "doubleTap" passes validateSequenceStep without needing a device to execute on. + let response = executeSequenceForTest(steps: [ + sequenceStep(kind: "doubleTap", x: nil) + ]) + XCTAssertEqual(response.ok, false) + XCTAssertEqual(response.error?.code, "INVALID_ARGS") + XCTAssertTrue(response.error?.message.contains("requires finite x and y") ?? false) + XCTAssertFalse(response.error?.message.contains("unsupported kind") ?? true) + } + + func testSequenceRejectsUnknownKind() throws { + let response = executeSequenceForTest(steps: [ + sequenceStep(kind: "tap", x: 1, y: 2), + sequenceStep(kind: "pinch", x: 3, y: 4), + ]) + XCTAssertEqual(response.ok, false) + XCTAssertEqual(response.error?.code, "INVALID_ARGS") + XCTAssertTrue(response.error?.message.contains("step 1") ?? false) + XCTAssertTrue(response.error?.message.contains("pinch") ?? false) + } + + func testSequenceRejectsEmpty() { + let response = executeSequenceForTest(steps: []) + XCTAssertEqual(response.ok, false) + XCTAssertEqual(response.error?.code, "INVALID_ARGS") + } + + func testSequenceRejectsTooManySteps() { + let steps = (0..<21).map { _ in sequenceStep(kind: "tap", x: 1, y: 2) } + let response = executeSequenceForTest(steps: steps) + XCTAssertEqual(response.ok, false) + XCTAssertEqual(response.error?.code, "INVALID_ARGS") + XCTAssertTrue(response.error?.message.contains("at most 20") ?? false) + } + + func testAssembleSequencePreservesOrderOnSuccess() { + let steps = [ + sequenceStep(kind: "tap", x: 1, y: 1), + sequenceStep(kind: "longPress", x: 2, y: 2), + sequenceStep(kind: "tap", x: 3, y: 3), + ] + var calls: [Int] = [] + let execution = assembleSequenceExecution(steps: steps) { index, _ in + calls.append(index) + return SequenceStepOutcome( + outcome: .performed, + gestureStartUptimeMs: Double(index * 10), + gestureEndUptimeMs: Double(index * 10 + 5) + ) + } + XCTAssertEqual(calls, [0, 1, 2]) + XCTAssertEqual(execution.completedSteps, 3) + XCTAssertNil(execution.failedStepIndex) + XCTAssertEqual(execution.results.map { $0.kind }, ["tap", "longPress", "tap"]) + XCTAssertEqual(execution.gestureStartUptimeMs, 0) + XCTAssertEqual(execution.gestureEndUptimeMs, 25) + } + + func testAssembleSequenceStopsAtFirstFailure() { + let steps = [ + sequenceStep(kind: "tap", x: 1, y: 1), + sequenceStep(kind: "longPress", x: 2, y: 2), + sequenceStep(kind: "tap", x: 3, y: 3), + ] + var calls: [Int] = [] + let execution = assembleSequenceExecution(steps: steps) { index, _ in + calls.append(index) + if index == 1 { + return SequenceStepOutcome( + outcome: .unsupported(message: "long press unsupported", hint: nil), + gestureStartUptimeMs: 10, + gestureEndUptimeMs: 15 + ) + } + return SequenceStepOutcome(outcome: .performed, gestureStartUptimeMs: 0, gestureEndUptimeMs: 5) + } + // Step 2 is never invoked. + XCTAssertEqual(calls, [0, 1]) + XCTAssertEqual(execution.completedSteps, 1) + XCTAssertEqual(execution.failedStepIndex, 1) + // results.count == completedSteps + 1 (the failed step). + XCTAssertEqual(execution.results.count, 2) + XCTAssertEqual(execution.results[1].ok, false) + XCTAssertEqual(execution.results[1].errorCode, "UNSUPPORTED_OPERATION") + XCTAssertEqual(execution.results[1].errorMessage, "long press unsupported") + } + + func testSequenceWorstCaseResponseStaysUnderJournalCap() throws { + let longMessage = String(repeating: "e", count: 200) + let results = (0..<20).map { index in + SequenceStepResult( + ok: index < 19, + kind: "longPress", + errorCode: index < 19 ? nil : "UNSUPPORTED_OPERATION", + errorMessage: index < 19 ? nil : longMessage, + gestureStartUptimeMs: 123456.789, + gestureEndUptimeMs: 123466.789 + ) + } + let response = Response( + ok: true, + data: DataPayload( + message: "sequence", + completedSteps: 19, + failedStepIndex: 19, + sequenceResults: results + ) + ) + let encoded = try JSONEncoder().encode(response) + XCTAssertLessThan(encoded.count, 16 * 1024) + } + + private func sequenceStep( + kind: String, + x: Double?, + y: Double? = nil + ) -> SequenceStep { + SequenceStep( + kind: kind, + x: x, + y: y, + durationMs: nil, + pauseMs: nil, + synthesized: nil + ) + } + + /// Validation runs before any executor call, so the INVALID_ARGS paths are exercised without + /// reaching the device executor (which is never invoked when validation rejects). + private func executeSequenceForTest(steps: [SequenceStep]) -> Response { + let command = makeSequenceCommand(steps: steps) + return executeSequence(command: command, activeApp: app) + } + + /// Build a sequence Command via JSON so the test does not depend on the memberwise init's + /// parameter order. + private func makeSequenceCommand(steps: [SequenceStep]) -> Command { + struct SequenceCommandFixture: Encodable { + let command = "sequence" + let commandId = "seq-test" + let steps: [SequenceStep] + } + let data = try! JSONEncoder().encode(SequenceCommandFixture(steps: steps)) + return try! JSONDecoder().decode(Command.self, from: data) + } +} +#endif + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS) +extension RunnerTests { + func testSynthesizedSequenceTapFallsBackToXCTestCoordinateTapWhenAccessibilityIsUnavailable() throws { + let restoreSynthesizedTap = try forceSynthesizedTapFailure() + app.launch() + currentApp = app + defer { + restoreSynthesizedTap() + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } + let label = app.staticTexts["Agent Device Runner"] + XCTAssertTrue(label.waitForExistence(timeout: appExistenceTimeout)) + let point = CGPoint(x: label.frame.midX, y: label.frame.midY) + runnerAccessibilityHealth = .unavailable + let command = try runnerCommandFixture( + #"{"command":"sequence","commandId":"sequence-synthesized-tap-fallback","steps":[{"kind":"tap","x":\#(point.x),"y":\#(point.y),"synthesized":true}]}"# + ) + + let response = try executeOnMainPrepared(command: command, activeApp: app) + + XCTAssertTrue(response.ok) + XCTAssertEqual(response.data?.completedSteps, 1) + XCTAssertNil(response.data?.failedStepIndex) + XCTAssertEqual(response.data?.sequenceResults?.first?.ok, true) + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotBackendCapabilitiesTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotBackendCapabilitiesTests.swift new file mode 100644 index 0000000000..a56627eed9 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotBackendCapabilitiesTests.swift @@ -0,0 +1,80 @@ +import XCTest + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +private struct SnapshotBackendParityFixture: Decodable { + struct Availability: Decodable { + let simulator: Bool + let physicalDevice: Bool + } + + struct Backend: Decodable { + let name: String + let forceable: Bool + let supportsRawProjection: Bool + let regularDepth: String + let hittable: String + let availability: Availability + } + + let backends: [Backend] +} + +extension RunnerTests { + private func loadSnapshotBackendParityFixture() throws -> SnapshotBackendParityFixture { + let fixtureURL = URL(fileURLWithPath: #filePath) + .deletingLastPathComponent() // UnitTests + .deletingLastPathComponent() // AgentDeviceRunnerUITests + .deletingLastPathComponent() // AgentDeviceRunner + .deletingLastPathComponent() // runner + .deletingLastPathComponent() // apple + .deletingLastPathComponent() // repo root + .appendingPathComponent("contracts") + .appendingPathComponent("fixtures") + .appendingPathComponent("ios-snapshot-backends.json") + return try JSONDecoder().decode( + SnapshotBackendParityFixture.self, + from: Data(contentsOf: fixtureURL) + ) + } + + /// The JSON table is the cross-runtime declaration used by the TypeScript capability registry + /// and this runner. A backend case, forceability branch, projection/depth claim, or availability + /// change that is not classified in both implementations fails before an iOS smoke can drift. + func testSnapshotBackendDeclarationsMatchCapabilityFixture() throws { + let fixture = try loadSnapshotBackendParityFixture() + XCTAssertEqual( + fixture.backends.map(\.name), + SnapshotBackendKind.allCases.map(\.rawValue) + ) + + for expected in fixture.backends { + guard let backend = SnapshotBackendKind(rawValue: expected.name) else { + XCTFail("fixture contains an unknown snapshot backend: \(expected.name)") + continue + } + XCTAssertEqual(backend.isForceable, expected.forceable, expected.name) + XCTAssertEqual(backend.supportsRawProjection, expected.supportsRawProjection, expected.name) + XCTAssertEqual( + backend.regularDepthCapability.rawValue, + expected.regularDepth, + "regular depth capability: \(expected.name)" + ) + XCTAssertEqual( + backend.hittableSemantics, + expected.hittable, + "hittable semantics: \(expected.name)" + ) + XCTAssertEqual( + backend.isAvailable(on: .simulator), + expected.availability.simulator, + "simulator availability: \(expected.name)" + ) + XCTAssertEqual( + backend.isAvailable(on: .physicalDevice), + expected.availability.physicalDevice, + "physical-device availability: \(expected.name)" + ) + } + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanOccupancyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanOccupancyTests.swift index afb15147d2..b191787fb5 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanOccupancyTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanOccupancyTests.swift @@ -15,9 +15,13 @@ private enum RunnerBlockingSnapshotGate { /// read the AX server gave up on. The swap is process-wide while installed, which serial XCTest /// execution tolerates. private final class RunnerBlockingSnapshotStub: NSObject { + /// Far past the plan deadline, so only the test's own release ends the block; it only stops a + /// stuck test from holding the main thread. + private static let leakGuard: TimeInterval = 75 + @objc(snapshotWithError:) func snapshot() throws -> XCUIElementSnapshot { - _ = RunnerBlockingSnapshotGate.release.wait(timeout: .now() + 20) + _ = RunnerBlockingSnapshotGate.release.wait(timeout: .now() + Self.leakGuard) throw NSError( domain: "AgentDeviceRunner.tests", code: 1, @@ -26,6 +30,44 @@ private final class RunnerBlockingSnapshotStub: NSObject { } } +/// Records when each `RunnerSlowSweepQueryStub` query started. The swizzled IMP cannot capture +/// test-local state, so each run resets it. +private enum RunnerSlowSweepQueryGate { + /// Shorter than `flatInteractiveQueryBudget`, so a sweep that stops at its slice deadline never + /// outlives the slice; a full sweep of these outlasts the slice, so one that ignores it does. + static let queryDuration: TimeInterval = 0.06 + private static let lock = NSLock() + private static var starts: [Date] = [] + + static func reset() { + lock.lock() + starts = [] + lock.unlock() + } + + static func recordStart() { + lock.lock() + starts.append(Date()) + lock.unlock() + } + + static func recordedStarts() -> [Date] { + lock.lock() + defer { lock.unlock() } + return starts + } +} + +/// Stands in for `-[XCUIElementQuery allElementsBoundByIndex]` so every query the sweep runs holds +/// the main thread for a fixed time and finds nothing. +private final class RunnerSlowSweepQueryStub: NSObject { + @objc var allElementsBoundByIndex: [XCUIElement] { + RunnerSlowSweepQueryGate.recordStart() + Thread.sleep(forTimeInterval: RunnerSlowSweepQueryGate.queryDuration) + return [] + } +} + extension RunnerTests { /// The Bluesky feed shape: the tree XPC grinds past its slice. The plan must recover through /// private AX without queueing the query sweep behind the abandoned XPC, and a fresh process's @@ -50,9 +92,13 @@ extension RunnerTests { // resolution is slow, and it must not be the block the plan abandons. XCTAssertTrue(app.wait(for: .runningForeground, timeout: 10)) XCTAssertFalse(app.frame.isEmpty) + // The traversal context reads the viewport under a 1 s cap; a cold first read can overrun it + // and abandon the wrong block, so pay it here, uncapped. + _ = safeSnapshotViewport(app: app, readingOrientation: true) currentApp = app currentBundleId = "com.callstack.agentdevice.runner.tree-capture-test" - snapshotXCTestPenaltyWarmupExemptionPending = true + snapshotXCTestPenaltyWarmupExemption.isPending = true + let captureTarget = takeSnapshotCaptureTarget(app: app) RunnerBlockingSnapshotGate.release = DispatchSemaphore(value: 0) let originalImplementation = method_getImplementation(snapshotMethod) method_setImplementation(snapshotMethod, method_getImplementation(stubMethod)) @@ -77,7 +123,7 @@ extension RunnerTests { do { box.payload = try self.runSnapshotCapturePlan( Self.regularVisiblePlan, - app: self.app, + target: captureTarget, options: PresentationOptions(interactiveOnly: false, depth: nil, scope: nil, raw: false), terminal: .sparseWithFatalOnAXFailure, deadline: Date().addingTimeInterval(12) @@ -88,7 +134,7 @@ extension RunnerTests { self.mainThreadWorkLock.lock() box.abandonedAfterPlan = self.abandonedMainThreadWorkCount self.mainThreadWorkLock.unlock() - box.penalizedAfterPlan = self.isSnapshotXCTestChannelPenalized(bundleId: self.currentBundleId) + box.penalizedAfterPlan = self.isSnapshotXCTestChannelPenalized(bundleId: captureTarget.bundleId) RunnerBlockingSnapshotGate.release.signal() planned.fulfill() } @@ -102,7 +148,7 @@ extension RunnerTests { XCTAssertNil(box.error) let quality = try XCTUnwrap(box.payload?.snapshotQuality) XCTAssertEqual(quality.backend, SnapshotBackendKind.privateAX.rawValue) - XCTAssertEqual(quality.state, "recovered") + XCTAssertEqual(quality.state, .recovered) XCTAssertTrue( quality.reason?.contains("tree capture exceeded") == true, "the tree XPC, not the viewport read, must be the abandoned block: \(quality.reason ?? "nil")" @@ -123,5 +169,201 @@ extension RunnerTests { return XCTFail("expected the runner idle once the tree XPC drained") } } + + /// A non-interactive query sweep runs its queries on the main thread one at a time. It must stop + /// starting them at the slice deadline its caller waits for, not at the plan deadline, or it holds + /// the main thread after the plan has answered (#2783). + func testNonInteractiveQuerySweepStopsAtTheSliceItsCallerWaitsFor() throws { + let sweepQueryCount = 19 + XCTAssertLessThan(RunnerSlowSweepQueryGate.queryDuration, Self.flatInteractiveQueryBudget) + XCTAssertGreaterThan( + RunnerSlowSweepQueryGate.queryDuration * Double(sweepQueryCount), + Self.flatInteractiveFallbackBudget + ) + guard + let queryMethod = class_getInstanceMethod( + XCUIElementQuery.self, + #selector(getter: XCUIElementQuery.allElementsBoundByIndex) + ), + let stubMethod = class_getInstanceMethod( + RunnerSlowSweepQueryStub.self, + #selector(getter: RunnerSlowSweepQueryStub.allElementsBoundByIndex) + ) + else { + XCTFail("unable to install the slow sweep query stub") + return + } + app.launchArguments = ["--agent-device-selector-read-regression"] + app.launch() + XCTAssertTrue(app.wait(for: .runningForeground, timeout: 10)) + XCTAssertFalse(app.frame.isEmpty) + currentApp = app + currentBundleId = "com.callstack.agentdevice.runner.query-sweep-slice-test" + let captureTarget = takeSnapshotCaptureTarget(app: app) + RunnerSlowSweepQueryGate.reset() + let originalImplementation = method_getImplementation(queryMethod) + method_setImplementation(queryMethod, method_getImplementation(stubMethod)) + defer { + method_setImplementation(queryMethod, originalImplementation) + clearSnapshotXCTestChannelPenalty(reason: "test-cleanup") + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } + + final class ResultBox { + var payload: DataPayload? + var error: Error? + var returnedAt: Date? + var abandonedAtReturn: Bool? + } + let box = ResultBox() + let planned = expectation(description: "query-sweep plan answered") + DispatchQueue(label: "agent-device.runner.tests.query-sweep-slice").async { + do { + box.payload = try self.runSnapshotCapturePlan( + [.querySweep], + target: captureTarget, + options: PresentationOptions(interactiveOnly: false, depth: nil, scope: nil, raw: false), + terminal: .sparseWithFatalOnAXFailure, + deadline: Date().addingTimeInterval(20) + ) + } catch { + box.error = error + } + box.returnedAt = Date() + box.abandonedAtReturn = self.hasAbandonedMainThreadWork() + planned.fulfill() + } + + wait(for: [planned], timeout: 30) + let drainDeadline = Date().addingTimeInterval(3) + while hasAbandonedMainThreadWork(), Date() < drainDeadline { + sleepFor(0.005) + } + let starts = RunnerSlowSweepQueryGate.recordedStarts() + + XCTAssertNil(box.error) + XCTAssertEqual(box.payload?.snapshotQuality?.backend, SnapshotBackendKind.querySweep.rawValue) + XCTAssertEqual( + box.abandonedAtReturn, + false, + "the sweep must finish inside the slice its caller waits for" + ) + let returnedAt = try XCTUnwrap(box.returnedAt) + XCTAssertTrue( + starts.allSatisfy { $0 <= returnedAt }, + "no sweep query may start after the plan answered" + ) + XCTAssertLessThan(starts.count, sweepQueryCount, "the slice must cut the sweep short") + XCTAssertFalse(hasAbandonedMainThreadWork()) + } + + /// A query sweep that stops at its own slice deadline collected a partial tree, not an answer, so + /// it is a tier timeout: the plan must arm the XCTest-channel penalty and let private AX answer, + /// keeping the partial sweep only as the fallback. Accepting it because it carries more nodes than + /// the sparse threshold ships a hierarchy-free capture and leaves every later capture of the same + /// screen to pay for the full sweep again (#2781). + func testQuerySweepThatEndsOnItsSliceDeadlinePenalizesChannelAndReachesPrivateAX() throws { + guard + let queryMethod = class_getInstanceMethod( + XCUIElementQuery.self, + #selector(getter: XCUIElementQuery.allElementsBoundByIndex) + ), + let stubMethod = class_getInstanceMethod( + RunnerSlowSweepQueryStub.self, + #selector(getter: RunnerSlowSweepQueryStub.allElementsBoundByIndex) + ) + else { + XCTFail("unable to install the slow sweep query stub") + return + } + app.launchArguments = ["--agent-device-selector-read-regression"] + app.launch() + XCTAssertTrue(app.wait(for: .runningForeground, timeout: 10)) + currentApp = app + currentBundleId = "com.callstack.agentdevice.runner.query-sweep-timeout-test" + snapshotXCTestPenaltyWarmupExemption.isPending = false + clearSnapshotXCTestChannelPenalty(reason: "test-setup") + RunnerSlowSweepQueryGate.reset() + let captureTarget = takeSnapshotCaptureTarget(app: app) + let originalImplementation = method_getImplementation(queryMethod) + method_setImplementation(queryMethod, method_getImplementation(stubMethod)) + defer { + method_setImplementation(queryMethod, originalImplementation) + clearSnapshotXCTestChannelPenalty(reason: "test-cleanup") + clearPrivateAXAcceptedDepth(reason: "test-cleanup") + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } + + final class ResultBox { + var payload: DataPayload? + var error: Error? + var abandonedAtReturn = false + var penalizedAtReturn = false + var sweepStartsAfterFirstPlan = 0 + var secondPayload: DataPayload? + var secondError: Error? + } + let box = ResultBox() + let planned = expectation(description: "slow-sweep plan answered") + DispatchQueue(label: "agent-device.runner.tests.query-sweep-timeout").async { + do { + box.payload = try self.runSnapshotCapturePlan( + [.querySweep, .privateAX], + target: captureTarget, + options: PresentationOptions(interactiveOnly: false, depth: nil, scope: nil, raw: false), + terminal: .sparseWithFatalOnAXFailure, + deadline: Date().addingTimeInterval(20) + ) + } catch { + box.error = error + } + box.abandonedAtReturn = self.hasAbandonedMainThreadWork() + box.penalizedAtReturn = self.isSnapshotXCTestChannelPenalized(bundleId: captureTarget.bundleId) + box.sweepStartsAfterFirstPlan = RunnerSlowSweepQueryGate.recordedStarts().count + do { + box.secondPayload = try self.runSnapshotCapturePlan( + Self.regularVisiblePlan, + target: self.takeSnapshotCaptureTarget(app: self.app), + options: PresentationOptions(interactiveOnly: false, depth: nil, scope: nil, raw: false), + terminal: .sparseWithFatalOnAXFailure, + deadline: Date().addingTimeInterval(20) + ) + } catch { + box.secondError = error + } + planned.fulfill() + } + + wait(for: [planned], timeout: 60) + XCTAssertNil(box.error) + let quality = box.payload?.snapshotQuality + XCTAssertEqual( + quality?.backend, + SnapshotBackendKind.privateAX.rawValue, + "a sweep that ended on its slice deadline is a tier timeout, not an accepted capture" + ) + XCTAssertEqual(quality?.state, .recovered) + XCTAssertGreaterThan(box.payload?.nodes?.count ?? 0, 1, "private AX answers with a real tree") + XCTAssertFalse(box.abandonedAtReturn, "the sweep must answer inside its own main-thread hop") + XCTAssertTrue( + box.penalizedAtReturn, + "the tier timeout must arm the XCTest-channel penalty for the captured target" + ) + + XCTAssertNil(box.secondError) + XCTAssertEqual( + box.secondPayload?.snapshotQuality?.backend, + SnapshotBackendKind.privateAX.rawValue, + "the armed penalty must defer the next plan's XCTest tiers, not re-run the sweep" + ) + XCTAssertEqual( + RunnerSlowSweepQueryGate.recordedStarts().count, + box.sweepStartsAfterFirstPlan, + "the deferred plan may start no sweep query at all" + ) + XCTAssertFalse(hasAbandonedMainThreadWork()) + } } #endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanTests.swift new file mode 100644 index 0000000000..b1ea3656d3 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanTests.swift @@ -0,0 +1,523 @@ +import XCTest +import AgentDeviceSnapshotPresentation + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +// MARK: - In-bundle unit tests + +extension RunnerTests { + private func planTestNode( + index: Int, + type: String, + label: String? = nil, + identifier: String? = nil, + hittable: Bool = false, + parentIndex: Int? = nil + ) -> PresentedNode { + SnapshotPresentation.singleElementRead( + RawAXNode( + index: index, + type: type, + label: label, + identifier: identifier, + value: nil, + rect: SnapshotRect(.zero), + enabled: true, + focused: nil, + selected: nil, + hittable: hittable, + depth: parentIndex == nil ? 0 : 1, + parentIndex: parentIndex, + hiddenContentAbove: nil, + hiddenContentBelow: nil + ) + ) + } + + func testSparsePayloadReasonMatrix() { + let root = planTestNode(index: 0, type: "Application", label: "Example App", hittable: true) + let window = planTestNode(index: 1, type: "Window", parentIndex: 0) + let button = planTestNode(index: 1, type: "Button", label: "Ok", hittable: true, parentIndex: 0) + let shell = planTestNode( + index: 1, + type: "Other", + identifier: "appShell", + parentIndex: 0 + ) + let serializationPlaceholder = planTestNode( + index: 2, + type: "Other", + label: "[object Object]", + parentIndex: 1 + ) + + // Labeled, hittable root over a bare window is still sparse. + XCTAssertNotNil(Self.sparsePayloadReason(DataPayload(nodes: [root, window], truncated: false))) + // Deadline-truncated near-empty sweep needs recovery even with one real control. + XCTAssertNotNil(Self.sparsePayloadReason(DataPayload(nodes: [root, button], truncated: true))) + // The same tiny tree from a completed sweep is a legitimately minimal screen. + XCTAssertNil(Self.sparsePayloadReason(DataPayload(nodes: [root, button], truncated: false))) + // Container metadata plus a stringified serialization placeholder is not readable UI. + XCTAssertNotNil( + Self.sparsePayloadReason( + DataPayload(nodes: [root, shell, serializationPlaceholder], truncated: false) + ) + ) + let actionableShell = planTestNode( + index: 1, + type: "Other", + identifier: "checkout", + hittable: true, + parentIndex: 0 + ) + XCTAssertNil( + Self.sparsePayloadReason(DataPayload(nodes: [root, actionableShell], truncated: false)) + ) + // Empty payloads are degraded. + XCTAssertNotNil(Self.sparsePayloadReason(DataPayload(nodes: [], truncated: false))) + } + + /// A sweep the slice cut short is rejected as a tier timeout even when what it collected clears + /// every bar the quality classifier has, while the identical payload from a sweep that ran all its + /// queries is accepted. The pair is the point: node count cannot see the deadline (#2781). + func testDeadlineExhaustedTierIsRejectedWhileItsIdenticalCompletedPayloadIsAccepted() { + let root = planTestNode(index: 0, type: "Application", label: "Example App", hittable: true) + let nodes: [PresentedNode] = [root] + (1..<13).map { index in + planTestNode(index: index, type: "Button", label: "Row \(index)", hittable: true, parentIndex: 0) + } + let payload = DataPayload(nodes: nodes, truncated: true) + XCTAssertNil( + Self.sparsePayloadReason(payload), + "this payload clears the classifier on its own, so only the tier outcome can reject it" + ) + XCTAssertEqual( + Self.snapshotTierRejectionReason( + outcome: .deadlineExhausted, + kind: .querySweep, + payload: payload + )?.code, + "budget", + "a tier that spent its capture slice is a timeout whatever it collected" + ) + XCTAssertNil( + Self.snapshotTierRejectionReason( + outcome: .completed, + kind: .querySweep, + payload: payload + ) + ) + } + + func testCollapsedLeafIndexesFlagsMergedContainersOnly() { + let root = planTestNode(index: 0, type: "Application", label: "App") + let merged = planTestNode( + index: 1, + type: "Other", + label: (0...30).map { "Row \($0), Tap" }.joined(separator: ", "), + parentIndex: 0 + ) + let prose = planTestNode( + index: 2, + type: "StaticText", + label: (0...30).map { "clause \($0)" }.joined(separator: ", "), + parentIndex: 0 + ) + XCTAssertEqual(Self.collapsedLeafIndexes([root, merged, prose]), [1]) + XCTAssertNil(Self.collapsedLeafIndexes([root, prose])) + } + + func testTerminalFailsClosedOnInteractiveAxFailureRegardlessOfSparseBest() { + // Interactive AX failure must invalidate + fail closed; a later tier's sparse synthetic-root + // "best" must never downgrade this to a returned-sparse payload (regression: best == nil guard). + XCTAssertEqual( + Self.resolveSnapshotPlanTerminal(terminal: .sparseWithFatalOnAXFailure, interactiveOnly: true), + .failClosed + ) + XCTAssertEqual( + Self.resolveSnapshotPlanTerminal(terminal: .sparseWithFatalOnAXFailure, interactiveOnly: false), + .sparseBest + ) + XCTAssertEqual( + Self.resolveSnapshotPlanTerminal(terminal: .throwOnAXFailure, interactiveOnly: true), + .throwAxFailure + ) + } + + func testXCTestChannelStateFirstFailureStampsDeferredCodeOnlyForDeferral() { + XCTAssertNil(Self.xcTestChannelStateFirstFailure(.normal)) + XCTAssertEqual(Self.xcTestChannelStateFirstFailure(.deferredToIndependentBackend)?.code, "deferred") + XCTAssertEqual(Self.xcTestChannelStateFirstFailure(.boundedXCTestProbe)?.code, "budget") + } + + func testSnapshotQualityCarriesPhaseTimingAtResponseLevel() { + let timing = SnapshotCaptureTiming(acquisitionMs: 12, presentationMs: 34) + let capture = SnapshotBackendCapture( + payload: DataPayload( + nodes: [planTestNode(index: 0, type: "Application", label: "App")], + truncated: false + ), + effectiveDepth: nil, + timing: timing + ) + + let payload = stampedSnapshotPayload( + capture, + backend: .recursiveTree, + state: .healthy, + reason: nil + ) + + XCTAssertEqual(payload.snapshotQuality?.timing, timing) + XCTAssertEqual(payload.nodes?.count, 1) + } + + func testStampedPayloadCarriesDisclosuresOnlyInTheVerdict() { + let root = planTestNode(index: 0, type: "Application", label: "App") + let merged = planTestNode( + index: 1, + type: "Other", + label: (0...30).map { "Tab \($0)" }.joined(separator: ", "), + parentIndex: 0 + ) + let coverage = SnapshotCustomActionCoverage( + read: 12, candidates: 19, truncated: 0, blocked: false) + let silent = stampedSnapshotPayload( + SnapshotBackendCapture( + payload: DataPayload(nodes: [root, merged], truncated: false), + effectiveDepth: nil, + customActions: coverage + ), + backend: .recursiveTree, + state: .healthy, + reason: nil + ) + XCTAssertNil(silent.message) + XCTAssertEqual(silent.snapshotQuality?.customActions, coverage) + XCTAssertEqual(silent.snapshotQuality?.collapsedLeafIndexes, [1]) + + let underlying = stampedSnapshotPayload( + SnapshotBackendCapture( + payload: DataPayload(message: "underlying", nodes: [root], truncated: false), + effectiveDepth: 4 + ), + backend: .privateAX, + state: .recovered, + reason: (reason: "tree capture timed out", code: "budget") + ) + XCTAssertEqual(underlying.message, "underlying") + } + + func testStampedPayloadTruncationTracksCompletenessNotRecoveryProvenance() { + let complete = SnapshotBackendCapture( + payload: DataPayload( + nodes: [ + planTestNode(index: 0, type: "Application", label: "App"), + planTestNode(index: 1, type: "Button", label: "Open", parentIndex: 0), + ], + truncated: false + ), + effectiveDepth: nil + ) + let deferred: (reason: String, code: String) = ( + "XCTest-backed snapshot tiers were deferred after recent slow accessibility work", "deferred" + ) + + // The CI signature behind `is absent ... capture was truncated`: a complete private AX + // tree selected while the XCTest channel is penalized is whole, and must say so. + let recovered = stampedSnapshotPayload( + complete, backend: .privateAX, state: .recovered, reason: deferred) + XCTAssertEqual(recovered.snapshotQuality?.state, .recovered) + XCTAssertEqual(recovered.truncated, false) + + let depthLimited = stampedSnapshotPayload( + SnapshotBackendCapture(payload: complete.payload, effectiveDepth: 56), + backend: .privateAX, state: .recovered, reason: deferred) + XCTAssertEqual(depthLimited.truncated, true) + + let cappedPayload = stampedSnapshotPayload( + SnapshotBackendCapture( + payload: DataPayload(nodes: complete.payload.nodes ?? [], truncated: true), + effectiveDepth: nil), + backend: .recursiveTree, state: .healthy, reason: nil) + XCTAssertEqual(cappedPayload.truncated, true) + + let sparse = stampedSnapshotPayload( + complete, backend: .querySweep, state: .sparse, + reason: ("snapshot returned no semantic controls or content", "sparse-tree")) + XCTAssertEqual(sparse.truncated, true) + } + + func testSnapshotQualityCarriesUnscopedQualityPayload() { + let quality = DataPayload( + nodes: [planTestNode(index: 0, type: "Application", label: "App")], + truncated: false + ) + let capture = SnapshotBackendCapture( + payload: quality, + effectiveDepth: nil, + qualityPayload: quality + ) + + let payload = stampedSnapshotPayload( + capture, + backend: .recursiveTree, + state: .healthy, + reason: nil + ) + + XCTAssertEqual(payload.qualityPayload?.nodes.count, 1) + XCTAssertEqual(payload.qualityPayload?.truncated, false) + XCTAssertNil(payload.qualityPayload?.scope) + } + + func testDirectPresentationDoesNotClaimPlanTiming() { + let options = PresentationOptions( + interactiveOnly: false, + depth: nil, + scope: nil, + raw: true + ) + let result = SnapshotPresentation.presentRaw( + SnapshotAcquisition( + hint: SnapshotPresentation.captureHint(for: options), + nodes: [], + truncated: false, + effectiveDepth: nil, + viewport: .reported(box: CGRect(x: 0, y: 0, width: 402, height: 874)) + ), + options: options + ) + let capture = Self.makeSnapshotBackendCapture(from: result) + + let payload = stampedSnapshotPayload( + capture, + backend: .recursiveTree, + state: .healthy, + reason: nil + ) + + XCTAssertNil(payload.snapshotQuality?.timing) + } + + /// The raw plan is derived from what each backend can actually serve, not from a second + /// hand-maintained list. Non-vacuity: flipping `querySweep.supportsRawProjection` to true adds it + /// to the plan and fails the first two assertions — which is exactly the shape of #1797 D4, a + /// `--raw` request answered by a backend that has no hierarchy to return. + func testRawDiagnosticPlanCarriesOnlyBackendsThatCanServeRaw() { + XCTAssertEqual(Self.rawDiagnosticPlan, [.recursiveTree, .privateAX]) + XCTAssertEqual( + SnapshotBackendKind.allCases.filter { !$0.supportsRawProjection }, [.querySweep]) + XCTAssertTrue(Self.rawDiagnosticPlan.allSatisfy(\.supportsRawProjection)) + // Tree-first error propagation is the raw plan's other contract (ADR 0004). + XCTAssertEqual(Self.rawDiagnosticPlan.first, .recursiveTree) + } + + /// A projection mismatch is a runner bug, not an accessibility failure: it must not take the + /// AX-failure terminal route (rethrow / fail-closed), just drop its tier with a named reason. + func testProjectionMismatchFailureIsStructuredAndNotAnAxFailure() { + let failure = Self.snapshotProjectionMismatchFailure( + .querySweep, requested: .raw, acquired: .regular) + XCTAssertEqual(failure.code, "IOS_SNAPSHOT_PROJECTION_MISMATCH") + XCTAssertTrue(failure.message.contains("queries")) + XCTAssertTrue(failure.message.contains("raw")) + XCTAssertFalse(Self.isAxSnapshotFailure(failure)) + } + + /// #1634 P2: the decoded wire field must reach capture options and its + /// applicable plan. A pinned REGULAR capture defers to privateAX-first; the + /// RAW diagnostic plan is never rerouted by the pin — raw keeps tree-first + /// error propagation, which is exactly why raw baselines are excluded from + /// corroboration daemon-side. + func testDecodedPreferredBackendReachesOptionsAndApplicablePlan() throws { + let json = #"{"command":"snapshot","preferredBackend":"private-ax"}"# + let command = try JSONDecoder().decode(Command.self, from: Data(json.utf8)) + let options = Self.presentationOptions(from: command) + XCTAssertEqual(options.preferredBackend, "private-ax") + XCTAssertFalse(options.raw) + + let treated = Self.snapshotXCTestChannelTreatedAsPenalized( + penalized: false, preferredBackend: options.preferredBackend) + let pinned = Self.effectiveSnapshotCapturePlan( + Self.regularVisiblePlan, + xCTestChannelPenalized: treated, + preferredBackend: options.preferredBackend + ) + XCTAssertEqual(pinned.plan, [.privateAX]) + XCTAssertEqual(pinned.xCTestChannelState, .deferredToIndependentBackend) + + let raw = Self.effectiveSnapshotCapturePlan( + Self.rawDiagnosticPlan, + xCTestChannelPenalized: treated, + preferredBackend: options.preferredBackend + ) + XCTAssertEqual(raw.plan, Self.rawDiagnosticPlan) + + // A command without the field decodes to no pin and a normal plan. + let bare = try JSONDecoder().decode( + Command.self, from: Data(#"{"command":"snapshot"}"#.utf8)) + XCTAssertNil(Self.presentationOptions(from: bare).preferredBackend) + } + + /// #1635: the force seam must select the recursive tree even when the XCTest + /// channel is currently penalized. Without the preferred-backend argument, + /// this call returns the independent private-AX recovery plan instead. + func testPreferredTreeBackendPinsRegularPlanAndLeavesStructuredEvidence() { + let forced = Self.effectiveSnapshotCapturePlan( + Self.regularVisiblePlan, + xCTestChannelPenalized: true, + preferredBackend: SnapshotBackendKind.recursiveTree.rawValue + ) + XCTAssertEqual(forced.plan, [.recursiveTree]) + XCTAssertEqual(forced.xCTestChannelState, .normal) + XCTAssertEqual( + Self.xcTestChannelStateFirstFailure( + forced.xCTestChannelState, + preferredBackend: forced.preferredBackend?.rawValue + )?.code, + "requested-backend" + ) + } + + /// Same-backend evidence probes: a daemon-pinned private-AX capture takes the + /// penalized route even with a healthy channel, so tap-outcome corroboration + /// baselines and probes are always captured by the same backend (backends are + /// never comparable views of a screen). Composed with the plan rule, the pin + /// yields the privateAX-first deferred plan. + func testPreferredPrivateAXBackendPlansAsPenalized() { + XCTAssertTrue( + Self.snapshotXCTestChannelTreatedAsPenalized(penalized: false, preferredBackend: "private-ax")) + XCTAssertTrue( + Self.snapshotXCTestChannelTreatedAsPenalized(penalized: true, preferredBackend: nil)) + XCTAssertFalse( + Self.snapshotXCTestChannelTreatedAsPenalized(penalized: false, preferredBackend: nil)) + XCTAssertFalse( + Self.snapshotXCTestChannelTreatedAsPenalized(penalized: false, preferredBackend: "tree")) + + let pinned = Self.effectiveSnapshotCapturePlan( + Self.regularVisiblePlan, + xCTestChannelPenalized: Self.snapshotXCTestChannelTreatedAsPenalized( + penalized: false, preferredBackend: "private-ax" + ), + preferredBackend: "private-ax" + ) + XCTAssertEqual(pinned.plan, [.privateAX]) + XCTAssertEqual(pinned.xCTestChannelState, .deferredToIndependentBackend) + } + + func testEffectiveSnapshotCapturePlanDefersXCTestBackedTiersOnlyWhenPenalizedRegularPlan() { + let regular = Self.effectiveSnapshotCapturePlan( + Self.regularVisiblePlan, + xCTestChannelPenalized: true + ) + XCTAssertEqual(regular.plan, [.privateAX]) + XCTAssertEqual(regular.xCTestChannelState, .deferredToIndependentBackend) + XCTAssertNil(regular.treeCaptureSliceBudgetOverride) + + let unpenalized = Self.effectiveSnapshotCapturePlan( + Self.regularVisiblePlan, + xCTestChannelPenalized: false + ) + XCTAssertEqual(unpenalized.plan, Self.regularVisiblePlan) + XCTAssertEqual(unpenalized.xCTestChannelState, .normal) + XCTAssertNil(unpenalized.treeCaptureSliceBudgetOverride) + + // The raw diagnostic plan preserves tree-first error propagation even under penalty. + let raw = Self.effectiveSnapshotCapturePlan( + Self.rawDiagnosticPlan, + xCTestChannelPenalized: true + ) + XCTAssertEqual(raw.plan, Self.rawDiagnosticPlan) + XCTAssertEqual(raw.xCTestChannelState, .normal) + XCTAssertNil(raw.treeCaptureSliceBudgetOverride) + } + + func testEffectiveSnapshotCapturePlanUsesBoundedXCTestProbeWhenNoIndependentBackendRuns() { + let physicalDevicePlan = Self.effectiveSnapshotCapturePlan( + Self.regularVisiblePlan, + xCTestChannelPenalized: true, + availableBackends: [.recursiveTree, .querySweep] + ) + + XCTAssertEqual(physicalDevicePlan.plan, [.recursiveTree, .querySweep]) + XCTAssertEqual(physicalDevicePlan.xCTestChannelState, .boundedXCTestProbe) + XCTAssertEqual( + physicalDevicePlan.treeCaptureSliceBudgetOverride, + Self.penalizedXCTestProbeTreeSliceBudget + ) + } + + func testSnapshotXCTestChannelPenaltyMatchesBundleAndExpires() { + defer { + snapshotXCTestChannelPenaltyBundleId = nil + snapshotXCTestChannelPenaltyUntil = .distantPast + } + + penalizeSnapshotXCTestChannel(bundleId: "xyz.blueskyweb.app", reason: "test") + XCTAssertTrue(isSnapshotXCTestChannelPenalized(bundleId: "xyz.blueskyweb.app")) + XCTAssertFalse(isSnapshotXCTestChannelPenalized(bundleId: "com.other.app")) + + // A penalty recorded without a bundle applies to any current target. + penalizeSnapshotXCTestChannel(bundleId: nil, reason: "test") + XCTAssertTrue(isSnapshotXCTestChannelPenalized(bundleId: "com.other.app")) + + // Expired penalties stop applying. + snapshotXCTestChannelPenaltyUntil = Date(timeIntervalSinceNow: -1) + XCTAssertFalse(isSnapshotXCTestChannelPenalized(bundleId: "com.other.app")) + } + + func testAbandonedMainThreadWorkSkipsOnlyXCTestBackedSnapshotTiers() { + abandonedMainThreadWorkCount = 1 + defer { abandonedMainThreadWorkCount = 0 } + + XCTAssertTrue(shouldSkipSnapshotBackendForAbandonedMainThreadWork(.recursiveTree)) + XCTAssertTrue(shouldSkipSnapshotBackendForAbandonedMainThreadWork(.querySweep)) + XCTAssertFalse(shouldSkipSnapshotBackendForAbandonedMainThreadWork(.privateAX)) + } + +#if os(iOS) + /// #2403: a plan pinned to private AX serves a regular `--depth` request through acquisition + /// and presentation. With a backend depth gate in `captureWithBackend`, private AX returns no + /// capture, the plan falls through to the synthetic sparse root, and the daemon rejects that + /// zero-rect root as a missing viewport. + func testPrivateAXPinnedRegularDepthReachesAcquisitionAndPresentation() throws { + app.launchArguments = ["--agent-device-selector-read-regression"] + app.launch() + currentApp = app + currentBundleId = nil + defer { + currentApp = nil + clearPrivateAXAcceptedDepth(reason: "test-cleanup") + app.terminate() + } + func capture(depth: Int?) throws -> DataPayload { + try runSnapshotCapturePlan( + Self.regularVisiblePlan, + target: takeSnapshotCaptureTarget(app: app), + options: PresentationOptions( + interactiveOnly: false, + depth: depth, + scope: nil, + raw: false, + preferredBackend: SnapshotBackendKind.privateAX.rawValue + ), + terminal: .sparseWithFatalOnAXFailure + ) + } + + let capped = try capture(depth: 1) + + let quality = try XCTUnwrap(capped.snapshotQuality) + XCTAssertEqual(quality.backend, SnapshotBackendKind.privateAX.rawValue) + XCTAssertNotEqual(quality.state, .sparse) + let nodes = try XCTUnwrap(capped.nodes) + XCTAssertGreaterThan(nodes.count, 1) + XCTAssertEqual(nodes.map(\.depth).max(), 1) + XCTAssertNotEqual(nodes[0].rect, SnapshotRect(x: 0, y: 0, width: 0, height: 0)) + XCTAssertTrue(nodes.contains { $0.label == "Readable target" }) + + // The presented cut only ever narrows the unscoped capture from the same backend. + let unscoped = try XCTUnwrap(try capture(depth: nil).nodes) + XCTAssertLessThanOrEqual(nodes.count, unscoped.count) + } +#endif +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCaptureTargetTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCaptureTargetTests.swift new file mode 100644 index 0000000000..2139a30ccb --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCaptureTargetTests.swift @@ -0,0 +1,104 @@ +import XCTest +import AgentDeviceSnapshotPresentation + +extension RunnerTests { +#if AGENT_DEVICE_RUNNER_UNIT_TESTS + func testSnapshotCaptureTargetKeepsPreparedIdentityAndLeavesWarmupExemptionPending() { + currentApp = app + currentBundleId = "com.example.prepared" + currentAppProcessIdentifier = 42 + snapshotXCTestPenaltyWarmupExemption.isPending = true + defer { invalidateCachedTarget(reason: "unit_test_cleanup") } + + let target = takeSnapshotCaptureTarget(app: app) + currentBundleId = "com.example.replaced" + currentAppProcessIdentifier = 43 + + XCTAssertTrue(target.app === app) + XCTAssertEqual(target.bundleId, "com.example.prepared") + XCTAssertEqual(target.processIdentifier, 42) + XCTAssertTrue( + snapshotXCTestPenaltyWarmupExemption.isPending, + "only a capture plan that runs may spend the exemption" + ) + } + +#if os(iOS) + func testBlockingModalSnapshotLeavesWarmupExemptionForTheFirstCapturePlan() throws { + currentApp = app + currentBundleId = "com.example.fresh-process" + currentAppProcessIdentifier = 42 + snapshotXCTestPenaltyWarmupExemption.isPending = true + systemModalProbeOverrideForTesting = { _ in DataPayload(message: "blocking system modal") } + defer { + systemModalProbeOverrideForTesting = nil + runnerAccessibilityHealth = .unknown + invalidateCachedTarget(reason: "unit_test_cleanup") + } + let options = PresentationOptions(interactiveOnly: false, depth: nil, scope: nil, raw: false) + + let fast = try snapshotFast(target: takeSnapshotCaptureTarget(app: app), options: options) + let raw = try snapshotRaw(target: takeSnapshotCaptureTarget(app: app), options: options) + + XCTAssertEqual(fast.message, "blocking system modal") + XCTAssertEqual(raw.message, "blocking system modal") + XCTAssertTrue( + snapshotXCTestPenaltyWarmupExemption.isPending, + "a snapshot answered by the modal probe runs no capture plan, so the exemption stays pending" + ) + + _ = try runSnapshotCapturePlan( + [], + target: takeSnapshotCaptureTarget(app: app), + options: options, + terminal: .sparseWithFatalOnAXFailure + ) + + XCTAssertFalse( + snapshotXCTestPenaltyWarmupExemption.isPending, + "the first capture plan that runs spends the exemption" + ) + } +#endif + + func testProbePenaltyIdentityTakesTheIdentityItsCallerMayLegallyKnow() { + let prepared = SnapshotProbePenaltyIdentity(.prepared(bundleId: "com.example.prepared")) + prepared.captureFromMain(bundleId: "com.example.settled") + XCTAssertEqual( + prepared.penalizedBundleId, + "com.example.prepared", + "a capture penalizes the target it was prepared for, whatever lifecycle binds meanwhile" + ) + + let mainOwned = SnapshotProbePenaltyIdentity(.mainOwnedTarget) + XCTAssertNil( + mainOwned.penalizedBundleId, + "a command-queue caller knows no identity until the probe's main-side block runs" + ) + mainOwned.captureFromMain(bundleId: "com.example.settled") + XCTAssertEqual(mainOwned.penalizedBundleId, "com.example.settled") + } + + func testMainOwnedSnapshotStateWriteRunsOnMainBeforeReturningWhenMainIsFree() { + final class ResultBox { + var ranOnMainThread: Bool? + var appliedBeforeReturn: Bool? + } + let box = ResultBox() + let finished = expectation(description: "off-main write returned") + + DispatchQueue(label: "agent-device.runner.tests.main-owned-snapshot-state").async { + self.applyMainOwnedSnapshotState("unit_test") { + box.ranOnMainThread = Thread.isMainThread + } + box.appliedBeforeReturn = box.ranOnMainThread != nil + finished.fulfill() + } + + wait(for: [finished], timeout: 3) + XCTAssertEqual(box.ranOnMainThread, true) + XCTAssertEqual(box.appliedBeforeReturn, true) + XCTAssertFalse(hasAbandonedMainThreadWork()) + } +#endif +} diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotExecutionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotExecutionTests.swift new file mode 100644 index 0000000000..fa637a853b --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotExecutionTests.swift @@ -0,0 +1,84 @@ +import XCTest +import AgentDeviceSnapshotPresentation + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +extension RunnerTests { + func testPostSnapshotDelayMarkDoesNotQueueBehindAbandonedMainThreadWork() { + abandonedMainThreadWorkCount = 1 + defer { + abandonedMainThreadWorkCount = 0 + needsPostSnapshotInteractionDelay = false + } + + let finished = expectation(description: "off-main caller finished") + DispatchQueue(label: "agent-device.runner.tests.post-snapshot-delay").async { + self.setNeedsPostSnapshotInteractionDelay() + finished.fulfill() + } + + wait(for: [finished], timeout: 1) + mainThreadWorkLock.lock() + let abandonedWorkCount = abandonedMainThreadWorkCount + mainThreadWorkLock.unlock() + XCTAssertEqual(abandonedWorkCount, 1, "the skipped mark must not add an abandoned unit") + XCTAssertFalse(needsPostSnapshotInteractionDelay) + } + + func testSnapshotFailureInvalidationQueuesBehindAbandonedMainThreadWorkWithoutWaiting() { + currentBundleId = "com.example.stale-target" + defer { currentBundleId = nil } + + final class ResultBox { + var elapsed: TimeInterval? + var bundleStillCachedWhileBlocked: Bool? + var abandonedWhileBlocked: Int? + } + let box = ResultBox() + let mainBlocked = DispatchSemaphore(value: 0) + let releaseMain = DispatchSemaphore(value: 0) + let finished = expectation(description: "invalidation returned while main was blocked") + + DispatchQueue(label: "agent-device.runner.tests.snapshot-invalidation").async { + _ = try? self.runMainThreadWork( + "command_execution", + timeout: 0, + timeoutError: self.mainThreadExecutionTimeoutError + ) { + mainBlocked.signal() + _ = releaseMain.wait(timeout: .now() + 5) + return true + } + _ = mainBlocked.wait(timeout: .now() + 2) + let startedAt = Date() + self.invalidateCachedTargetAfterSnapshotFailure() + box.elapsed = Date().timeIntervalSince(startedAt) + box.bundleStillCachedWhileBlocked = self.currentBundleId != nil + self.mainThreadWorkLock.lock() + box.abandonedWhileBlocked = self.abandonedMainThreadWorkCount + self.mainThreadWorkLock.unlock() + releaseMain.signal() + finished.fulfill() + } + + wait(for: [finished], timeout: 8) + let drainDeadline = Date().addingTimeInterval(2) + while hasAbandonedMainThreadWork() || currentBundleId != nil, Date() < drainDeadline { + sleepFor(0.005) + } + + XCTAssertLessThan( + box.elapsed ?? .infinity, + 0.5, + "the failed capture must not wait behind abandoned main-thread work" + ) + XCTAssertEqual( + box.bundleStillCachedWhileBlocked, + true, + "the drop must queue behind the blocked main thread, not run early" + ) + XCTAssertEqual(box.abandonedWhileBlocked, 1, "the deferred drop must not add an abandoned unit") + XCTAssertFalse(hasAbandonedMainThreadWork()) + XCTAssertNil(currentBundleId, "the drop must run once the main thread frees") + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotHittabilityTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotHittabilityTests.swift index 61e0024121..89828f62f7 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotHittabilityTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotHittabilityTests.swift @@ -97,7 +97,7 @@ extension RunnerTests { nodes: nodes, truncated: false, effectiveDepth: nil, - viewport: CGRect(x: 0, y: 0, width: 100, height: 100) + viewport: .reported(box: CGRect(x: 0, y: 0, width: 100, height: 100)) ), options: options ) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationConformanceTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationConformanceTests.swift index 718eb5651f..2c16b49851 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationConformanceTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationConformanceTests.swift @@ -97,7 +97,7 @@ extension RunnerTests { }, truncated: false, effectiveDepth: nil, - viewport: fixture.viewport.cgRect + viewport: .reported(box: fixture.viewport.cgRect) ) let options = PresentationOptions( interactiveOnly: false, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationGeometryTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationGeometryTests.swift index 087f00e510..db52ad5d2f 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationGeometryTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationGeometryTests.swift @@ -6,7 +6,7 @@ extension RunnerTests { func testEffectiveGeometryIntersectsViewportAndAncestorClip() { let effective = SnapshotGeometry.effectiveFrame( reportedFrame: CGRect(x: 350, y: 80, width: 100, height: 100), - viewport: CGRect(x: 0, y: 0, width: 402, height: 874), + viewport: .reported(box: CGRect(x: 0, y: 0, width: 402, height: 874)), ancestorClip: CGRect(x: 300, y: 100, width: 80, height: 80) ) @@ -17,7 +17,7 @@ extension RunnerTests { let reported = CGRect(x: 500, y: 120, width: 100, height: 44) let effective = SnapshotGeometry.effectiveFrame( reportedFrame: reported, - viewport: CGRect(x: 0, y: 0, width: 402, height: 874), + viewport: .reported(box: CGRect(x: 0, y: 0, width: 402, height: 874)), ancestorClip: nil ) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationInvariantTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationInvariantTests.swift index 65012e982d..9aeea4f490 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationInvariantTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationInvariantTests.swift @@ -97,7 +97,7 @@ extension RunnerTests { nodes: nodes, truncated: false, effectiveDepth: nil, - viewport: viewport + viewport: .reported(box: viewport) ) } @@ -147,7 +147,7 @@ extension RunnerTests { ], truncated: false, effectiveDepth: nil, - viewport: CGRect(x: 0, y: 0, width: 320, height: 240) + viewport: .reported(box: CGRect(x: 0, y: 0, width: 320, height: 240)) ) let options = PresentationOptions( @@ -203,7 +203,7 @@ extension RunnerTests { ], truncated: false, effectiveDepth: nil, - viewport: CGRect(x: 0, y: 0, width: 320, height: 240) + viewport: .reported(box: CGRect(x: 0, y: 0, width: 320, height: 240)) ) let options = PresentationOptions( @@ -254,7 +254,7 @@ extension RunnerTests { nodes: nodes, truncated: false, effectiveDepth: nil, - viewport: .infinite + viewport: .reported(box: CGRect(x: 0, y: 0, width: 100, height: 100)) ), options: options ).nodes) @@ -309,7 +309,7 @@ extension RunnerTests { XCTAssertThrowsError( try SnapshotPresentationInvariant.validateRegular( folded, - viewport: viewport, + viewport: .reported(box: viewport), policy: .cursorProjected ) ) { error in @@ -338,20 +338,6 @@ extension RunnerTests { XCTAssertEqual(Self.snapshotQualityReasonCode(for: captureFailure), "presentation-failed") XCTAssertNotEqual(Self.snapshotQualityReasonCode(for: captureFailure), "capture-failed") XCTAssertTrue(captureFailure.message.contains("cumulative clip")) - - let warning = Self.legacyQualityMessage( - SnapshotQuality( - state: "recovered", - backend: "queries", - reason: captureFailure.message, - reasonCode: captureFailure.qualityReasonCode, - effectiveDepth: nil, - collapsedLeafIndexes: nil, - customActions: nil - ) - ) - XCTAssertTrue(warning?.contains("runner bug") == true) - XCTAssertFalse(warning?.contains("fixing the app's accessibility") == true) } } #endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationTests.swift index c103678186..855abbd25b 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationTests.swift @@ -40,7 +40,7 @@ extension RunnerTests { truncated: 0, blocked: false ), - viewport: .infinite + viewport: .reported(box: CGRect(x: 0, y: 0, width: 100, height: 100)) ), options: PresentationOptions( interactiveOnly: true, @@ -113,7 +113,7 @@ extension RunnerTests { projection: .regular, depth: nil, regularPresentedDepth: nil, interactiveOnly: false, customActions: false), nodes: acquired, truncated: false, effectiveDepth: nil, - viewport: CGRect(x: 0, y: 0, width: 1_000, height: 1_000)), + viewport: .reported(box: CGRect(x: 0, y: 0, width: 1_000, height: 1_000))), options: PresentationOptions(interactiveOnly: false, depth: nil, scope: nil, raw: false) ).nodes ) @@ -124,7 +124,7 @@ extension RunnerTests { projection: .regular, depth: nil, regularPresentedDepth: nil, interactiveOnly: true, customActions: false), nodes: acquired, truncated: false, effectiveDepth: nil, - viewport: CGRect(x: 0, y: 0, width: 1_000, height: 1_000)), + viewport: .reported(box: CGRect(x: 0, y: 0, width: 1_000, height: 1_000))), options: PresentationOptions(interactiveOnly: true, depth: nil, scope: nil, raw: false) ).nodes ) @@ -147,7 +147,8 @@ extension RunnerTests { hint: CaptureHint( projection: .raw, depth: nil, regularPresentedDepth: nil, interactiveOnly: false, customActions: false), - nodes: acquired, truncated: false, effectiveDepth: nil, viewport: .infinite), + nodes: acquired, truncated: false, effectiveDepth: nil, + viewport: .reported(box: CGRect(x: 0, y: 0, width: 1_000, height: 1_000)) ), options: PresentationOptions(interactiveOnly: true, depth: nil, scope: nil, raw: true) ).nodes ) @@ -176,7 +177,7 @@ extension RunnerTests { nodes: nodes, truncated: false, effectiveDepth: nil, - viewport: CGRect(x: 0, y: 0, width: 100, height: 100) + viewport: .reported(box: CGRect(x: 0, y: 0, width: 100, height: 100)) ) let presented = try SnapshotPresentation.presentRegular( @@ -211,7 +212,7 @@ extension RunnerTests { nodes: acquired, truncated: false, effectiveDepth: nil, - viewport: viewport + viewport: .reported(box: viewport) ), options: regularOptions ).nodes @@ -231,7 +232,7 @@ extension RunnerTests { nodes: acquired, truncated: false, effectiveDepth: nil, - viewport: .infinite + viewport: .reported(box: CGRect(x: 0, y: 0, width: 100, height: 100)) ), options: rawOptions ).nodes @@ -286,7 +287,7 @@ extension RunnerTests { ], truncated: false, effectiveDepth: nil, - viewport: CGRect(x: 0, y: 0, width: 1_000, height: 1_000) + viewport: .reported(box: CGRect(x: 0, y: 0, width: 1_000, height: 1_000)) ) let options = PresentationOptions( interactiveOnly: true, @@ -313,7 +314,7 @@ extension RunnerTests { nodes: acquisition.nodes, truncated: false, effectiveDepth: nil, - viewport: .infinite + viewport: .reported(box: CGRect(x: 0, y: 0, width: 100, height: 100)) ), options: PresentationOptions( interactiveOnly: true, @@ -374,10 +375,11 @@ extension RunnerTests { let regularAcquisition = SnapshotAcquisition( hint: SnapshotPresentation.captureHint(for: regularRequest), nodes: nodes, truncated: false, effectiveDepth: nil, - viewport: CGRect(x: 0, y: 0, width: 100, height: 100)) + viewport: .reported(box: CGRect(x: 0, y: 0, width: 100, height: 100))) let rawAcquisition = SnapshotAcquisition( hint: SnapshotPresentation.captureHint(for: rawRequest), - nodes: nodes, truncated: false, effectiveDepth: nil, viewport: .infinite) + nodes: nodes, truncated: false, effectiveDepth: nil, + viewport: .reported(box: CGRect(x: 0, y: 0, width: 100, height: 100))) let regularCaptureForRawRequest = try SnapshotPresentation.present( regularAcquisition, options: rawRequest) @@ -470,7 +472,7 @@ extension RunnerTests { ], truncated: false, effectiveDepth: nil, - viewport: CGRect(x: 0, y: 0, width: 100, height: 100) + viewport: .reported(box: CGRect(x: 0, y: 0, width: 100, height: 100)) ), options: options ) @@ -552,7 +554,7 @@ extension RunnerTests { nodes: nodes, truncated: false, effectiveDepth: nil, - viewport: viewport + viewport: .reported(box: viewport) ) let presented = try XCTUnwrap( SnapshotPresentation.present(acquisition, options: options)?.nodes) @@ -596,8 +598,10 @@ extension RunnerTests { ] let normalized = SnapshotGeometrySpace.normalized( nodes: acquired, - viewport: viewport, - interfaceOrientation: RunnerInterfaceOrientation.landscapeRight + viewport: .reported( + box: viewport, + interfaceOrientation: RunnerInterfaceOrientation.landscapeRight + ) ) let options = PresentationOptions(interactiveOnly: false, depth: 3, scope: nil, raw: false) let hint = SnapshotPresentation.captureHint(for: options) @@ -608,7 +612,7 @@ extension RunnerTests { nodes: normalized, truncated: false, effectiveDepth: nil, - viewport: viewport + viewport: .reported(box: viewport) ), options: options )?.nodes diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotQualityStateTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotQualityStateTests.swift new file mode 100644 index 0000000000..8ef84b8fc8 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotQualityStateTests.swift @@ -0,0 +1,68 @@ +import XCTest + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +extension RunnerTests { + private struct StampedWireVerdict: Decodable { + struct Quality: Decodable { + let state: String + } + let snapshotQuality: Quality + } + + private func loadSnapshotQualityStatesFixture() throws -> [String] { + let fixtureURL = URL(fileURLWithPath: #filePath) + .deletingLastPathComponent() // UnitTests + .deletingLastPathComponent() // AgentDeviceRunnerUITests + .deletingLastPathComponent() // AgentDeviceRunner + .deletingLastPathComponent() // runner + .deletingLastPathComponent() // apple + .deletingLastPathComponent() // repo root + .appendingPathComponent("contracts") + .appendingPathComponent("fixtures") + .appendingPathComponent("ios-snapshot-quality-states.json") + return try JSONDecoder().decode([String].self, from: Data(contentsOf: fixtureURL)) + } + + /// The one claim of this file: the runner's closed enum and the shared TypeScript table name the + /// same states. The kernel's `SNAPSHOT_QUALITY_STATES` is pinned to it too, so the two runtimes + /// cannot drift into a verdict the host drops along with its disclosure. Compared as a set: the + /// names are the contract, and a reordering of `allCases` cannot produce a wrong verdict. + func testSnapshotQualityStatesMatchSharedWireFixture() throws { + XCTAssertEqual( + Set(try loadSnapshotQualityStatesFixture()), + Set(SnapshotQualityState.allCases.map(\.rawValue)), + "update the fixture and the kernel tuple together with the enum" + ) + } + + /// What the daemon receives for each state, taken from the production stamping path rather than a + /// hand-built verdict: the wire string is the case's own raw value, so a change of + /// representation — an `Int` backing, a nested object — goes red here on the actual payload, and + /// a renamed raw value goes red in the fixture test above. + func testStampedVerdictEncodesTheCaseRawValue() throws { + let capture = SnapshotBackendCapture( + payload: DataPayload(nodes: [], truncated: false), + effectiveDepth: nil + ) + for state in SnapshotQualityState.allCases { + let payload = stampedSnapshotPayload( + capture, + backend: .recursiveTree, + state: state, + reason: nil + ) + let wire = try JSONDecoder().decode( + StampedWireVerdict.self, + from: JSONEncoder().encode(payload) + ) + XCTAssertEqual(wire.snapshotQuality.state, state.rawValue) + } + } + + /// Closed in both directions: a wire string nobody declared never becomes a verdict. + func testVerdictStateRejectsAnUndeclaredWireString() throws { + let json = Data(#"{"state":"degraded","backend":"tree"}"#.utf8) + XCTAssertThrowsError(try JSONDecoder().decode(SnapshotQuality.self, from: json)) + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotTests.swift new file mode 100644 index 0000000000..20b3b9631e --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotTests.swift @@ -0,0 +1,354 @@ +import XCTest +import AgentDeviceSnapshotPresentation + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +extension RunnerTests { + func testSnapshotAccessibilityUnavailableMarksSparseSnapshotRunnerFatal() { + currentApp = app + currentBundleId = "com.example.app" + + let payload = snapshotAccessibilityUnavailable( + failure: SnapshotCaptureFailure( + code: Self.axSnapshotErrorCode, + message: Self.axSnapshotFailureMessage, + hint: Self.axSnapshotHint + ) + ) + + XCTAssertEqual(payload.message, "\(Self.axSnapshotFailureMessage) Hint: \(Self.axSnapshotHint)") + XCTAssertEqual(payload.nodes?.count, 1) + XCTAssertEqual(payload.nodes?.first?.type, "Application") + XCTAssertEqual(payload.truncated, true) + XCTAssertEqual(payload.runnerFatal, true) + XCTAssertEqual(payload.runnerFatalReason, Self.axSnapshotUnavailableReason) + // The planned terminal result carries the structured verdict like every other planned + // snapshot — downstream sparse handling keys off it, not off node shapes. + XCTAssertEqual(payload.snapshotQuality?.state, .sparse) + XCTAssertEqual(payload.snapshotQuality?.reasonCode, "ax-rejected") + XCTAssertEqual(payload.snapshotQuality?.reason, Self.axSnapshotFailureMessage) + XCTAssertNil(currentApp) + XCTAssertNil(currentBundleId) + } + + func testRecoveredSnapshotMessagePreservesHint() { + let message = recoveredSnapshotMessage( + SnapshotCaptureFailure( + code: Self.axSnapshotErrorCode, + message: Self.axSnapshotFailureMessage, + hint: Self.axSnapshotHint + ) + ) + + XCTAssertTrue(message.contains(Self.axSnapshotFailureMessage)) + XCTAssertTrue(message.contains(Self.axSnapshotHint)) + } + + func testRawSnapshotTooLargeFailureIsStructured() { + let failure = rawSnapshotTooLargeFailure(nodeCount: Self.rawSnapshotMaxNodes + 1) + + XCTAssertEqual(failure.code, Self.rawSnapshotTooLargeCode) + XCTAssertTrue(failure.message.contains("\(Self.rawSnapshotMaxNodes) nodes")) + XCTAssertEqual(failure.hint, Self.rawSnapshotTooLargeHint) + } + + func testSystemModalProbeSliceSharesAndClampsToPlanDeadline() { + // Fresh plan deadline: the probe gets its full dedicated budget. + XCTAssertEqual(Self.systemModalProbeSlice(budget: 4, deadlineRemaining: 20), 4) + // Nearly-spent plan deadline: the probe is clamped so it can't run past the shared budget. + XCTAssertEqual(Self.systemModalProbeSlice(budget: 4, deadlineRemaining: 1.5), 1.5) + // Exactly/already exhausted deadline: skip the probe entirely (0), never a negative timeout. + XCTAssertEqual(Self.systemModalProbeSlice(budget: 4, deadlineRemaining: 0), 0) + XCTAssertEqual(Self.systemModalProbeSlice(budget: 4, deadlineRemaining: -5), 0) + } + + func testSnapshotAccessibilityUnavailableQueuesInvalidationBehindAbandonedMainThreadWork() { + currentBundleId = "com.example.stale-target" + runnerAccessibilityHealth = .healthy + defer { + currentBundleId = nil + runnerAccessibilityHealth = .unknown + } + + final class ResultBox { + var elapsed: TimeInterval? + var payload: DataPayload? + var bundleStillCachedWhileBlocked: Bool? + var healthWhileBlocked: RunnerAccessibilityHealth? + var abandonedWhileBlocked: Int? + } + let box = ResultBox() + let mainBlocked = DispatchSemaphore(value: 0) + let releaseMain = DispatchSemaphore(value: 0) + let finished = expectation(description: "fail-closed capture returned while main was blocked") + + DispatchQueue(label: "agent-device.runner.tests.ax-unavailable-invalidation").async { + _ = try? self.runMainThreadWork( + "command_execution", + timeout: 0, + timeoutError: self.mainThreadExecutionTimeoutError + ) { + mainBlocked.signal() + _ = releaseMain.wait(timeout: .now() + 5) + return true + } + _ = mainBlocked.wait(timeout: .now() + 2) + let startedAt = Date() + box.payload = self.snapshotAccessibilityUnavailable( + failure: SnapshotCaptureFailure( + code: Self.axSnapshotErrorCode, + message: Self.axSnapshotFailureMessage, + hint: Self.axSnapshotHint + ) + ) + box.elapsed = Date().timeIntervalSince(startedAt) + box.bundleStillCachedWhileBlocked = self.currentBundleId != nil + box.healthWhileBlocked = self.runnerAccessibilityHealth + self.mainThreadWorkLock.lock() + box.abandonedWhileBlocked = self.abandonedMainThreadWorkCount + self.mainThreadWorkLock.unlock() + releaseMain.signal() + finished.fulfill() + } + + wait(for: [finished], timeout: 8) + let drainDeadline = Date().addingTimeInterval(2) + while hasAbandonedMainThreadWork() || currentBundleId != nil, Date() < drainDeadline { + sleepFor(0.005) + } + + XCTAssertEqual(box.payload?.runnerFatal, true) + XCTAssertLessThan( + box.elapsed ?? .infinity, + 0.5, + "the fail-closed capture must not wait behind abandoned main-thread work" + ) + XCTAssertEqual( + box.bundleStillCachedWhileBlocked, + true, + "the invalidation must queue behind the blocked main thread, not run on the command queue" + ) + XCTAssertEqual( + box.healthWhileBlocked, + .healthy, + "the health write must queue behind the blocked main thread, not run on the command queue" + ) + XCTAssertEqual(box.abandonedWhileBlocked, 1, "the deferred write must not add an abandoned unit") + XCTAssertFalse(hasAbandonedMainThreadWork()) + XCTAssertNil(currentBundleId, "the invalidation must run once the main thread frees") + XCTAssertEqual(runnerAccessibilityHealth, .unavailable) + } + + func testQuerySweepSliceDeadlineIsTheTierSliceNotThePlanDeadline() { + let startedAt = Date(timeIntervalSinceReferenceDate: 1_000) + let slice = startedAt.addingTimeInterval(Self.flatInteractiveFallbackBudget) + + XCTAssertEqual( + Self.querySweepSliceDeadline( + startedAt: startedAt, + planDeadline: startedAt.addingTimeInterval(Self.snapshotPlanBudget) + ), + slice + ) + let nearPlanDeadline = startedAt.addingTimeInterval(Self.flatInteractiveFallbackBudget / 2) + XCTAssertEqual( + Self.querySweepSliceDeadline(startedAt: startedAt, planDeadline: nearPlanDeadline), + nearPlanDeadline + ) + } + + func testQuerySweepStopsAtTheSliceDeadlineBeforeAQueryThatCannotFinish() { + let startedAt = Date(timeIntervalSinceReferenceDate: 1_000) + let sliceDeadline = Self.querySweepSliceDeadline( + startedAt: startedAt, + planDeadline: startedAt.addingTimeInterval(Self.snapshotPlanBudget) + ) + var clock = startedAt + var startedQueries: [Int] = [] + + let sweep = Self.runFlatInteractiveQueries( + Array(0..<19), + deadline: sliceDeadline, + now: { clock } + ) { query -> (elements: [Int], axUnavailable: Bool) in + startedQueries.append(query) + clock = clock.addingTimeInterval(0.475) + return ([query], false) + } + + XCTAssertEqual(startedQueries, [0, 1], "a query must not start with less than one query budget left") + XCTAssertEqual(sweep.elements, [0, 1]) + XCTAssertEqual(sweep.outcome, .deadlineExhausted) + XCTAssertLessThanOrEqual(clock, sliceDeadline, "no started query may outlive the tier slice") + + XCTAssertTrue( + Self.querySweepCanStartQuery(deadline: sliceDeadline, now: sliceDeadline.addingTimeInterval(-0.2)) + ) + XCTAssertFalse( + Self.querySweepCanStartQuery(deadline: sliceDeadline, now: sliceDeadline.addingTimeInterval(-0.05)) + ) + XCTAssertFalse(Self.querySweepCanStartQuery(deadline: sliceDeadline, now: sliceDeadline)) + } + + func testQuerySweepRunsEveryQueryThatFitsAndStopsOnAXUnavailable() { + let startedAt = Date(timeIntervalSinceReferenceDate: 1_000) + let deadline = startedAt.addingTimeInterval(Self.flatInteractiveFallbackBudget) + + let complete = Self.runFlatInteractiveQueries( + [0, 1, 2], + deadline: deadline, + now: { startedAt } + ) { query -> (elements: [Int], axUnavailable: Bool) in ([query], false) } + XCTAssertEqual(complete.elements, [0, 1, 2]) + XCTAssertEqual(complete.outcome, .completed) + + let rejected = Self.runFlatInteractiveQueries( + [0, 1, 2], + deadline: deadline, + now: { startedAt } + ) { query -> (elements: [Int], axUnavailable: Bool) in ([query], query == 1) } + XCTAssertEqual(rejected.elements, [0, 1]) + XCTAssertEqual( + rejected.outcome, + .completed, + "an AX refusal ends the sweep on its own terms, not on the deadline" + ) + } + + // Simulator-only: the bounded probe body returns nil on macOS (no SpringBoard host), so the + // timeout/penalty/drain machinery below only exists on the iOS branch. +#if os(iOS) + /// Regression for #1244/#1248: drives the bounded system-modal probe through a real, + /// production-only command entry point (`snapshotFast` or `snapshotRaw` -- see the two test + /// methods below), not `boundedBlockingSystemAlertSnapshot` directly, with + /// `systemModalProbeOverrideForTesting` set to a closure that blocks past the probe's real + /// slice, forcing a real `runMainThreadWork` timeout. This is revert-sensitive on both halves + /// of the fix, for either entry point: + /// - if the entry point reverted to calling the unbounded `blockingSystemAlertSnapshot` + /// directly (or dropped the `runMainThreadWork` wrap), nothing here would ever time out, + /// so the mid-flight busy/penalty assertions below would never be met; + /// - if the `onAbandoned` penalty hook or the abandoned-work accounting were dropped, the + /// timeout would still fire, but the busy/penalty and drain assertions would not hold. + /// + /// The drain assertion is synchronized on the *real* release rather than raced: after + /// signaling the probe to finish, the background queue polls `hasAbandonedMainThreadWork()` + /// (bounded) and only then fulfills `drained`, which the test `wait(for:timeout:)`s on before + /// asserting `.idle`/`hasAbandonedMainThreadWork() == false` below -- so a slow drain fails that + /// assertion instead of racing a fixed-timing guess. + private func assertBoundedSystemModalProbeTimeoutRecoversThenReleasesOnDrain( + entryPointName: String, + callEntryPoint: @escaping (SnapshotCaptureTarget, PresentationOptions) throws -> DataPayload + ) { + let targetBundleId = "com.callstack.agentdevice.runner.missing.snapshot-timeout-test" + let snapshotTarget = XCUIApplication(bundleIdentifier: targetBundleId) + let probeReleaseGate = DispatchSemaphore(value: 0) + currentApp = snapshotTarget + currentBundleId = targetBundleId + let captureTarget = takeSnapshotCaptureTarget(app: snapshotTarget) + defer { + probeReleaseGate.signal() + currentApp = nil + currentBundleId = nil + systemModalProbeOverrideForTesting = nil + clearSnapshotXCTestChannelPenalty(reason: "test-cleanup") + } + + final class ResultBox { + var payload: DataPayload? + var wasBusyBeforeDrain = false + var hadAbandonedCaptureBeforeDrain = false + var wasPenalizedBeforeDrain = false + } + let box = ResultBox() + // The test owns release of the injected probe. A fixed timeout races the capture plan's + // independent fallback tiers on loaded CI hosts and can drain before the test records the + // abandoned-work state. The defer above still releases the probe if an earlier assertion or + // expectation fails. + systemModalProbeOverrideForTesting = { _ in + probeReleaseGate.wait() + return nil + } + + let completion = expectation( + description: "\(entryPointName) recovered while the probe was abandoned, then released it" + ) + let drained = expectation(description: "\(entryPointName) modal probe drained") + DispatchQueue(label: "agent-device.runner.tests.modal-probe-timeout").async { + box.payload = try? callEntryPoint( + captureTarget, + PresentationOptions(interactiveOnly: false, depth: nil, scope: nil, raw: false) + ) + + // 1) Penalty/busy accounting: must already be in place by the time the entry point + // returns, well before we release the still-blocked probe below. + if case .busy = self.currentMainThreadBusyState() { + box.wasBusyBeforeDrain = true + } + box.hadAbandonedCaptureBeforeDrain = self.hasAbandonedMainThreadWork() + box.wasPenalizedBeforeDrain = self.isSnapshotXCTestChannelPenalized(bundleId: targetBundleId) + + // 2) `box.payload` above was already produced -- through the capture plan's recovery + // tiers -- while the probe is still blocked on `probeReleaseGate`, i.e. recovered before + // drain, not queued behind it. + completion.fulfill() + + // 3) Only now let the abandoned probe finish, then block this queue (never the test's + // main-thread wait) on the *real* drain signal -- the abandoned-work count reaching zero + // -- bounded so a revert that never drains fulfills `drained` anyway and lets the + // assertions below report the regression explicitly instead of just timing out. + probeReleaseGate.signal() + let drainDeadline = Date().addingTimeInterval(5) + while self.hasAbandonedMainThreadWork(), Date() < drainDeadline { + self.sleepFor(0.002) + } + drained.fulfill() + } + + wait(for: [completion], timeout: 15) + + // 1) Penalty/busy accounting. + XCTAssertTrue( + box.wasBusyBeforeDrain, + "expected RUNNER_BUSY while the \(entryPointName) modal probe timeout is outstanding" + ) + XCTAssertTrue( + box.hadAbandonedCaptureBeforeDrain, + "onAbandoned must retain the abandoned XCTest channel work for \(entryPointName)" + ) + XCTAssertTrue( + box.wasPenalizedBeforeDrain, + "a timed-out modal probe must penalize the XCTest snapshot channel for \(entryPointName)" + ) + + // 2) Recovered response before drain. + XCTAssertNotNil( + box.payload, + "\(entryPointName) must recover a payload through the capture plan while the probe drains" + ) + + // 3) Bounded, deterministic drain barrier, then release assertions. + wait(for: [drained], timeout: 6) + guard case .idle = currentMainThreadBusyState() else { + return XCTFail("expected the runner to be idle once the abandoned \(entryPointName) probe drained") + } + XCTAssertFalse( + hasAbandonedMainThreadWork(), + "the drained probe must release the main thread for \(entryPointName)" + ) + } + + func testBoundedSystemModalProbeTimeoutRecoversThenReleasesOnDrain() { + assertBoundedSystemModalProbeTimeoutRecoversThenReleasesOnDrain(entryPointName: "snapshotFast") { + target, options in + try self.snapshotFast(target: target, options: options) + } + } + + func testBoundedSystemModalProbeTimeoutRecoversThenReleasesOnDrainForSnapshotRaw() { + assertBoundedSystemModalProbeTimeoutRecoversThenReleasesOnDrain(entryPointName: "snapshotRaw") { + target, options in + try self.snapshotRaw(target: target, options: options) + } + } +#endif +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotTimingTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotTimingTests.swift new file mode 100644 index 0000000000..b815c96725 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotTimingTests.swift @@ -0,0 +1,123 @@ +import Foundation +import XCTest + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +extension RunnerTests { + func testXCTestPenaltyDecisionSeparatesAcquisitionAndPresentation() { + let slowPresentation = SnapshotBackendAttempt( + outcome: .noCapture, + timing: SnapshotCaptureTiming(acquisitionMs: 100, presentationMs: 4_000) + ) + XCTAssertNil( + Self.snapshotXCTestPenaltyReason( + kind: .recursiveTree, + attempt: slowPresentation, + slowThresholdMs: 3_000 + ) + ) + + let slowAcquisition = SnapshotBackendAttempt( + outcome: .noCapture, + timing: SnapshotCaptureTiming(acquisitionMs: 3_001, presentationMs: 100) + ) + XCTAssertEqual( + Self.snapshotXCTestPenaltyReason( + kind: .recursiveTree, + attempt: slowAcquisition, + slowThresholdMs: 3_000 + ), + "slow_tree_capture_3001ms" + ) + + let timeout = SnapshotCaptureFailure( + code: Self.xCTestSnapshotTimeoutCode, + message: "test timeout", + hint: "test" + ) + let acquisitionFailure = SnapshotBackendAttempt( + outcome: .failed(timeout, phase: .acquisition), + timing: SnapshotCaptureTiming(acquisitionMs: 100, presentationMs: 100) + ) + XCTAssertEqual( + Self.snapshotXCTestPenaltyReason( + kind: .recursiveTree, + attempt: acquisitionFailure, + slowThresholdMs: 3_000 + ), + "tree_backend_timeout" + ) + + let presentationFailure = SnapshotBackendAttempt( + outcome: .failed(timeout, phase: .presentation), + timing: SnapshotCaptureTiming(acquisitionMs: 100, presentationMs: 100) + ) + XCTAssertNil( + Self.snapshotXCTestPenaltyReason( + kind: .recursiveTree, + attempt: presentationFailure, + slowThresholdMs: 3_000 + ) + ) + } + + /// A tier that stopped starting work at its own deadline is a timeout for the breaker even when it + /// answered fast and with a payload: the partial sweep must arm the penalty exactly as a slow tier + /// does, and a tier that finished fast must not (#2781). + func testDeadlineExhaustedTierAttemptArmsTheChannelPenaltyWithoutBeingSlow() { + let capture = SnapshotBackendCapture( + payload: DataPayload(nodes: [], truncated: true), + effectiveDepth: nil + ) + let exhausted = SnapshotBackendAttempt( + outcome: .captured(capture), + timing: SnapshotCaptureTiming(acquisitionMs: 1_000, presentationMs: 10), + tierOutcome: .deadlineExhausted + ) + XCTAssertEqual( + Self.snapshotXCTestPenaltyReason( + kind: .querySweep, + attempt: exhausted, + slowThresholdMs: 3_000 + ), + "queries_backend_timeout" + ) + + let completed = SnapshotBackendAttempt( + outcome: .captured(capture), + timing: SnapshotCaptureTiming(acquisitionMs: 1_000, presentationMs: 10), + tierOutcome: .completed + ) + XCTAssertNil( + Self.snapshotXCTestPenaltyReason( + kind: .querySweep, + attempt: completed, + slowThresholdMs: 3_000 + ) + ) + + XCTAssertNil( + Self.snapshotXCTestPenaltyReason( + kind: .privateAX, + attempt: exhausted, + slowThresholdMs: 3_000 + ), + "a tier that owes nothing to the XCTest channel cannot penalize it" + ) + } + + func testSnapshotPhaseTimerReportsAcquisitionAndPresentationSeparately() { + var now = Date(timeIntervalSinceReferenceDate: 100) + var timer = SnapshotPhaseTimer(now: { now }) + + _ = timer.measure(.acquisition) { + now = now.addingTimeInterval(2) + } + _ = timer.measure(.presentation) { + now = now.addingTimeInterval(5) + } + + XCTAssertEqual(timer.timing.acquisitionMs, 2_000, accuracy: 0.001) + XCTAssertEqual(timer.timing.presentationMs, 5_000, accuracy: 0.001) + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotVisibilityFoldTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotVisibilityFoldTests.swift index 2063c56f14..ee6c4400dd 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotVisibilityFoldTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotVisibilityFoldTests.swift @@ -25,7 +25,11 @@ extension RunnerTests { policy: SnapshotVisibilityFold.Policy = .cursorProjected ) -> [SnapshotPresentationNode] { SnapshotVisibilityFold.fold( - nodes, viewport: viewport, interactiveOnly: interactiveOnly, policy: policy) + nodes, + viewport: .reported(box: viewport), + interactiveOnly: interactiveOnly, + policy: policy + ) } func testRegularFoldClipsScrollOverflowReparentsAndBooksHints() { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedCommitDeadlineTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedCommitDeadlineTests.swift index 840a6b0ef4..f4be5b61e7 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedCommitDeadlineTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedCommitDeadlineTests.swift @@ -2,7 +2,7 @@ import XCTest extension RunnerTests { #if AGENT_DEVICE_RUNNER_UNIT_TESTS - /// A hand-driven clock for the commit waits. Time moves only where the wait sleeps, which is + /// A hand-driven clock for the commit wait. Time moves only where the wait sleeps, which is /// what makes "the burst kept landing" and "the pipeline froze" expressible as two sequences of /// the same length rather than as wall-clock luck — and a test that never advances it cannot /// expire any budget, so only a test asking about time names `stallBudget`/`ceiling`. @@ -29,7 +29,7 @@ extension RunnerTests { let expected = "hardware" let clock = CommitWaitClock() var landed = 0 - let outcome = Self.awaitSynthesizedCommitOutcome( + let outcome = Self.awaitSynthesizedReplacementCommitOutcome( expectedText: expected, placeholder: nil, stallBudget: 3, @@ -55,7 +55,7 @@ extension RunnerTests { let clock = CommitWaitClock() clock.advance(60) var polls = 0 - let outcome = Self.awaitSynthesizedCommitOutcome( + let outcome = Self.awaitSynthesizedReplacementCommitOutcome( expectedText: "hardware", placeholder: nil, stallBudget: 3, @@ -76,7 +76,7 @@ extension RunnerTests { // fails today starts passing merely by waiting longer. func testCommitWaitCondemnsAFrozenPipelineAtTheStallBudget() { let clock = CommitWaitClock() - let outcome = Self.awaitSynthesizedCommitOutcome( + let outcome = Self.awaitSynthesizedReplacementCommitOutcome( expectedText: "hardware", placeholder: nil, stallBudget: 3, @@ -89,32 +89,9 @@ extension RunnerTests { XCTAssertEqual(clock.elapsed, 3, "a frozen prefix must give up on the stall budget, not the ceiling") } - // Progress buys time, but not without bound: one character per stall window would otherwise - // hold the command open until the daemon's own 45s budget killed the request. Here every poll - // lands a character, so only the ceiling can stop it. - func testCommitWaitCeilingStopsAnIndefinitelyThrottledPipeline() { - let clock = CommitWaitClock() - var landed = 0 - let outcome = Self.awaitSynthesizedCommitOutcome( - expectedText: String(repeating: "a", count: 100), - placeholder: nil, - stallBudget: 3, - ceiling: 10, - now: clock.read, - observe: { String(repeating: "a", count: landed) }, - waitForNextObservation: { - landed += 1 - clock.advance(2) - } - ) - XCTAssertEqual(outcome, .notObserved) - XCTAssertEqual(clock.elapsed, 10, "the ceiling is absolute, however long characters keep arriving") - } - // Only forward movement is evidence the burst is still landing. A field the app clears // mid-flight would otherwise reset the stall clock on every poll and hold every wedged wait - // open to the ceiling. Replacement mode, because that is where a non-matching value keeps - // polling rather than settling as `.diverged`. + // open to the ceiling. func testCommitWaitTreatsARetreatingValueAsNoProgress() { let clock = CommitWaitClock() let observations = ["ada@", "", "ada@", "", "ada@"] @@ -135,11 +112,11 @@ extension RunnerTests { XCTAssertEqual(clock.elapsed, 3, "churn between two values is not progress and must not buy time") } - // The replacement route earns time the same way, and is bounded the same way — it is the route - // `fill` takes when the XCTest channel is penalized, i.e. the one that runs on a loaded host. - // Here every poll lands one more character of a value that never completes, so the wait can only - // end at the ceiling: it proves the growing prefix carried it past the 3s stall budget (a flat - // deadline stops at t=3) and that the ceiling still stops it. + // Progress buys time, but not without bound: one character per stall window would otherwise + // hold the command open until the daemon's own 45s budget killed the request. Here every poll + // lands one more character of a value that never completes, so the wait can only end at the + // ceiling: it proves the growing prefix carried it past the 3s stall budget (a flat deadline + // stops at t=3) and that the ceiling still stops it. func testReplacementCommitWaitOutlivesTheFlatDeadlineThenStopsAtTheCeiling() { let expected = "ada@example.com" let clock = CommitWaitClock() diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGesturePolicyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGesturePolicyTests.swift new file mode 100644 index 0000000000..ed7d2cb4ab --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGesturePolicyTests.swift @@ -0,0 +1,176 @@ +import XCTest + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS) +extension RunnerTests { + func testSynthesizedGesturePolicyMarkerWritesOncePerKindUntilTheDecisionChanges() { + var written: [String] = [] + runnerMarkerWriter = { written.append($0) } + defer { + runnerMarkerWriter = { NSLog("%@", $0) } + invalidateCachedTarget(reason: "unit_test_cleanup") + } + logSynthesizedGesturePolicyDecision(kind: .coordinateTap, context: nil, fallbackAttempted: false) + logSynthesizedGesturePolicyDecision(kind: .coordinateTap, context: nil, fallbackAttempted: false) + XCTAssertEqual(written.count, 1, "a repeated decision writes no second line") + logSynthesizedGesturePolicyDecision(kind: .scroll, context: nil, fallbackAttempted: false) + XCTAssertEqual(written.count, 2, "each gesture kind states its own decision") + logSynthesizedGesturePolicyDecision(kind: .coordinateTap, context: nil, fallbackAttempted: true) + XCTAssertEqual(written.count, 3, "a changed decision writes a new line") + resetTargetBoundState() + logSynthesizedGesturePolicyDecision(kind: .coordinateTap, context: nil, fallbackAttempted: true) + XCTAssertEqual(written.count, 4, "a rebind states the same decision once more") + } +} +#endif + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +extension RunnerTests { + func testSynthesizedFallbackPolicyRequiresPrivateSynthesisForScrollWhenAxUnavailableOrUnknown() { + XCTAssertFalse( + SynthesizedFallbackPolicy.privateSynthesisRequired + .allowsXCTestCoordinateFallback(accessibilityHealth: .unavailable) + ) + XCTAssertFalse( + SynthesizedFallbackPolicy.privateSynthesisRequired + .allowsXCTestCoordinateFallback(accessibilityHealth: .unknown) + ) + XCTAssertFalse( + SynthesizedFallbackPolicy.privateSynthesisRequired + .allowsXCTestCoordinateFallback(accessibilityHealth: .healthy) + ) + } + + func testSynthesizedDragCoordinateFallbackAllowsUnknownButNotUnavailableAccessibility() { + XCTAssertTrue( + SynthesizedFallbackPolicy.xctestCoordinateWhenAccessibilityAvailable + .allowsXCTestCoordinateFallback(accessibilityHealth: .healthy) + ) + XCTAssertFalse( + SynthesizedFallbackPolicy.xctestCoordinateWhenAccessibilityAvailable + .allowsXCTestCoordinateFallback(accessibilityHealth: .unavailable) + ) + XCTAssertTrue( + SynthesizedFallbackPolicy.xctestCoordinateWhenAccessibilityAvailable + .allowsXCTestCoordinateFallback(accessibilityHealth: .unknown) + ) + } + + /// Keyboard-policy semantics only. Which command gets which policy is the table below; a probe + /// that is merely permitted still costs a live AX fetch, so the two questions stay separate. + func testSynthesizedKeyboardPolicyAllowsProbeOnlyWhenAccessibilityPermitsIt() { + XCTAssertFalse(SynthesizedKeyboardPolicy.never.allowsProbe(accessibilityHealth: .healthy)) + XCTAssertTrue( + SynthesizedKeyboardPolicy.requiredWhenAvailable + .allowsProbe(accessibilityHealth: .unknown) + ) + XCTAssertFalse( + SynthesizedKeyboardPolicy.requiredWhenAvailable + .allowsProbe(accessibilityHealth: .unavailable) + ) + } + + func testSynthesizedGesturePoliciesMatchCommandContracts() { + XCTAssertEqual( + synthesizedGesturePolicy(.coordinateTap), + SynthesizedGesturePolicy( + keyboardPolicy: .never, + fallbackPolicy: .xctestCoordinateAllowed + ) + ) + XCTAssertEqual( + synthesizedGesturePolicy(.scroll), + SynthesizedGesturePolicy( + keyboardPolicy: .requiredWhenAvailable, + fallbackPolicy: .privateSynthesisRequired + ) + ) + XCTAssertEqual( + synthesizedGesturePolicy(.synthesizedDrag), + SynthesizedGesturePolicy( + keyboardPolicy: .requiredWhenAvailable, + fallbackPolicy: .xctestCoordinateWhenAccessibilityAvailable + ) + ) + } + + func testCoordinateTapTextInputProbeSkipsPenalizedXCTestChannel() { + XCTAssertTrue(shouldProbeCoordinateTapTextInput(xCTestChannelPenalized: false)) + XCTAssertFalse(shouldProbeCoordinateTapTextInput(xCTestChannelPenalized: true)) + } + + func testOnlySynthesizedSequenceTapStepsTakeTheCoordinateTapPolicy() { + XCTAssertEqual(synthesizedPolicyKind(forSequenceStep: sequenceStep("tap", synthesized: true)), .coordinateTap) + XCTAssertNil(synthesizedPolicyKind(forSequenceStep: sequenceStep("tap", synthesized: nil))) + XCTAssertNil(synthesizedPolicyKind(forSequenceStep: sequenceStep("doubleTap", synthesized: true))) + XCTAssertNil(synthesizedPolicyKind(forSequenceStep: sequenceStep("longPress", synthesized: true))) + } + + func testFailedCoordinateTapSynthesisFallsBackToXCTestAtEveryAccessibilityHealth() { + for health: RunnerAccessibilityHealth in [.unknown, .healthy, .unavailable] { + runnerAccessibilityHealth = health + for context in [nil, synthesizedGestureTestContext(accessibilityHealth: health)] { + let label = "axHealth=\(health.rawValue) context=\(context == nil ? "unresolved" : "resolved")" + let attempt = performSynthesizedGesture( + app, + kind: .coordinateTap, + context: context, + synthesize: failedSynthesis + ) + XCTAssertEqual(synthesizedGestureRoute(attempt), "xctestFallback", label) + } + } + } + + func testSynthesizedGestureFallbackFollowsItsKindAndTheResolvedAccessibilityHealth() { + runnerAccessibilityHealth = .healthy + let unavailable = synthesizedGestureTestContext(accessibilityHealth: .unavailable) + let unknown = synthesizedGestureTestContext(accessibilityHealth: .unknown) + let cases: [(SynthesizedGesturePolicyKind, SynthesizedCoordinateContext?, String)] = [ + (.scroll, synthesizedGestureTestContext(accessibilityHealth: .healthy), "refused"), + (.synthesizedDrag, unavailable, "refused"), + (.synthesizedDrag, unknown, "xctestFallback"), + (.coordinateTap, unavailable, "xctestFallback"), + ] + for (kind, context, expected) in cases { + let attempt = performSynthesizedGesture(app, kind: kind, context: context, synthesize: failedSynthesis) + XCTAssertEqual(synthesizedGestureRoute(attempt), expected, "kind=\(kind.rawValue)") + switch attempt { + case .xctestFallback(let message, let hint), .refused(_, let message, let hint): + XCTAssertEqual(message, "forced private synthesis failure") + XCTAssertEqual(hint, "forced hint") + case .performed: + break + } + let performed = performSynthesizedGesture(app, kind: kind, context: context) { .performed } + XCTAssertEqual(synthesizedGestureRoute(performed), "performed", "kind=\(kind.rawValue)") + } + } + + private func failedSynthesis() -> RunnerInteractionOutcome { + .unsupported(message: "forced private synthesis failure", hint: "forced hint") + } + + private func synthesizedGestureRoute(_ attempt: SynthesizedGestureAttempt) -> String { + switch attempt { + case .performed: return "performed" + case .xctestFallback: return "xctestFallback" + case .refused: return "refused" + } + } + + private func synthesizedGestureTestContext( + accessibilityHealth: RunnerAccessibilityHealth + ) -> SynthesizedCoordinateContext { + SynthesizedCoordinateContext( + referenceFrame: CGRect(x: 0, y: 0, width: 390, height: 844), + resolvedWindow: app.windows.firstMatch, + keyboardPolicy: .never, + accessibilityHealth: accessibilityHealth + ) + } + + private func sequenceStep(_ kind: String, synthesized: Bool?) -> SequenceStep { + SequenceStep(kind: kind, x: 10, y: 20, durationMs: nil, pauseMs: nil, synthesized: synthesized) + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift new file mode 100644 index 0000000000..9937ba8362 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift @@ -0,0 +1,116 @@ +import XCTest +import AgentDeviceSnapshotPresentation + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +extension RunnerTests { + func testSynthesizedDispatchInvertsCaptureOnEveryInterfaceOrientation() { + let frames = [ + CGRect(x: 0, y: 0, width: 430, height: 932), + CGRect(x: 0, y: 0, width: 951, height: 669), + ] + for frame in frames { + for orientation in [ + RunnerInterfaceOrientation.portrait, + RunnerInterfaceOrientation.portraitUpsideDown, + RunnerInterfaceOrientation.landscapeLeft, + RunnerInterfaceOrientation.landscapeRight, + ] { + let corners = [ + CGPoint(x: frame.minX + 1, y: frame.minY + 1), + CGPoint(x: frame.midX, y: frame.midY), + CGPoint(x: frame.maxX - 2, y: frame.maxY - 3), + ] + for displayed in corners { + let captured = CoordinateSpaceRotation.oriented( + point: displayed, + in: frame, + interfaceOrientation: orientation + ) + let dispatched = CoordinateSpaceRotation.native( + point: captured, + in: frame, + interfaceOrientation: orientation + ) + XCTAssertEqual(dispatched.x, displayed.x, accuracy: 0.001, "orientation \(orientation)") + XCTAssertEqual(dispatched.y, displayed.y, accuracy: 0.001, "orientation \(orientation)") + } + } + } + } + + func testPlannedMultiTouchGestureAcceptsMatchingInBoundsTrajectories() throws { + let plan = try JSONDecoder().decode( + RunnerGesturePlan.self, + from: Data( + #"{"topology":"two","intent":"pan","durationMs":32,"viewport":{"x":0,"y":0,"width":200,"height":300},"pointers":[{"pointerId":0,"samples":[{"offsetMs":0,"point":{"x":80,"y":80}},{"offsetMs":16,"point":{"x":90,"y":85}},{"offsetMs":32,"point":{"x":100,"y":90}}]},{"pointerId":1,"samples":[{"offsetMs":0,"point":{"x":80,"y":120}},{"offsetMs":16,"point":{"x":90,"y":125}},{"offsetMs":32,"point":{"x":100,"y":130}}]}]}"#.utf8 + ) + ) + + XCTAssertNil(plannedGestureValidationError(plan)) + XCTAssertEqual(plannedGestureExecution(for: plan), .sampled) + } + + func testPlannedMultiTouchGestureRejectsMismatchedOffsets() throws { + let plan = try JSONDecoder().decode( + RunnerGesturePlan.self, + from: Data( + #"{"topology":"two","intent":"transform","durationMs":32,"viewport":{"x":0,"y":0,"width":200,"height":300},"pointers":[{"pointerId":0,"samples":[{"offsetMs":0,"point":{"x":80,"y":80}},{"offsetMs":32,"point":{"x":100,"y":90}}]},{"pointerId":1,"samples":[{"offsetMs":0,"point":{"x":80,"y":120}},{"offsetMs":31,"point":{"x":100,"y":130}}]}]}"#.utf8 + ) + ) + + XCTAssertEqual( + plannedGestureValidationError(plan), + "planned pointer sample offsets must match and strictly increase" + ) + } + + func testSinglePointerFlingUsesFastSwipeExecution() throws { + let plan = try JSONDecoder().decode( + RunnerGesturePlan.self, + from: Data( + #"{"topology":"single","intent":"fling","executionProfile":"endpoint-hold","durationMs":100,"viewport":{"x":0,"y":0,"width":200,"height":300},"pointers":[{"pointerId":0,"samples":[{"offsetMs":0,"point":{"x":160,"y":150}},{"offsetMs":100,"point":{"x":40,"y":150}}]}]}"#.utf8 + ) + ) + + XCTAssertEqual(plannedGestureExecution(for: plan), .fastSwipe) + } + + func testSinglePointerTimedPanUsesSampledExecution() throws { + let plan = try JSONDecoder().decode( + RunnerGesturePlan.self, + from: Data( + #"{"topology":"single","intent":"pan","executionProfile":"timed-pan","durationMs":500,"viewport":{"x":0,"y":0,"width":200,"height":300},"pointers":[{"pointerId":0,"samples":[{"offsetMs":0,"point":{"x":160,"y":150}},{"offsetMs":250,"point":{"x":100,"y":150}},{"offsetMs":500,"point":{"x":40,"y":150}}]}]}"#.utf8 + ) + ) + + XCTAssertEqual(plannedGestureExecution(for: plan), .sampled) + } + + func testSinglePointerEndpointHoldUsesFastSwipeExecution() throws { + let plan = try JSONDecoder().decode( + RunnerGesturePlan.self, + from: Data( + #"{"topology":"single","intent":"pan","executionProfile":"endpoint-hold","durationMs":500,"viewport":{"x":0,"y":0,"width":200,"height":300},"pointers":[{"pointerId":0,"samples":[{"offsetMs":0,"point":{"x":160,"y":150}},{"offsetMs":500,"point":{"x":40,"y":150}}]}]}"#.utf8 + ) + ) + + XCTAssertNil(plannedGestureValidationError(plan)) + XCTAssertEqual(plannedGestureExecution(for: plan), .fastSwipe) + } + + func testSinglePointerGestureRejectsMissingExecutionProfile() throws { + let plan = try JSONDecoder().decode( + RunnerGesturePlan.self, + from: Data( + #"{"topology":"single","intent":"pan","durationMs":500,"viewport":{"x":0,"y":0,"width":200,"height":300},"pointers":[{"pointerId":0,"samples":[{"offsetMs":0,"point":{"x":160,"y":150}},{"offsetMs":500,"point":{"x":40,"y":150}}]}]}"#.utf8 + ) + ) + + XCTAssertEqual( + plannedGestureValidationError(plan), + "single-pointer gesture requires a supported execution profile" + ) + } + +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SystemSurfaceHostPolicyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SystemSurfaceHostPolicyTests.swift new file mode 100644 index 0000000000..e16aac6e3b --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SystemSurfaceHostPolicyTests.swift @@ -0,0 +1,50 @@ +import Foundation + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +import XCTest + +private struct SystemSurfaceHostFixture: Decodable { + struct Host: Decodable { + let bundleId: String + let kind: String + } + let hosts: [Host] +} + +extension RunnerTests { + func testSystemSurfaceHostRegistryMirrorsGoldenFixture() throws { + let fixtureURL = URL(fileURLWithPath: #filePath) + .deletingLastPathComponent() // UnitTests + .deletingLastPathComponent() // AgentDeviceRunnerUITests + .deletingLastPathComponent() // AgentDeviceRunner + .deletingLastPathComponent() // runner + .deletingLastPathComponent() // apple + .deletingLastPathComponent() // repo root + .appendingPathComponent("contracts") + .appendingPathComponent("fixtures") + .appendingPathComponent("ios-system-surface-hosts.json") + let fixture = try JSONDecoder().decode( + SystemSurfaceHostFixture.self, + from: Data(contentsOf: fixtureURL) + ) + let registry = SystemSurfaceHostRegistry.hosts.map { [$0.bundleId, $0.kind.rawValue] } + let golden = fixture.hosts.map { [$0.bundleId, $0.kind] } + XCTAssertEqual(registry, golden, "SystemSurfaceHostRegistry drifted from the golden fixture") + } + + func testSystemSurfaceHostRegistryRecognizesRegisteredHosts() { + XCTAssertTrue(SystemSurfaceHostRegistry.isSystemSurfaceHost("com.apple.SafariViewService")) + XCTAssertTrue(SystemSurfaceHostRegistry.isSystemSurfaceHost("com.apple.PassbookUIService")) + XCTAssertFalse(SystemSurfaceHostRegistry.isSystemSurfaceHost("com.example.app")) + XCTAssertFalse(SystemSurfaceHostRegistry.isSystemSurfaceHost(nil)) + XCTAssertEqual( + SystemSurfaceHostRegistry.host(forBundleId: "com.apple.SafariViewService")?.kind, + .webAuth + ) + XCTAssertEqual( + SystemSurfaceHostRegistry.host(forBundleId: "com.apple.PassbookUIService")?.kind, + .payment + ) + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TapPointPolicyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TapPointPolicyTests.swift new file mode 100644 index 0000000000..791b1b2530 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TapPointPolicyTests.swift @@ -0,0 +1,52 @@ +import XCTest + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +private struct TapPointPolicyFixture: Decodable { + struct Frame: Decodable { + let x: Double + let y: Double + let width: Double + let height: Double + + var cgRect: CGRect { + CGRect(x: x, y: y, width: width, height: height) + } + } + + let name: String + let elementFrame: Frame + let windowFrame: Frame + let allowed: Bool +} + +extension RunnerTests { + // Golden parity table (ADR 0011 Layer 2): every case in + // contracts/fixtures/tap-point-policy.json must agree with the vitest twin + // (tap-point-policy-parity.test.ts). Add cases there, never fork the rule. + func testTapPointPolicyMatchesGoldenParityTable() throws { + let fixtureURL = URL(fileURLWithPath: #filePath) + .deletingLastPathComponent() // UnitTests + .deletingLastPathComponent() // AgentDeviceRunnerUITests + .deletingLastPathComponent() // AgentDeviceRunner + .deletingLastPathComponent() // runner + .deletingLastPathComponent() // apple + .deletingLastPathComponent() // repo root + .appendingPathComponent("contracts") + .appendingPathComponent("fixtures") + .appendingPathComponent("tap-point-policy.json") + let data = try Data(contentsOf: fixtureURL) + let cases = try JSONDecoder().decode([TapPointPolicyFixture].self, from: data) + XCTAssertFalse(cases.isEmpty, "parity table must not be empty") + for fixture in cases { + XCTAssertEqual( + TapPointPolicy.isAllowed( + elementFrame: fixture.elementFrame.cgRect, + windowFrame: fixture.windowFrame.cgRect + ), + fixture.allowed, + fixture.name + ) + } + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift index beff83964c..f0563c6476 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift @@ -41,155 +41,34 @@ extension RunnerTests { } } - func testSynthesizedFirstResponderTypeRequiresHiddenKeyboardTapWitness() { - let cases: [(TextTypingRepairMode, Bool, Bool, Bool)] = [ - (.none, true, false, true), - (.none, true, true, false), - (.none, false, false, false), - (.append, true, false, false), - (.replacement, true, false, false), + func testSynthesizedFirstResponderTypeAdmitsOnlyTheBareSubmitKeyAfterAHiddenKeyboardTap() { + let cases: [(TextTypingRepairMode, String, Bool, Bool, Bool)] = [ + (.none, "\n", true, false, true), + (.none, "\n", true, true, false), + (.none, "\n", false, false, false), + (.none, "hardware-keyboard", true, false, false), + (.none, "\r", true, false, false), + (.none, "search\n", true, false, false), + (.append, "\n", true, false, false), + (.append, "hardware-keyboard", true, false, false), + (.replacement, "\n", true, false, false), ] - for (mode, fromTapWitness, softwareKeyboardVisible, expected) in cases { + for (mode, text, fromTapWitness, softwareKeyboardVisible, expected) in cases { XCTAssertEqual( Self.shouldUseSynthesizedFirstResponderType( repairMode: mode, + text: text, fromTapWitness: fromTapWitness, softwareKeyboardVisible: softwareKeyboardVisible ), - expected + expected, + "mode: \(mode), text: \(text.debugDescription)" ) } } - func testSynthesizedTextCommitProgressWalksExpectedPrefixOnly() { - let expected = "hardware-keyboard" - XCTAssertEqual( - Self.synthesizedTextCommitProgress(observedText: "hardware-keyboard", expectedText: expected), - .committed - ) - XCTAssertEqual( - Self.synthesizedTextCommitProgress(observedText: "", expectedText: expected), - .pending - ) - XCTAssertEqual( - Self.synthesizedTextCommitProgress(observedText: "hardware-keyboa", expectedText: expected), - .pending - ) - // Transformed input (formatter, mid-text caret, autocomplete) must stop the wait. - XCTAssertEqual( - Self.synthesizedTextCommitProgress(observedText: "hardwarX", expectedText: expected), - .diverged - ) - XCTAssertEqual( - Self.synthesizedTextCommitProgress(observedText: "hardware-keyboards", expectedText: expected), - .diverged - ) - XCTAssertEqual( - Self.synthesizedTextCommitProgress(observedText: nil, expectedText: expected), - .diverged - ) - } - - // The regression behind #1874/#1844: the wait used to return Void, so an expired deadline was - // indistinguishable from a commit and `type` reported ok over a partially committed field. The - // CI signature was a field holding "h" out of "hardware-keyboard" with the command successful. - func testSynthesizedCommitDeadlineIsNotReportedAsACommit() { - let clock = CommitWaitClock() - var observations = 0 - let outcome = Self.awaitSynthesizedCommitOutcome( - expectedText: "hardware-keyboard", - placeholder: nil, - stallBudget: 3, - ceiling: 10, - now: clock.read, - observe: { "h" }, - waitForNextObservation: { - observations += 1 - clock.advance(1) - } - ) - XCTAssertEqual(outcome, .notObserved) - XCTAssertEqual(observations, 3, "a pending prefix must keep polling until the deadline") - } - - func testSynthesizedCommitStopsAtTheFirstSettledObservation() { - for observed in ["hardware-keyboard", "hardwarX", nil] { - var polls = 0 - let outcome = Self.awaitSynthesizedCommitOutcome( - expectedText: "hardware-keyboard", - placeholder: nil, - observe: { observed }, - waitForNextObservation: { polls += 1 } - ) - // `.diverged` settles the wait too: the app transformed the input and the runner must not - // second-guess it. Only an outstanding strict prefix keeps waiting. - XCTAssertEqual(outcome, .settled, "observed: \(observed ?? "nil")") - XCTAssertEqual(polls, 0, "observed: \(observed ?? "nil")") - } - } - - func testSynthesizedCommitWalksAPrefixToCompletion() { - let steps = ["", "hardware-", "hardware-keyboard"] - var index = 0 - let outcome = Self.awaitSynthesizedCommitOutcome( - expectedText: "hardware-keyboard", - placeholder: nil, - observe: { steps[min(index, steps.count - 1)] }, - waitForNextObservation: { index += 1 } - ) - XCTAssertEqual(outcome, .settled) - XCTAssertEqual(index, 2) - } - - // Adversarial-review finding: the deadline used to be checked BEFORE observing, so a commit - // landing during the final poll sleep was condemned as never observed — a false failure under - // exactly the loaded-host timing this wait exists for. Red against that ordering. - func testCommitLandingDuringTheFinalSleepIsStillObserved() { - let clock = CommitWaitClock() - var polls = 0 - let outcome = Self.awaitSynthesizedCommitOutcome( - expectedText: "hardware-keyboard", - placeholder: nil, - stallBudget: 3, - ceiling: 10, - now: clock.read, - // The value lands during the sleep that takes the clock past the stall budget: the read - // happens first, so it is still observed. - observe: { polls == 0 ? "hardware-" : "hardware-keyboard" }, - waitForNextObservation: { - polls += 1 - clock.advance(9) - } - ) - XCTAssertEqual(outcome, .settled) - } - - // A pre-dispatch value cannot identify what a later placeholder-equal AX value represents. - // Here the field starts at "0", but its input handler clears it after `type ".00"`; the empty - // field then renders its "0.00" placeholder. Reporting success would describe an empty field as - // committed text. - func testClearAfterDispatchCannotTurnThePlaceholderIntoCommitEvidence() { - let textBeforeDispatch = "0" - let expectedText = textBeforeDispatch + ".00" - var observations = 0 - let outcome = Self.awaitSynthesizedCommitOutcome( - expectedText: expectedText, - placeholder: "0.00", - observe: { - observations += 1 - return "0.00" - }, - waitForNextObservation: {} - ) - XCTAssertEqual( - Self.textEntryFailure(forCommitOutcome: outcome)?.rawValue, - "TEXT_INPUT_COMMIT_NOT_OBSERVED" - ) - XCTAssertEqual(observations, 0, "no post-dispatch read can resolve this collision") - } - // The guard must stay narrow: it fires only when the WHOLE expected value is the placeholder. - // Widening it would refuse ordinary typing into any placeheld field, which is most of them. + // Widening it would refuse ordinary entry into any placeheld field, which is most of them. func testPlaceholderGuardDoesNotFireOnOrdinaryTyping() { let cases: [(placeholder: String?, expectedText: String)] = [ ("0.00", "0.005"), @@ -199,7 +78,7 @@ extension RunnerTests { (" ", ""), ] for testCase in cases { - let outcome = Self.awaitSynthesizedCommitOutcome( + let outcome = Self.awaitSynthesizedReplacementCommitOutcome( expectedText: testCase.expectedText, placeholder: testCase.placeholder, observe: { testCase.expectedText }, @@ -209,32 +88,16 @@ extension RunnerTests { } } - // The bug this whole route exists to fix: `awaitSynthesizedCommitOutcome` (append/`type`) treats - // any non-prefix value as `.diverged` -> `.settled`, i.e. "trust the app, don't second-guess it." - // That rule is correct for `type` (an autocomplete/formatter can legitimately transform bare - // input) but silently swallows a dropped-character corruption in `.replacement` mode, because a - // value with a hole in the middle is neither a matching prefix nor an exact match — it still hits - // `.diverged`. These are the two corruption strings actually observed in CI on `fill` + // A value with a hole in the middle is neither a matching prefix nor an exact match, and must + // never settle. These are the two corruption strings actually observed in CI on `fill` // (id="field-name" "Ada Lovelace" -> "Avelace", id="field-email" "ada@example" -> "aexample"; - // first character and tail survive, a middle run is missing). Confirms - // `awaitSynthesizedReplacementCommitOutcome` reports `.notObserved` for both, where - // `awaitSynthesizedCommitOutcome` (proven by the assertion inside the loop) reports `.settled`. + // first character and tail survive, a middle run is missing). func testSynthesizedReplacementCommitCatchesDroppedMiddleCharacters() { let corruptions: [(expected: String, observedAfterDrop: String)] = [ (expected: "Ada Lovelace", observedAfterDrop: "Avelace"), (expected: "ada@example", observedAfterDrop: "aexample"), ] for corruption in corruptions { - XCTAssertEqual( - Self.awaitSynthesizedCommitOutcome( - expectedText: corruption.expected, - placeholder: nil, - observe: { corruption.observedAfterDrop }, - waitForNextObservation: {} - ), - .settled, - "append-mode's diverge-trusting outcome must stay unchanged by this fix" - ) let clock = CommitWaitClock() var polls = 0 let outcome = Self.awaitSynthesizedReplacementCommitOutcome( @@ -255,10 +118,7 @@ extension RunnerTests { } // The non-failure counterpart: replacement mode must still tolerate real commit lag (the value - // converges to an exact match over a few polls), not just instant matches. Mirrors - // `testSynthesizedCommitWalksAPrefixToCompletion`, but replacement mode has no "prefix" concept — - // every intermediate read here is deliberately NOT a prefix of the final value, to prove the wait - // does not depend on prefix-walking to keep polling. + // converges to an exact match over a few polls), not just instant matches. func testSynthesizedReplacementCommitToleratesLagUntilExactMatch() { let steps = ["", "ad", "ada@example"] var index = 0 @@ -272,8 +132,8 @@ extension RunnerTests { XCTAssertEqual(index, 2) } - // Same ordering guarantee as `testCommitLandingDuringTheFinalSleepIsStillObserved`: the deadline - // is checked AFTER an observation, so a match landing during the final poll sleep is still caught. + // The deadline is checked AFTER an observation, so a match landing during the final poll sleep + // is still caught. func testSynthesizedReplacementCommitLandingDuringTheFinalSleepIsStillObserved() { let clock = CommitWaitClock() var polls = 0 @@ -292,8 +152,9 @@ extension RunnerTests { XCTAssertEqual(outcome, .settled) } - // Same placeholder-collision guard as append mode, and for the same reason: a pre-dispatch value - // cannot identify what a later placeholder-equal AX value represents, so refuse before polling. + // A pre-dispatch value cannot identify what a later placeholder-equal AX value represents: an + // input handler may clear the field after dispatch and the empty field then renders the + // placeholder. Reporting success would describe an empty field as committed text. func testSynthesizedReplacementCommitPlaceholderGuardRefusesWithoutPolling() { var observations = 0 let outcome = Self.awaitSynthesizedReplacementCommitOutcome( @@ -310,8 +171,7 @@ extension RunnerTests { } // The mapping the command actually refuses on. `.unobservable` must stay a success: it is the - // pre-existing contract for submit-key text and unreadable fields, so inverting it would fail - // every `type "...\n"`. + // contract for submit-key text, so inverting it would fail every `fill` ending in a submit key. func testOnlyAnUnobservedCommitBecomesACommandFailure() { XCTAssertNil(Self.textEntryFailure(forCommitOutcome: .settled)) XCTAssertNil(Self.textEntryFailure(forCommitOutcome: .unobservable)) @@ -344,15 +204,6 @@ extension RunnerTests { } #endif - func testResolvedCoordinateTextEntryFallsBackWhenSynthesizedFocusIsUnavailable() { - XCTAssertFalse(Self.shouldFallbackFromSynthesizedTextEntryFocus(.performed)) - XCTAssertTrue( - Self.shouldFallbackFromSynthesizedTextEntryFocus( - .unsupported(message: "private synthesis unavailable", hint: "use XCTest") - ) - ) - } - func testResolvedCoordinateTextEntryRouteRequiresReplacementCoordinatesAndPenalizedXCTest() { let cases: [(TextTypingRepairMode, Bool, Bool, Bool, Bool)] = [ (.replacement, true, true, false, false), @@ -465,9 +316,8 @@ extension RunnerTests { XCTAssertNil(result.observedText) } - // Companion to the above: text carrying a submit key must skip the wait entirely, same as the - // append route (`awaitSynthesizedFirstResponderCommit`) — the app may clear or rewrite the field - // on submit, so there is nothing meaningful to poll toward. + // Companion to the above: text carrying a submit key must skip the wait entirely — the app may + // clear or rewrite the field on submit, so there is nothing meaningful to poll toward. func testSynthesizedReplacementCommitSkipsSubmitKeyText() { for expectedText in ["ada@example.test\n", "ada@example.test\r"] { XCTAssertEqual( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextInputCandidatePolicyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextInputCandidatePolicyTests.swift new file mode 100644 index 0000000000..ec3201275b --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextInputCandidatePolicyTests.swift @@ -0,0 +1,33 @@ +import CoreGraphics +import XCTest + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +extension RunnerTests { + func testCoordinateTextInputCandidateMustBeEnabledAndContainTheTouchPoint() { + let frame = CGRect(x: 10, y: 20, width: 100, height: 40) + let point = CGPoint(x: 50, y: 40) + + XCTAssertTrue( + isCoordinateTextInputCandidate( + enabled: true, + frame: frame, + point: point + ) + ) + XCTAssertFalse( + isCoordinateTextInputCandidate( + enabled: false, + frame: frame, + point: point + ) + ) + XCTAssertFalse( + isCoordinateTextInputCandidate( + enabled: true, + frame: frame, + point: CGPoint(x: 200, y: 200) + ) + ) + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextInputProbeTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextInputProbeTests.swift index fd1a0d51c9..28686cec65 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextInputProbeTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextInputProbeTests.swift @@ -75,10 +75,11 @@ extension RunnerTests { XCTAssertNotNil(textEntryTapWitness) XCTAssertFalse(isSnapshotXCTestChannelPenalized(bundleId: currentBundleId)) try XCTSkipIf(isKeyboardVisible(app: app), "software keyboard is up; hidden-keyboard witness cannot be exercised") - let type = try runnerCommandFixture(#"{"appBundleId":"com.callstack.agentdevice.runner","command":"type","commandId":"type-healthy-probe","text":"probe-witness"}"#) + let type = try runnerCommandFixture(#"{"appBundleId":"com.callstack.agentdevice.runner","command":"type","commandId":"type-healthy-probe","text":"probe-witness","textEntryMode":"append"}"#) let typed = try execute(command: type) XCTAssertTrue(typed.ok, String(describing: typed.error)) - XCTAssertEqual(typed.data?.textEntryRoute, "synthesized-first-responder") + XCTAssertEqual(typed.data?.textEntryRoute, "xctest-element") + XCTAssertNil(textEntryTapWitness, "the type must consume the tap witness it was addressed by") XCTAssertEqual(field.value as? String, "probe-witness") XCTAssertFalse(didRecordXCTestFailure(since: failures)) } @@ -153,7 +154,7 @@ extension RunnerTests { XCTAssertFalse(didRecordXCTestFailure(since: failures)) XCTAssertFalse(isSnapshotXCTestChannelPenalized(bundleId: currentBundleId)) XCTAssertNil(textEntryTapWitness) - let type = try runnerCommandFixture(#"{"appBundleId":"com.callstack.agentdevice.runner","command":"type","commandId":"type-after-unavailable-probe","text":"must-not-type"}"#) + let type = try runnerCommandFixture(#"{"appBundleId":"com.callstack.agentdevice.runner","command":"type","commandId":"type-after-unavailable-probe","text":"must-not-type","textEntryMode":"append"}"#) let typed = try execute(command: type) XCTAssertFalse(typed.ok) XCTAssertEqual(typed.error?.code, "TEXT_INPUT_NOT_FOCUSED") diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift new file mode 100644 index 0000000000..214f599bd6 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift @@ -0,0 +1,193 @@ +import XCTest + +// Command-level `type` coverage in the request shapes the daemon sends: ordinary text in +// `textEntryMode: "append"`, and the bare submit key with no mode. +extension RunnerTests { +#if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS) + func testTypeWithoutResolvedInputReturnsTypedFailureBeforeDispatchingText() throws { + let command = try runnerCommandFixture( + #"{"command":"type","commandId":"type-without-focus","text":"hello","textEntryMode":"append"}"# + ) + + let response = executeTypeCommand( + activeApp: XCUIApplication(bundleIdentifier: "com.example.agentdevice.missing-input"), + command: command + ) + + XCTAssertFalse(response.ok) + XCTAssertEqual(response.error?.code, "TEXT_INPUT_NOT_FOCUSED") + XCTAssertEqual( + response.error?.hint, + "Focus a visible text input, then retry type or fill. If the input is not exposed by accessibility, use a coordinate focus command before typing." + ) + } + + func testBareTypeUsesTappedInputWhenSoftwareKeyboardIsHidden() throws { + // The fixture uses a real text responder with an empty input view to model hardware-keyboard input. + let textField = try launchHardwareKeyboardFixture() + try tapHardwareKeyboardInput(commandId: "tap-hardware-keyboard-input") + try skipUnlessSoftwareKeyboardIsHidden() + + let failureCountBefore = currentXCTestFailureCount() + let typeResponse = executeTypeCommand( + activeApp: app, + command: try runnerCommandFixture( + #"{"command":"type","commandId":"type-hardware-keyboard","text":"hardware-keyboard","textEntryMode":"append"}"# + ) + ) + + XCTAssertTrue(typeResponse.ok, String(describing: typeResponse.error)) + XCTAssertFalse(didRecordXCTestFailure(since: failureCountBefore)) + XCTAssertEqual(typeResponse.data?.textEntryRoute, "xctest-element") + XCTAssertNil(textEntryTapWitness, "the type must consume the tap witness it was addressed by") + XCTAssertEqual(textField.value as? String, "hardware-keyboard") + + // The tap witness is one-shot: a second bare type without a new tap has no target. + let unfocusedFailureCountBefore = currentXCTestFailureCount() + let unfocusedResponse = executeTypeCommand( + activeApp: app, + command: try runnerCommandFixture( + #"{"command":"type","commandId":"type-hardware-keyboard-unfocused","text":"-again","textEntryMode":"append"}"# + ) + ) + + XCTAssertFalse(unfocusedResponse.ok) + XCTAssertEqual(unfocusedResponse.error?.code, "TEXT_INPUT_NOT_FOCUSED") + XCTAssertFalse(didRecordXCTestFailure(since: unfocusedFailureCountBefore)) + XCTAssertEqual(textField.value as? String, "hardware-keyboard") + + try tapHardwareKeyboardInput(commandId: "tap-hardware-keyboard-input-again") + // The first tap already proved this simulator keeps the keyboard down for the fixture, so a + // keyboard here is a product change, not an environment fact. + XCTAssertFalse(isKeyboardVisible(app: app)) + let appendFailureCountBefore = currentXCTestFailureCount() + let appendResponse = executeTypeCommand( + activeApp: app, + command: try runnerCommandFixture( + #"{"command":"type","commandId":"type-hardware-keyboard-again","text":"-again","textEntryMode":"append"}"# + ) + ) + + XCTAssertTrue(appendResponse.ok, String(describing: appendResponse.error)) + XCTAssertFalse(didRecordXCTestFailure(since: appendFailureCountBefore)) + XCTAssertEqual(appendResponse.data?.textEntryRoute, "xctest-element") + XCTAssertEqual(textField.value as? String, "hardware-keyboard-again") + } + + func testBareSubmitKeyUsesSynthesizedFirstResponderAfterHiddenKeyboardTap() throws { + _ = try launchHardwareKeyboardFixture() + try tapHardwareKeyboardInput(commandId: "tap-hardware-keyboard-submit") + try skipUnlessSoftwareKeyboardIsHidden() + + let failureCountBefore = currentXCTestFailureCount() + let submitResponse = executeTypeCommand( + activeApp: app, + command: try runnerCommandFixture( + #"{"command":"type","commandId":"type-hardware-keyboard-submit","text":"\n"}"# + ) + ) + + XCTAssertTrue(submitResponse.ok, String(describing: submitResponse.error)) + XCTAssertFalse(didRecordXCTestFailure(since: failureCountBefore)) + XCTAssertEqual(submitResponse.data?.textEntryRoute, "synthesized-first-responder") + XCTAssertNil(textEntryTapWitness, "the submit must consume the tap witness it was addressed by") + } + + func testBareSubmitKeyRefusesWhenPrivateSynthesisIsUnavailable() throws { + let textField = try launchHardwareKeyboardFixture() + try skipUnlessSoftwareKeyboardIsHidden() + + let failureCountBefore = currentXCTestFailureCount() + let result = typeTextReliably( + app: app, + target: TextEntryTarget( + element: textField, + refreshPoint: nil, + prefersFocusedElement: false, + fromTapWitness: true + ), + text: "\n", + delaySeconds: 0, + synthesizer: UnavailableTextEntrySynthesizer() + ) + + XCTAssertEqual(result.failure, .synthesisUnavailable) + XCTAssertEqual(result.textEntryRoute, "synthesized-first-responder") + XCTAssertFalse(didRecordXCTestFailure(since: failureCountBefore)) + } + + func testBareDelayedTypeFailsWhenTappedInputDisappearsMidCommand() throws { + app.launchArguments = [ + "--agent-device-text-entry-regression", + "--agent-device-text-entry-disappear-after-input", + ] + app.launch() + defer { + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } + XCTAssertTrue(app.waitForExistence(timeout: appExistenceTimeout)) + + let textField = app.textFields["agent-device-hardware-keyboard-input"] + XCTAssertTrue(textField.waitForExistence(timeout: appExistenceTimeout)) + let tapCommand = try runnerCommandFixture( + #"{"command":"tap","commandId":"tap-disappearing-input","selectorKey":"id","selectorValue":"agent-device-hardware-keyboard-input"}"# + ) + let tapResponse = try executeOnMainPrepared(command: tapCommand, activeApp: app) + XCTAssertTrue(tapResponse.ok, String(describing: tapResponse.error)) + + let failureCountBefore = currentXCTestFailureCount() + let typeCommand = try runnerCommandFixture( + #"{"command":"type","commandId":"type-disappearing-input","text":"ab","delayMs":50,"textEntryMode":"append"}"# + ) + let typeResponse = executeTypeCommand(activeApp: app, command: typeCommand) + + XCTAssertFalse(didRecordXCTestFailure(since: failureCountBefore)) + XCTAssertFalse(typeResponse.ok) + XCTAssertEqual(typeResponse.error?.code, "TEXT_INPUT_NOT_FOCUSED") + XCTAssertFalse(textField.exists) + } + + private struct UnavailableTextEntrySynthesizer: TextEntrySynthesizing { + func enterText( + app _: XCUIApplication, + text _: String, + replacingExistingText _: Bool + ) -> SynthesizedTextEntryAction { + .fallback + } + } + + private func launchHardwareKeyboardFixture() throws -> XCUIElement { + app.launchArguments = ["--agent-device-text-entry-regression"] + app.launch() + addTeardownBlock { [self] in + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } + XCTAssertTrue(app.waitForExistence(timeout: appExistenceTimeout)) + let textField = app.textFields["agent-device-hardware-keyboard-input"] + XCTAssertTrue(textField.waitForExistence(timeout: appExistenceTimeout)) + XCTAssertFalse(textField.frame.isEmpty) + return textField + } + + private func tapHardwareKeyboardInput(commandId: String) throws { + let tapCommand = try runnerCommandFixture( + #"{"command":"tap","commandId":"\#(commandId)","selectorKey":"id","selectorValue":"agent-device-hardware-keyboard-input"}"# + ) + let tapResponse = try executeOnMainPrepared(command: tapCommand, activeApp: app) + XCTAssertTrue(tapResponse.ok, String(describing: tapResponse.error)) + } + + // A precondition, not a product claim. The fixture's empty `inputView` is what keeps the + // keyboard down, but nothing in this bundle owns the simulator's own keyboard settings, so an + // ambient keyboard here is an environment fact rather than a product regression. + private func skipUnlessSoftwareKeyboardIsHidden() throws { + try XCTSkipIf( + isKeyboardVisible(app: app), + "software keyboard is up: this simulator cannot exercise the hidden-keyboard responder path" + ) + } +#endif +} diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TransportTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TransportTests.swift new file mode 100644 index 0000000000..ffa3d56d36 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TransportTests.swift @@ -0,0 +1,71 @@ +import XCTest +import Network + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +extension RunnerTests { + func testDuplicateCommandIdCoalescesOntoInFlightExecution() throws { + let command = try JSONDecoder().decode( + Command.self, + from: Data(#"{"command":"snapshot","commandId":"snapshot-coalesce"}"#.utf8) + ) + var primaryData: Data? + var waiterData: Data? + defer { + inFlightCommandIds.removeAll() + inFlightCommandWaiters.removeAll() + } + + XCTAssertFalse( + attachToInFlightCommandIfNeeded(command: command) { result in + primaryData = result.data + } + ) + XCTAssertTrue( + attachToInFlightCommandIfNeeded(command: command) { result in + waiterData = result.data + } + ) + + let delivered = Data("single-result".utf8) + deliverCommandResult( + command: command, + result: (delivered, false) + ) { result in + primaryData = result.data + } + + XCTAssertEqual(primaryData, delivered) + XCTAssertEqual(waiterData, delivered) + XCTAssertFalse(inFlightCommandIds.contains("snapshot-coalesce")) + XCTAssertNil(inFlightCommandWaiters["snapshot-coalesce"]) + } + + /// Routes `command` through the transport's inline and queued paths. The calling test's main + /// thread serves the command's main-thread work while it waits. + func execute(command: Command) throws -> Response { + dispatchPrecondition(condition: .onQueue(.main)) + if let response = inlineResponse(for: command) { + return response + } + final class ResultBox { + var result: Result? + } + let box = ResultBox() + let executed = XCTestExpectation(description: "\(command.command.rawValue) executed off main") + enqueueAccepted(command: command) { result in + box.result = result + executed.fulfill() + } + guard XCTWaiter.wait(for: [executed], timeout: mainThreadExecutionTimeout + 5) == .completed, + let result = box.result + else { + throw NSError( + domain: RunnerErrorDomain.general, + code: RunnerErrorCode.commandReturnedNoResponse, + userInfo: [NSLocalizedDescriptionKey: "command did not finish on the command queue"] + ) + } + return try result.get() + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TvRemoteTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TvRemoteTests.swift new file mode 100644 index 0000000000..3cdcc89651 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TvRemoteTests.swift @@ -0,0 +1,25 @@ +import XCTest + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +extension RunnerTests { + func testTvRemoteButtonMappingAcceptsSupportedNamesAndRejectsUnknown() { + let supported = [ + ("select", "select"), + ("SELECT", "select"), + ("menu", "menu"), + ("home", "home"), + ("up", "up"), + ("down", "down"), + ("left", "left"), + ("right", "right"), + ] + for (raw, expected) in supported { + XCTAssertEqual(tvRemoteButton(from: raw)?.rawValue, expected) + } + + for raw in [String?(nil), "", "volumeUp", "select "] { + XCTAssertNil(tvRemoteButton(from: raw)) + } + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift new file mode 100644 index 0000000000..22350597db --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift @@ -0,0 +1,215 @@ +import XCTest + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +private final class DisplayScreenFixture: NSObject { + @objc let displayID: UInt + init(_ displayID: UInt) { self.displayID = displayID } +} + +/// A window the runtime answers only once its query has resolved. `frame` is the read that +/// resolves it: while the query is unresolved `screen` names main, which is what a fresh runner +/// process that has not queried this app yet reports (#2728). +private final class DisplayWindowFixture: NSObject { + private let windowFrame: CGRect + private let resolvedDisplayID: UInt + private let present: Bool + private(set) var resolved = false + private lazy var windowScreen = DisplayScreenFixture(resolvedDisplayID) + init(frame: CGRect, displayID: UInt, present: Bool = true) { + self.windowFrame = frame + self.resolvedDisplayID = displayID + self.present = present + } + @objc var exists: Bool { present } + @objc var frame: CGRect { + resolved = true + return windowFrame + } + // Handed back as one object so a test can prove the resolver captures the screen the window + // reported rather than a fresh value for the same key. Unresolved, it names main. + @objc var screen: NSObject { resolved ? windowScreen : DisplayScreenFixture(1) } +} + +private final class DisplayWindowsFixture: NSObject { + @objc let firstMatch: NSObject + init(_ window: NSObject) { firstMatch = window } +} + +private final class DisplayApplicationFixture: NSObject { + @objc let windows: NSObject + convenience init(_ window: NSObject) { + self.init(windows: DisplayWindowsFixture(window)) + } + init(windows: NSObject) { self.windows = windows } +} + +/// A `windows` query the runtime refuses to answer. KVC raises on a key an object does not expose +/// rather than returning nil, which is exactly how a private XCTest read fails when it renames. +private final class UnanswerableWindowsFixture: NSObject {} + +/// A window whose frame resolves but whose screen key the runtime would not answer. +private final class ScreenlessWindowFixture: NSObject { + @objc let exists: Bool = true + @objc var frame: CGRect { CGRect(x: 0, y: 0, width: 951, height: 669) } +} + +/// A window whose frame key the runtime would not answer. The gesture path holds this object across +/// a command boundary, so a frame read that raises must fail closed inside the resolver. +private final class FramelessWindowFixture: NSObject { + @objc let exists: Bool = true +} + +extension RunnerTests { + func testSynthesizedDisplayRoutesByTheResolvedWindow() { + let window = DisplayWindowFixture(frame: CGRect(x: 0, y: 0, width: 951, height: 669), displayID: 3) + var displayID: UInt = 0 + XCTAssertNil(RunnerResolveWindowDisplayID(window, &displayID)) + XCTAssertEqual(displayID, 3) + XCTAssertTrue(window.resolved, "the window path must resolve the frame before reading its screen") + } + + func testSynthesizedDisplayPreservesPrimaryWindowIdentity() { + let window = DisplayWindowFixture(frame: CGRect(x: 0, y: 0, width: 466, height: 678), displayID: 1) + var displayID: UInt = 0 + XCTAssertNil(RunnerResolveWindowDisplayID(window, &displayID)) + XCTAssertEqual(displayID, 1) + } + + func testSynthesizedDisplayRefusesWindowWithoutQualifyingFrame() { + for frame in [CGRect.zero, .null, .infinite] { + let window = DisplayWindowFixture(frame: frame, displayID: 3) + var displayID: UInt = 99 + XCTAssertNotNil(RunnerResolveWindowDisplayID(window, &displayID)) + XCTAssertEqual(displayID, 99) + } + } + + func testSynthesizedDisplayRefusesWindowWithNoDisplayIDOrNoScreenAnswer() { + let noDisplayWindow = DisplayWindowFixture(frame: CGRect(x: 0, y: 0, width: 951, height: 669), displayID: 0) + let screenlessWindow = ScreenlessWindowFixture() + for window in [noDisplayWindow, screenlessWindow] as [NSObject] { + var displayID: UInt = 99 + let error = RunnerResolveWindowDisplayID(window, &displayID) + XCTAssertNotNil(error) + XCTAssertTrue( + error?.contains("no resolved window display ID") ?? false, + "expected the typed screen refusal, got \(error ?? "nil")" + ) + XCTAssertEqual(displayID, 99) + } + } + + func testSynthesizedDisplayRefusesAWindowWhoseFrameLookupRaises() { + var displayID: UInt = 99 + let reason = RunnerResolveWindowDisplayID(FramelessWindowFixture(), &displayID) + XCTAssertNotNil(reason, "a frame read that raises must be caught, not handed to the gesture path") + XCTAssertTrue( + reason?.contains("no resolved application window") ?? false, + "expected the typed window refusal, got \(reason ?? "nil")" + ) + XCTAssertEqual(displayID, 99) + } + + func testFirstUsableWindowSkipsAbsentAndEmptyWindows() { + let lit = CGRect(x: 0, y: 0, width: 402, height: 874) + XCTAssertEqual(RunnerTests.firstUsableWindowIndex(frames: [lit]), 0) + XCTAssertEqual(RunnerTests.firstUsableWindowIndex(frames: [nil, .zero, lit]), 2) + XCTAssertNil(RunnerTests.firstUsableWindowIndex(frames: [nil, .zero, nil])) + XCTAssertNil(RunnerTests.firstUsableWindowIndex(frames: [])) + } + + func testResolvedScreenHandsBackTheScreenTheResolvedWindowReported() { + let window = DisplayWindowFixture(frame: CGRect(x: 0, y: 0, width: 951, height: 669), displayID: 3) + var screen: AnyObject? + var displayID: UInt = 0 + var failure = RunnerApplicationScreenFailure.unresolvedScreen + XCTAssertTrue( + RunnerResolveApplicationScreen(DisplayApplicationFixture(window), &screen, &displayID, &failure) + ) + XCTAssertEqual(displayID, 3) + XCTAssertTrue(window.resolved, "the window's screen only answers once its query resolved") + XCTAssertTrue( + screen === window.screen, + "the capture must go to the screen object this window reported, not a re-queried one" + ) + } + + func testResolvedScreenRefusesAWindowLookupThatRaises() { + var screen: AnyObject? = NSObject() + var displayID: UInt = 99 + var failure = RunnerApplicationScreenFailure.none + XCTAssertFalse( + RunnerResolveApplicationScreen( + DisplayApplicationFixture(windows: UnanswerableWindowsFixture()), + &screen, + &displayID, + &failure + ) + ) + XCTAssertEqual(failure, .unresolvedWindow) + XCTAssertNil(screen) + XCTAssertEqual(displayID, 0) + } + + func testResolvedScreenRefusesAResolvedWindowWithNoAnswerableScreen() { + var screen: AnyObject? = NSObject() + var displayID: UInt = 99 + var failure = RunnerApplicationScreenFailure.none + XCTAssertFalse( + RunnerResolveApplicationScreen( + DisplayApplicationFixture(ScreenlessWindowFixture()), + &screen, + &displayID, + &failure + ) + ) + XCTAssertEqual(failure, .unresolvedScreen) + XCTAssertNil(screen) + XCTAssertEqual(displayID, 0) + } + + func testResolvedScreenRefusesAWindowReportingNoDisplayID() { + let window = DisplayWindowFixture(frame: CGRect(x: 0, y: 0, width: 466, height: 678), displayID: 0) + var screen: AnyObject? = NSObject() + var displayID: UInt = 99 + var failure = RunnerApplicationScreenFailure.none + XCTAssertFalse( + RunnerResolveApplicationScreen(DisplayApplicationFixture(window), &screen, &displayID, &failure) + ) + XCTAssertEqual(failure, .unresolvedScreen) + XCTAssertNil(screen) + XCTAssertEqual(displayID, 0) + } + + func testResolvedScreenRefusesAnEmptyWindowFrameWithoutNamingAnyDisplay() { + let window = DisplayWindowFixture(frame: .zero, displayID: 3) + var screen: AnyObject? = NSObject() + var displayID: UInt = 99 + var failure = RunnerApplicationScreenFailure.none + XCTAssertFalse( + RunnerResolveApplicationScreen(DisplayApplicationFixture(window), &screen, &displayID, &failure) + ) + XCTAssertEqual(failure, .unresolvedWindow) + XCTAssertNil(screen, "a dark panel must never become the capture target by default") + } + + /// A cold query answers `frame` with a usable rect on some runtimes while its `exists` says the + /// window is not there. Geometry alone must not be trusted into capturing a panel nobody resolved. + func testResolvedScreenRefusesAWindowTheQuerySaysIsNotThere() { + let window = DisplayWindowFixture( + frame: CGRect(x: 0, y: 0, width: 951, height: 669), + displayID: 3, + present: false + ) + var screen: AnyObject? = NSObject() + var displayID: UInt = 99 + var failure = RunnerApplicationScreenFailure.none + XCTAssertFalse( + RunnerResolveApplicationScreen(DisplayApplicationFixture(window), &screen, &displayID, &failure) + ) + XCTAssertEqual(failure, .unresolvedWindow) + XCTAssertNil(screen) + XCTAssertEqual(displayID, 0) + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/RecordingScripts/RecordingExportSupport.swift b/apple/runner/AgentDeviceRunner/RecordingScripts/RecordingExportSupport.swift index 5837d32ff4..8892f41584 100644 --- a/apple/runner/AgentDeviceRunner/RecordingScripts/RecordingExportSupport.swift +++ b/apple/runner/AgentDeviceRunner/RecordingScripts/RecordingExportSupport.swift @@ -9,11 +9,14 @@ enum RecordingScriptError: Error, CustomStringConvertible { case invalidArgs(String) case missingVideoTrack case exportFailed(String) + case frameWriteFailed(String) var description: String { switch self { case .invalidArgs(let message): return message + case .frameWriteFailed(let message): + return message case .missingVideoTrack: return "Input video does not contain a video track." case .exportFailed(let message): diff --git a/apple/runner/AgentDeviceRunner/RecordingScripts/recording-frames.swift b/apple/runner/AgentDeviceRunner/RecordingScripts/recording-frames.swift new file mode 100644 index 0000000000..2fc72fbcf2 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/RecordingScripts/recording-frames.swift @@ -0,0 +1,203 @@ +import AVFoundation +import Foundation +import ImageIO +import UniformTypeIdentifiers + +/// Extracts the frames at the requested presentation times from a recording and writes each one as +/// a PNG, then prints a JSON manifest naming the time each returned image actually carries. +/// +/// The caller owns the sampling grid: it passes an explicit, bounded list of times, so this script +/// never walks a whole clip. `--times` is the whole job description, which keeps the amount of +/// decoding, the bytes on disk, and the wall clock all bounded by the request rather than by the +/// recording's length. +/// +/// Reported times are the generator's *actual* return times, not the ones that were asked for. A +/// decoder answers a request with the frame it holds at that moment; the manifest says which frame +/// that was, so nothing downstream can print a requested time as though it had been decoded. + +struct FrameManifest: Encodable { + struct Frame: Encodable { + let index: Int + let requestedTimeMs: Double + let actualTimeMs: Double + let width: Int + let height: Int + let path: String + } + + struct Skipped: Encodable { + let index: Int + let requestedTimeMs: Double + let reason: String + } + + let inputPath: String + let durationMs: Double + let frames: [Frame] + let skipped: [Skipped] +} + +/// Entry point: `@main` because multi-file swiftc compilation reserves top-level statements for +/// `main.swift`, which cannot be shared per-script. +@main +enum RecordingFrames { + static func main() { + do { + try run() + } catch { + fputs("recording-frames: \(error)\n", stderr) + exit(1) + } + } +} + +func run() throws { + let arguments = Array(CommandLine.arguments.dropFirst()) + let inputURL = URL(fileURLWithPath: try requiredOption(arguments, "--input")) + let outputDirectoryURL = URL(fileURLWithPath: try requiredOption(arguments, "--output-dir"), isDirectory: true) + let requestedTimesMs = try parseTimes(try requiredOption(arguments, "--times")) + let maxWidth = try parseMaxWidth(optionValue(arguments, "--max-width")) + + guard !requestedTimesMs.isEmpty else { + throw RecordingScriptError.invalidArgs("--times must name at least one presentation time") + } + try FileManager.default.createDirectory(at: outputDirectoryURL, withIntermediateDirectories: true) + + let asset = AVURLAsset(url: inputURL) + _ = try sourceVideoTrack(of: asset) + let durationMs = max(0, asset.duration.seconds * 1000) + + let generator = AVAssetImageGenerator(asset: asset) + generator.appliesPreferredTrackTransform = true + generator.maximumSize = CGSize(width: maxWidth, height: maxWidth) + generator.requestedTimeToleranceBefore = .zero + generator.requestedTimeToleranceAfter = .zero + + var frames: [FrameManifest.Frame] = [] + var skipped: [FrameManifest.Skipped] = [] + + for (index, requestedTimeMs) in requestedTimesMs.enumerated() { + let requestedTime = CMTime(seconds: requestedTimeMs / 1000, preferredTimescale: 600) + var actualTime = CMTime.invalid + let cgImage: CGImage + do { + cgImage = try generator.copyCGImage(at: requestedTime, actualTime: &actualTime) + } catch { + skipped.append( + FrameManifest.Skipped( + index: index, + requestedTimeMs: requestedTimeMs, + reason: "\(error)" + ) + ) + continue + } + + // A frame the decoder cannot place on the timeline is not a frame at 0 ms. Writing it would + // label a mid-clip moment as the opening one, so the sample goes unanswered instead and the + // manifest says so. + guard CMTIME_IS_VALID(actualTime), actualTime.timescale != 0 else { + skipped.append( + FrameManifest.Skipped( + index: index, + requestedTimeMs: requestedTimeMs, + reason: "Decoder returned no presentation time for this frame" + ) + ) + continue + } + + let outputURL = outputDirectoryURL.appendingPathComponent(frameFileName(index: index)) + do { + try writePNG(cgImage: cgImage, to: outputURL) + } catch { + // A frame the decoder produced but storage refused is a failed run, not a moment the + // recording has nothing to show: reporting it as skipped would let a partial sheet claim + // to cover the clip. + try? FileManager.default.removeItem(at: outputURL) + throw RecordingScriptError.frameWriteFailed( + "Could not write frame \(index) at \(requestedTimeMs)ms to \(outputURL.path): \(error)" + ) + } + + frames.append( + FrameManifest.Frame( + index: index, + requestedTimeMs: requestedTimeMs, + actualTimeMs: timeIntervalMs(actualTime), + width: cgImage.width, + height: cgImage.height, + path: outputURL.path + ) + ) + } + + let manifest = FrameManifest( + inputPath: inputURL.path, + durationMs: durationMs, + frames: frames, + skipped: skipped + ) + let encoder = JSONEncoder() + encoder.outputFormatting = [.sortedKeys] + FileHandle.standardOutput.write(try encoder.encode(manifest)) +} + +func requiredOption(_ arguments: [String], _ flag: String) throws -> String { + guard let index = arguments.firstIndex(of: flag) else { + throw RecordingScriptError.invalidArgs("Missing \(flag)") + } + return try recordingOptionValue(arguments, index + 1, flag) +} + +func optionValue(_ arguments: [String], _ flag: String) -> String? { + guard let index = arguments.firstIndex(of: flag), index + 1 < arguments.count else { + return nil + } + return arguments[index + 1] +} + +func parseTimes(_ value: String) throws -> [Double] { + let parts = value.split(separator: ",", omittingEmptySubsequences: true) + guard !parts.isEmpty else { + throw RecordingScriptError.invalidArgs("--times must be a comma-separated list of milliseconds") + } + return try parts.map { part in + guard let parsed = Double(part), parsed.isFinite, parsed >= 0 else { + throw RecordingScriptError.invalidArgs("Invalid sample time: \(part)") + } + return parsed + } +} + +func parseMaxWidth(_ value: String?) throws -> Int { + guard let value else { return 360 } + guard let parsed = Int(value), parsed > 0 else { + throw RecordingScriptError.invalidArgs("Invalid --max-width: \(value)") + } + return parsed +} + +func timeIntervalMs(_ time: CMTime) -> Double { + guard CMTIME_IS_VALID(time), time.timescale != 0 else { return 0 } + return CMTimeGetSeconds(time) * 1000 +} + +func frameFileName(index: Int) -> String { + String(format: "frame-%04d.png", index) +} + +func writePNG(cgImage: CGImage, to url: URL) throws { + guard let destination = CGImageDestinationCreateWithURL( + url as CFURL, + UTType.png.identifier as CFString, + 1, + nil + ) else { + throw RecordingScriptError.exportFailed("Failed to open PNG destination for \(url.lastPathComponent)") + } + CGImageDestinationAddImage(destination, cgImage, nil) + guard CGImageDestinationFinalize(destination) else { + throw RecordingScriptError.exportFailed("Failed to encode PNG for \(url.lastPathComponent)") + } +} diff --git a/apple/runner/AgentDeviceRunner/RecordingScripts/recording-overlay.swift b/apple/runner/AgentDeviceRunner/RecordingScripts/recording-overlay.swift index 60f036921f..9c8077fb09 100644 --- a/apple/runner/AgentDeviceRunner/RecordingScripts/recording-overlay.swift +++ b/apple/runner/AgentDeviceRunner/RecordingScripts/recording-overlay.swift @@ -11,6 +11,13 @@ let minimumPinchVisibility: CFTimeInterval = 0.5 let swipeVisibilityTail: CFTimeInterval = 0.16 let trailOpacityKeyTimes: [NSNumber] = [0.0, 0.08, 0.62, 1.0] +// A compositor that cannot render frames still completes with an empty track, so a whole frame at +// or below this mean luma counts as black. +let compositedBlackFrameLuma: Double = 1.0 +// Above this mean luma the raw capture is taken to hold visible content worth preserving. Below it +// the recording itself is dark, so a dark composite is the content and not a compositor failure. +let sourceVisibleFrameLuma: Double = 2.0 + struct GestureEnvelope: Decodable { let events: [GestureEvent] } @@ -126,10 +133,10 @@ func run() throws { in: parentLayer ) - // Overlay burn-in forces a full re-encode; medium quality keeps simulator videos readable - // while avoiding very slow highest-quality exports. Pass --quality high to opt into - // the slower highest-quality export. - let presetName = exportPresetName(for: parsedArgs.exportQuality, compatibleWith: composition) + // Overlay burn-in forces a full re-encode. The export has to keep the captured track's own + // dimensions, so it uses the one preset that preserves arbitrary capture geometry; the hardware + // encoder makes the full-resolution re-encode cheap (measured ~1-2s for a 90s 1206x2622 clip). + let presetName = try exportPresetName(compatibleWith: composition) let exporter = try makeRecordingExporter( composition, presetName: presetName, @@ -141,17 +148,19 @@ func run() throws { timeoutMessage: "Touch overlay export timed out.", failureMessage: "Touch overlay export failed." ) + try verifyCompositedOverlay( + input: inputURL, + output: outputURL, + expectedRenderSize: renderSize + ) } func parseArguments( _ arguments: [String] -) throws -> (inputPath: String, outputPath: String, eventsPath: String, exportQuality: ExportQuality) { +) throws -> (inputPath: String, outputPath: String, eventsPath: String) { var inputPath: String? var outputPath: String? var eventsPath: String? - // Export quality defaults to medium so existing callers keep the fast, simulator-friendly - // export. Pass --quality high to opt into a slower highest-quality export. - var exportQuality: ExportQuality = .medium var index = 0 while index < arguments.count { @@ -168,11 +177,13 @@ func parseArguments( eventsPath = try recordingOptionValue(arguments, nextIndex, "--events") index += 2 case "--quality": + // Accepted for CLI parity with the other backends and still validated, but the composited + // export always preserves the captured geometry (see `exportPresetName`), so the tier never + // picks a resolution-capping preset and is not retained. let rawValue = try recordingOptionValue(arguments, nextIndex, "--quality") - guard let parsed = ExportQuality(rawValue: rawValue) else { + guard ExportQuality(rawValue: rawValue) != nil else { throw RecordingScriptError.invalidArgs("--quality must be one of: medium, high") } - exportQuality = parsed index += 2 default: throw RecordingScriptError.invalidArgs("Unknown argument: \(argument)") @@ -184,24 +195,22 @@ func parseArguments( "Usage: recording-overlay.swift --input