From 74b8ac68681c8660724c462f40b806d835924fea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 24 Sep 2026 13:47:55 +0200 Subject: [PATCH 1/6] perf(ios): reduce smoke critical path with affected XCTest selection --- .github/workflows/ios.yml | 56 ++++++-------- .github/workflows/macos.yml | 25 ++++++ scripts/ios-xctest-impact.ts | 76 +++++++++++++++++++ .../replays/ios/simulator/01-settings.ad | 3 - 4 files changed, 123 insertions(+), 37 deletions(-) create mode 100644 scripts/ios-xctest-impact.ts diff --git a/.github/workflows/ios.yml b/.github/workflows/ios.yml index 59c74bf5df..52e8ae072c 100644 --- a/.github/workflows/ios.yml +++ b/.github/workflows/ios.yml @@ -98,6 +98,16 @@ jobs: pnpm-lock.yaml examples/test-app/pnpm-lock.yaml + - name: Select iOS runner XCTests + id: xctest-impact + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + if [ "${{ github.event_name }}" = 'pull_request' ]; then + git fetch origin "$BASE_SHA" --depth=1 || echo 'Base fetch failed; XCTest selection will fail open.' + fi + node --experimental-strip-types scripts/ios-xctest-impact.ts + - name: Establish host focus canary id: host-focus-canary run: | @@ -126,30 +136,8 @@ jobs: runtime-version: ${{ env.IOS_RUNTIME_VERSION }} preferred-device-name: iPhone 17 Pro - - name: Verify clean-installed Simulator snapshot bridge preparation and the fold-helper -Werror gate - if: github.event_name == 'pull_request' - env: - BASE_SHA: ${{ github.event.pull_request.base.sha }} - run: | - git fetch origin "$BASE_SHA" --depth=1 - if git diff --quiet "$BASE_SHA"...HEAD -- \ - apple/snapshot-bridge \ - apple/fold-helper \ - packages/platform-apple/src/snapshot-source \ - packages/platform-apple/src/foldable \ - scripts/check-package.ts \ - scripts/size-report-install.mjs \ - scripts/size-report-package.mjs; then - echo "Snapshot bridge and fold-helper sources are unchanged; skipping preparation proof." - exit 0 - fi - pnpm build - pnpm exec vitest run \ - packages/platform-apple/src/snapshot-source/native-runtime.test.ts \ - packages/platform-apple/src/foldable/fold-helper-cache.test.ts - pnpm check:package -- --verify-snapshot-bridge-preparation - - name: Run targeted iOS runner XCTest regressions + if: steps.xctest-impact.outputs.run == 'true' run: | set -o pipefail XCTESTRUN_PATH="$(find "$AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH/Build/Products" -maxdepth 1 -name '*.xctestrun' -print -quit)" @@ -300,17 +288,6 @@ jobs: run: | echo "Fixture app source: ${{ steps.fixture-app.outputs.source }}" >> "$GITHUB_STEP_SUMMARY" - - name: Run fixture-backed iOS simulator E2E smoke - env: - AGENT_DEVICE_FIXTURE_APP_ID: ${{ steps.fixture-app.outputs.app-id }} - AGENT_DEVICE_FIXTURE_APP_PATH: ${{ steps.fixture-app.outputs.app-path }} - AGENT_DEVICE_IOS_E2E: '1' - AGENT_DEVICE_IOS_E2E_TIER: smoke - AGENT_DEVICE_IOS_UDID: ${{ steps.ios-simulator.outputs.simulator-udid }} - run: | - pnpm gate build - node --experimental-strip-types scripts/node-test-tmpdir.ts --test test/integration/smoke-ios-simulator-coverage.test.ts test/integration/smoke-ios-simulator.test.ts - # #1584: isolated, cheap automatic guard for the #1562 class of regression (iOS silently # ignoring a `gesture pan` duration). Split out of gesture-lab.ad, which stays full-tier # (dispatch-only via replays-manual.yml) because its multi-touch commands are a separate, @@ -325,6 +302,17 @@ jobs: node --experimental-strip-types src/bin.ts test examples/test-app/replays/gesture-pan-duration.ad --udid "${{ steps.ios-simulator.outputs.simulator-udid }}" --timeout 180000 --retries 2 --artifacts-dir test/artifacts/replays-ios-gesture-pan-duration --report-junit test/artifacts/replays-ios-gesture-pan-duration.junit.xml pnpm clean:daemon + - name: Run fixture-backed iOS simulator E2E smoke + env: + AGENT_DEVICE_FIXTURE_APP_ID: ${{ steps.fixture-app.outputs.app-id }} + AGENT_DEVICE_FIXTURE_APP_PATH: ${{ steps.fixture-app.outputs.app-path }} + AGENT_DEVICE_IOS_E2E: '1' + AGENT_DEVICE_IOS_E2E_TIER: smoke + AGENT_DEVICE_IOS_UDID: ${{ steps.ios-simulator.outputs.simulator-udid }} + run: | + pnpm gate build + node --experimental-strip-types scripts/node-test-tmpdir.ts --test test/integration/smoke-ios-simulator-coverage.test.ts test/integration/smoke-ios-simulator.test.ts + - name: Assert simulator automation preserved host focus if: ${{ always() }} run: | diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 32320e23d9..e096415f71 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -66,6 +66,31 @@ jobs: - name: Setup toolchain uses: ./.github/actions/setup-node-pnpm + - name: Verify clean-installed Simulator snapshot bridge preparation and the fold-helper -Werror gate + if: github.event_name == 'pull_request' + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + git fetch origin "$BASE_SHA" --depth=1 + if git diff --quiet "$BASE_SHA"...HEAD -- \ + apple/snapshot-bridge \ + apple/fold-helper \ + packages/platform-apple/src/snapshot-source \ + packages/platform-apple/src/foldable \ + scripts/check-package.ts \ + scripts/size-report-install.mjs \ + scripts/size-report-package.mjs \ + .github/workflows/ios.yml \ + .github/workflows/macos.yml; then + echo "Snapshot bridge and fold-helper sources are unchanged; skipping preparation proof." + exit 0 + fi + pnpm build + pnpm exec vitest run \ + packages/platform-apple/src/snapshot-source/native-runtime.test.ts \ + packages/platform-apple/src/foldable/fold-helper-cache.test.ts + pnpm check:package -- --verify-snapshot-bridge-preparation + - name: Run macOS command coverage contract uses: ./.github/actions/run-gate with: { gate: macos-coverage } diff --git a/scripts/ios-xctest-impact.ts b/scripts/ios-xctest-impact.ts new file mode 100644 index 0000000000..65d55ea3e6 --- /dev/null +++ b/scripts/ios-xctest-impact.ts @@ -0,0 +1,76 @@ +import fs from 'node:fs'; +import { spawnSync } from 'node:child_process'; + +const RUNNER_INPUTS = [ + 'apple/runner/', + 'apple/snapshot-presentation/', + 'contracts/fixtures/', + 'packages/platform-apple/src/runner/', + '.github/actions/setup-apple-runner-build/', + '.github/workflows/ios.yml', + 'package.json', + 'pnpm-lock.yaml', + 'scripts/build-xcuitest-apple.sh', + 'scripts/patch-xcuitest-runner-icon.ts', + 'scripts/swift-toolchain-tmpdir.ts', + 'scripts/write-xcuitest-cache-metadata.mjs', + 'scripts/check-xctest-selection.ts', + 'scripts/xctest-declarations.ts', + 'scripts/swift-conditional-compilation.ts', + 'scripts/ios-xctest-impact.ts', +] as const; + +export function affectsIosXctests(file: string): boolean { + return RUNNER_INPUTS.some((input) => + input.endsWith('/') ? file.startsWith(input) : file === input, + ); +} + +export function selectIosXctests( + eventName: string, + changedPaths: readonly string[] | null, +): { run: boolean; reason: string } { + if (eventName !== 'pull_request') + return { run: true, reason: 'main and manual runs exercise XCTest' }; + if (changedPaths === null || changedPaths.length === 0) { + return { run: true, reason: 'the PR change set could not be established' }; + } + const input = changedPaths.find(affectsIosXctests); + return input + ? { run: true, reason: `XCTest input changed: ${input}` } + : { run: false, reason: 'the PR changed no XCTest runner, test, or golden-table input' }; +} + +export function uncoveredRunnerCacheInputs(action: string): string[] { + const hashFiles = action.match(/hashFiles\(([^\n]+)\)/)?.[1]; + if (!hashFiles) return ['runner cache hashFiles declaration']; + return [...hashFiles.matchAll(/'([^']+)'/g)] + .map((match) => match[1] as string) + .filter((input) => !affectsIosXctests(input.replace(/\*\*?$/, 'probe.swift'))); +} + +function changedPathsFromGit(baseSha: string): string[] | null { + if (!/^[a-f0-9]{40}$/.test(baseSha)) return null; + const result = spawnSync('git', ['diff', '--name-only', '-z', baseSha, 'HEAD'], { + encoding: 'utf8', + }); + if (result.error || result.status !== 0 || result.stdout === null) return null; + return result.stdout.split('\0').filter(Boolean); +} + +if (process.argv[1]?.endsWith('/ios-xctest-impact.ts')) { + const eventName = process.env.GITHUB_EVENT_NAME ?? ''; + const changedPaths = + eventName === 'pull_request' ? changedPathsFromGit(process.env.BASE_SHA ?? '') : []; + const selection = selectIosXctests(eventName, changedPaths); + process.stdout.write(`iOS XCTest: ${selection.run ? 'run' : 'skip'}; ${selection.reason}\n`); + if (process.env.GITHUB_OUTPUT) { + fs.appendFileSync(process.env.GITHUB_OUTPUT, `run=${selection.run}\n`); + } + if (process.env.GITHUB_STEP_SUMMARY) { + fs.appendFileSync( + process.env.GITHUB_STEP_SUMMARY, + `### iOS XCTest selection\n\n${selection.run ? 'Run' : 'Skip'}: ${selection.reason}.\n`, + ); + } +} diff --git a/test/integration/replays/ios/simulator/01-settings.ad b/test/integration/replays/ios/simulator/01-settings.ad index 418401db19..226d8ddcb5 100644 --- a/test/integration/replays/ios/simulator/01-settings.ad +++ b/test/integration/replays/ios/simulator/01-settings.ad @@ -1,12 +1,9 @@ # Dogfood iOS Settings flow through the replay suite runner. context platform=ios target=mobile open com.apple.Preferences --relaunch -screenshot "./test/screenshots/replays/ios-settings.png" -snapshot -i appstate click "role=cell label=General || role=button label=General" wait "label=About || label=\"Software Update\" || text=\"Manage your overall setup and preferences\"" 5000 -snapshot is exists "label=About || label=\"Software Update\" || text=\"Manage your overall setup and preferences\"" find text "Software Update" exists back From 4883b43e639eb7a2f5ba394ee11150224b3fae6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 24 Sep 2026 13:48:07 +0200 Subject: [PATCH 2/6] chore(gates): enforce iOS XCTest impact selection coverage --- docs/agents/testing.md | 1 + scripts/__tests__/ios-xctest-impact.test.ts | 53 +++++++++++++++++++++ scripts/check-xctest-selection.ts | 7 +-- vitest.config.ts | 1 + 4 files changed, 59 insertions(+), 3 deletions(-) create mode 100644 scripts/__tests__/ios-xctest-impact.test.ts diff --git a/docs/agents/testing.md b/docs/agents/testing.md index b3b5318ccf..0ef018e0ba 100644 --- a/docs/agents/testing.md +++ b/docs/agents/testing.md @@ -45,6 +45,7 @@ provider, and coverage tests mock the typed HDC seam. Real validation is local h Apple runner changes run `pnpm check:xctest-selection` and build the affected target. The source `#if` guard is the XCTest lane classification — never maintain a second test-name list. Pure runner decisions use the macOS host lane; iOS/XCTest semantics need a simulator lane. +The iOS PR lane runs its XCTest list for native inputs or uncertain diffs; main and nightly remain broad. Local host-lane XCTest may need signing and automation permission: diff --git a/scripts/__tests__/ios-xctest-impact.test.ts b/scripts/__tests__/ios-xctest-impact.test.ts new file mode 100644 index 0000000000..571b5cc236 --- /dev/null +++ b/scripts/__tests__/ios-xctest-impact.test.ts @@ -0,0 +1,53 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { expect, test } from 'vitest'; +import { + affectsIosXctests, + selectIosXctests, + uncoveredRunnerCacheInputs, +} from '../ios-xctest-impact.ts'; + +test('every runner build-cache input triggers the PR XCTest lane', () => { + const action = fs.readFileSync( + path.resolve('.github/actions/setup-apple-runner-build/action.yml'), + 'utf8', + ); + expect(uncoveredRunnerCacheInputs(action)).toEqual([]); + expect( + uncoveredRunnerCacheInputs(action.replace('apple/runner/**', 'new-native-input/**')), + ).toEqual(['new-native-input/**']); +}); + +test('the PR workflow applies the impact decision to the XCTest step', () => { + const workflow = fs.readFileSync(path.resolve('.github/workflows/ios.yml'), 'utf8'); + expect(workflow).toContain('node --experimental-strip-types scripts/ios-xctest-impact.ts'); + expect(workflow).toMatch( + /- name: Run targeted iOS runner XCTest regressions\n\s+if: steps\.xctest-impact\.outputs\.run == 'true'/, + ); +}); + +test('native runner and golden-table changes run XCTest; TypeScript runtime changes use live E2E', () => { + for (const file of [ + 'apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift', + 'apple/snapshot-presentation/Sources/Presenter.swift', + 'contracts/fixtures/scroll-gesture.json', + '.github/workflows/ios.yml', + 'scripts/ios-xctest-impact.ts', + ]) { + expect(affectsIosXctests(file), file).toBe(true); + } + for (const file of [ + 'packages/platform-apple/src/snapshot-source/cache.ts', + 'apple/fold-helper/fold-helper.c', + 'test/integration/ios-simulator-e2e/live-runner.ts', + ]) { + expect(selectIosXctests('pull_request', [file]), file).toMatchObject({ run: false }); + } +}); + +test('pushes and uncertain diffs keep the full XCTest selection', () => { + expect(selectIosXctests('push', ['src/index.ts']).run).toBe(true); + expect(selectIosXctests('pull_request', null).run).toBe(true); + expect(selectIosXctests('pull_request', []).run).toBe(true); + expect(selectIosXctests('pull_request', ['src/index.ts', 'package.json']).run).toBe(true); +}); diff --git a/scripts/check-xctest-selection.ts b/scripts/check-xctest-selection.ts index 6b8c36bbd1..be6226ed9a 100644 --- a/scripts/check-xctest-selection.ts +++ b/scripts/check-xctest-selection.ts @@ -8,7 +8,8 @@ // - host macos.yml, macOS host, every PR: the whole bundle as compiled for macOS, minus // `-skip-testing:` — the pure runner-decision tests, whose guard is // `#if AGENT_DEVICE_RUNNER_UNIT_TESTS` alone. -// - pr ios.yml, iOS Simulator, every PR: the hand-written `-only-testing:` list. +// - pr ios.yml, iOS Simulator, when runner inputs change on a PR (and on every main +// push): the hand-written `-only-testing:` list. // - nightly xctest-nightly.yml, iOS Simulator, scheduled: the whole bundle as compiled for // iOS, minus `-skip-testing:` — includes the simulator-only tests, whose guard is // `… && os(iOS)` (they launch the host app, route through SpringBoard, or assert an @@ -51,7 +52,7 @@ const packageAppleRunnerScript = path.join(repoRoot, 'scripts/package-apple-runn /** The macOS host lane, which runs the whole macOS-compiled bundle on every PR. */ export const HOST_WORKFLOW_FILE = '.github/workflows/macos.yml'; -/** The PR lane, whose `-only-testing:` list decides what every pull request runs on the simulator. */ +/** The PR lane, whose `-only-testing:` list decides what selected pull requests run on the simulator. */ export const PR_WORKFLOW_FILE = '.github/workflows/ios.yml'; /** The nightly lane, whose `-skip-testing:` list decides what the full simulator suite leaves out. */ @@ -349,7 +350,7 @@ export function formatSummary(report: SelectionReport): string { return ( `xctest selection: ${declared} declared ${report.target} methods — host lane ` + `(${HOST_WORKFLOW_FILE}, macOS, every PR) reaches ${host}, PR list (${PR_WORKFLOW_FILE}, ` + - `iOS Simulator, every PR) selects ${pr}, nightly (${NIGHTLY_WORKFLOW_FILE}, iOS Simulator) ` + + `iOS Simulator, selected PRs and main) selects ${pr}, nightly (${NIGHTLY_WORKFLOW_FILE}, iOS Simulator) ` + `reaches ${nightly}; ${dark} reachable by no lane; ${ENTRY_POINT_METHOD} skipped everywhere.\n` ); } diff --git a/vitest.config.ts b/vitest.config.ts index 77b2793122..202217d18f 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -128,6 +128,7 @@ export default defineConfig({ // Parses ios.yml and the runner's Swift sources: no Xcode, no simulator, and // the check it guards is what keeps the PR lane's `-only-testing:` list honest. 'scripts/__tests__/xctest-selection.test.ts', + 'scripts/__tests__/ios-xctest-impact.test.ts', // The nightly XCTest lane's reporter/liveness check, which otherwise only ever // executes on a macOS runner at 04:30. 'scripts/__tests__/xctest-run-summary.test.ts', From c741db0ede3b583ee16d0dffc2a6c62a725b85d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 24 Sep 2026 14:38:51 +0200 Subject: [PATCH 3/6] fix(ci): run bridge proof after macOS live replay --- .github/workflows/macos.yml | 51 +++++++++++++++++++------------------ 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index e096415f71..75194466b4 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -66,31 +66,6 @@ jobs: - name: Setup toolchain uses: ./.github/actions/setup-node-pnpm - - name: Verify clean-installed Simulator snapshot bridge preparation and the fold-helper -Werror gate - if: github.event_name == 'pull_request' - env: - BASE_SHA: ${{ github.event.pull_request.base.sha }} - run: | - git fetch origin "$BASE_SHA" --depth=1 - if git diff --quiet "$BASE_SHA"...HEAD -- \ - apple/snapshot-bridge \ - apple/fold-helper \ - packages/platform-apple/src/snapshot-source \ - packages/platform-apple/src/foldable \ - scripts/check-package.ts \ - scripts/size-report-install.mjs \ - scripts/size-report-package.mjs \ - .github/workflows/ios.yml \ - .github/workflows/macos.yml; then - echo "Snapshot bridge and fold-helper sources are unchanged; skipping preparation proof." - exit 0 - fi - pnpm build - pnpm exec vitest run \ - packages/platform-apple/src/snapshot-source/native-runtime.test.ts \ - packages/platform-apple/src/foldable/fold-helper-cache.test.ts - pnpm check:package -- --verify-snapshot-bridge-preparation - - name: Run macOS command coverage contract uses: ./.github/actions/run-gate with: { gate: macos-coverage } @@ -177,6 +152,32 @@ jobs: --report-junit test/artifacts/replays-macos.junit.xml + # Clean-install verification may raise macOS local-network permission UI; keep it after replay. + - name: Verify clean-installed Simulator snapshot bridge preparation and the fold-helper -Werror gate + if: github.event_name == 'pull_request' + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + git fetch origin "$BASE_SHA" --depth=1 + if git diff --quiet "$BASE_SHA"...HEAD -- \ + apple/snapshot-bridge \ + apple/fold-helper \ + packages/platform-apple/src/snapshot-source \ + packages/platform-apple/src/foldable \ + scripts/check-package.ts \ + scripts/size-report-install.mjs \ + scripts/size-report-package.mjs \ + .github/workflows/ios.yml \ + .github/workflows/macos.yml; then + echo "Snapshot bridge and fold-helper sources are unchanged; skipping preparation proof." + exit 0 + fi + pnpm build + pnpm exec vitest run \ + packages/platform-apple/src/snapshot-source/native-runtime.test.ts \ + packages/platform-apple/src/foldable/fold-helper-cache.test.ts + pnpm check:package -- --verify-snapshot-bridge-preparation + - name: Upload macOS artifacts if: always() uses: ./.github/actions/upload-agent-device-artifacts From c8f141e612df28a5a21a5e9f231070f3409c600c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 24 Sep 2026 14:38:58 +0200 Subject: [PATCH 4/6] chore(gates): guard macOS replay before bridge proof --- scripts/__tests__/ios-xctest-impact.test.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/scripts/__tests__/ios-xctest-impact.test.ts b/scripts/__tests__/ios-xctest-impact.test.ts index 571b5cc236..673978a55a 100644 --- a/scripts/__tests__/ios-xctest-impact.test.ts +++ b/scripts/__tests__/ios-xctest-impact.test.ts @@ -26,6 +26,16 @@ test('the PR workflow applies the impact decision to the XCTest step', () => { ); }); +test('macOS clean-install proof follows live UI replay', () => { + const workflow = fs.readFileSync(path.resolve('.github/workflows/macos.yml'), 'utf8'); + const replay = workflow.indexOf('- name: Run macOS integration test'); + const proof = workflow.indexOf( + '- name: Verify clean-installed Simulator snapshot bridge preparation', + ); + expect(replay).toBeGreaterThan(-1); + expect(proof).toBeGreaterThan(replay); +}); + test('native runner and golden-table changes run XCTest; TypeScript runtime changes use live E2E', () => { for (const file of [ 'apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift', From a19fd9651c46b5cfe523f80fbfb546288fa2ec7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 24 Sep 2026 15:27:59 +0200 Subject: [PATCH 5/6] chore(gates): derive iOS XCTest selection from Swift guards and ownership --- .../setup-apple-runner-build/action.yml | 2 +- .github/workflows/ios.yml | 139 +++---------- .github/workflows/macos.yml | 23 +-- ...rTests+ApplicationStateRawValueTests.swift | 4 +- docs/agents/testing.md | 3 +- scripts/__tests__/apple-ci-impact.test.ts | 114 +++++++++++ scripts/__tests__/ios-xctest-impact.test.ts | 63 ------ scripts/__tests__/xctest-declarations.test.ts | 44 +++++ scripts/__tests__/xctest-selection.test.ts | 183 +++++++++++++----- scripts/apple-ci-impact.ts | 88 +++++++++ scripts/check-affected/model.ts | 14 +- scripts/check-xctest-selection.ts | 180 +++++++++++++++-- scripts/ios-xctest-impact.ts | 76 -------- scripts/xctest-declarations.ts | 108 +++++++++-- vitest.config.ts | 7 +- 15 files changed, 676 insertions(+), 372 deletions(-) create mode 100644 scripts/__tests__/apple-ci-impact.test.ts delete mode 100644 scripts/__tests__/ios-xctest-impact.test.ts create mode 100644 scripts/apple-ci-impact.ts delete mode 100644 scripts/ios-xctest-impact.ts diff --git a/.github/actions/setup-apple-runner-build/action.yml b/.github/actions/setup-apple-runner-build/action.yml index aad21ef883..c05ecd9326 100644 --- a/.github/actions/setup-apple-runner-build/action.yml +++ b/.github/actions/setup-apple-runner-build/action.yml @@ -44,7 +44,7 @@ runs: id: source-hash run: | set -euo pipefail - echo "value=${{ hashFiles('apple/runner/**', 'apple/snapshot-presentation/**', 'scripts/build-xcuitest-apple.sh', 'scripts/swift-toolchain-tmpdir.ts', 'scripts/patch-xcuitest-runner-icon.ts', 'scripts/write-xcuitest-cache-metadata.mjs', 'packages/platform-apple/src/runner/apple-runner-platform.ts', 'packages/platform-apple/src/runner/runner-cache-metadata.ts', 'packages/platform-apple/src/runner/runner-icon.ts', 'packages/platform-apple/src/runner/runner-xctestrun.ts', 'packages/platform-apple/src/runner/runner-xctestrun-products.ts', '.github/actions/setup-apple-runner-build/action.yml') }}" >> "$GITHUB_OUTPUT" + echo "value=${{ hashFiles('apple/runner/**', 'apple/snapshot-presentation/**', 'scripts/build-xcuitest-apple.sh', 'scripts/swift-toolchain-tmpdir.ts', 'scripts/patch-xcuitest-runner-icon.ts', 'scripts/write-xcuitest-cache-metadata.mjs', 'packages/platform-apple/src/runner/**', '!packages/platform-apple/src/runner/__tests__/**', '.github/actions/setup-apple-runner-build/action.yml') }}" >> "$GITHUB_OUTPUT" shell: bash - name: Resolve Apple runner build variant diff --git a/.github/workflows/ios.yml b/.github/workflows/ios.yml index 52e8ae072c..112920e6f2 100644 --- a/.github/workflows/ios.yml +++ b/.github/workflows/ios.yml @@ -106,7 +106,7 @@ jobs: if [ "${{ github.event_name }}" = 'pull_request' ]; then git fetch origin "$BASE_SHA" --depth=1 || echo 'Base fetch failed; XCTest selection will fail open.' fi - node --experimental-strip-types scripts/ios-xctest-impact.ts + node --experimental-strip-types scripts/apple-ci-impact.ts xctest - name: Establish host focus canary id: host-focus-canary @@ -137,128 +137,22 @@ jobs: preferred-device-name: iPhone 17 Pro - name: Run targeted iOS runner XCTest regressions - if: steps.xctest-impact.outputs.run == 'true' + id: ios-xctest + if: steps.xctest-impact.outputs.run != 'false' run: | - set -o pipefail + set -euo pipefail XCTESTRUN_PATH="$(find "$AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH/Build/Products" -maxdepth 1 -name '*.xctestrun' -print -quit)" test -n "$XCTESTRUN_PATH" + IOS_PR_TEST_ARGS=() + IOS_PR_TEST_IDS="$(node --experimental-strip-types scripts/check-xctest-selection.ts --ios-pr-tests)" + while IFS= read -r test_id; do + IOS_PR_TEST_ARGS+=("-only-testing:$test_id") + done <<< "$IOS_PR_TEST_IDS" + test "${#IOS_PR_TEST_ARGS[@]}" -gt 0 xcodebuild test-without-building \ -xctestrun "$XCTESTRUN_PATH" \ -destination "platform=iOS Simulator,id=${{ steps.ios-simulator.outputs.simulator-udid }}" \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSinglePointerFlingFallsBackToXCTestCoordinateDragWhenPrivateSynthesisFails \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSynthesizedSequenceTapFallsBackToXCTestCoordinateTapWhenAccessibilityIsUnavailable \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRecordStartThrowsTheCaptureRefusalItReceived \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAlertDispatchResolvesItsOwnModalWithoutCoordinateTapRoutingProbe \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAlertResolutionCannotBypassRequestedDeadline \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testReadRefusesToLaunchANotRunningSessionApp \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testNonReadCommandStillLaunchesANotRunningSessionApp \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testTypeWithoutResolvedInputReturnsTypedFailureBeforeDispatchingText \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testBareTypeUsesTappedInputWhenSoftwareKeyboardIsHidden \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testHardwareKeyboardResponderConfirmsItsOwnKeyboardFocus \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testPenalizedCoordinateTapOnNonTextControlDoesNotAuthorizeBareType \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testBareDelayedTypeFailsWhenTappedInputDisappearsMidCommand \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testBareSubmitKeyUsesSynthesizedFirstResponderAfterHiddenKeyboardTap \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testBareSubmitKeyRefusesWhenPrivateSynthesisIsUnavailable \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testEmptyReplacementWithoutResolvableTargetFailsClosed \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testTextEntryTapWitnessIsBoundToTargetIdentity \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testCoordinateTapTextInputProbeSkipsPenalizedXCTestChannel \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testScrollViewportKeyboardClipMatchesGoldenParityTable \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testScrollViewportPolicyUsesParityTableConstants \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testScrollViewportDispatchKeepsTheUnclippedFrameAsItsCoordinateRotationBasis \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSynthesizedGesturePoliciesMatchCommandContracts \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testFreshCoordinateTapContainsUnavailableTextInputProbe \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testTextInputProbeIssueScopeIsThreadBound \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testTextInputProbePreservesEnclosingRunnerWait \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSuppressedAxIssueMakesTextInputProbeUnavailable \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testHealthyCoordinateTapPreservesBareTypingWitness \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testTextInputProbeContainmentExcludesRequiredReadsAndLaterIssues \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testCoordinateTextInputCandidateMustBeEnabledAndContainTheTouchPoint \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testQuerySelectorPrefersHittableMatchOverNonHittableDuplicate \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testActivateTargetSkipsForegroundAndActivatesNonForegroundApplication \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testApplicationStateRawValuesMatchTheActivationDecoder \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testMissingBundleCommandInvalidatesCompleteCachedTargetState \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testCachedTargetInvalidationClearsProcessBoundState \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testScreenshotDisplayFactsReEncodeEveryGoldenTableRowUnderTheTablesKey \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testScreenshotResultCarriesNoDisplayFactsWhenNothingResolved \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAlertResolutionCannotBypassRequestedDeadline \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAlertAcceptTreatsOpenAsAffirmative \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAlertAcceptDoesNotActivateAReplacementWithASharedButton \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAlertDismissDoesNotActivateAReplacementWithTheSameTitle \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAlertCannotProveAnIdenticalReplacementAndDoesNotActivateIt \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAlertDeadlineBeforeActivationLeavesTheOriginalUntouched \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAlertHittableProbeCompletingAfterDeadlineLeavesTheOriginalUntouched \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAlertActivationIgnoresAnAppThatNeverSettlesBeforeTheDeadline \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAlertActivationDoesNotWaitOutANotificationBanner \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAlertResolutionWithoutAnAlertDoesNotReadEveryElementOfTheScreen \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAlertResolutionFindsADismissPopupMarkerOnACrowdedScreen \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAlertResolutionFindsAWindowThatIsItselfTheDismissPopupMarker \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSystemModalProbeSliceSharesAndClampsToPlanDeadline \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testDispatchRecoverySkipsBookkeepingWhileXCTestChannelOccupied \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testBoundedSystemModalProbeTimeoutRecoversThenReleasesOnDrain \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testBoundedSystemModalProbeTimeoutRecoversThenReleasesOnDrainForSnapshotRaw \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSnapshotTraversalIdentityPreservesSameOriginNodesWithDifferentBounds \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSnapshotPresentationPreservesCurrentWireShape \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRegularPresentationKeepsNestedClipGeometryCumulative \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRegularPresentationKeepsFramelessSemanticCarriersNonActionableAndNonClipping \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRawPresentationKeepsReportedOffscreenAndFramelessFacts \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testFixedSeedRegularPresentationMaintainsCumulativeClipInvariant \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testPresentationFailureKeepsItsNamedSnapshotQualityReason \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRegularPresentationPublishesEffectiveRectWhileRawKeepsReportedFrame \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSnapshotPresentationOwnsBackendNeutralEligibility \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSnapshotPresentationOwnsScopeAndRelativeDepth \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testPresentationRefusesAnAcquisitionCapturedForTheOtherProjection \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testCaptureHintIsTheOnlyAcquisitionViewOfARequest \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRegularDepthCutsPresentationNotAcquisition \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testVisibilityFoldKeepsIndependentChildPastClippedParent \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRegularDepthKeepsTurnedKeyboardSubtreeAfterOneNormalizationPass \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRegularFoldClipsScrollOverflowReparentsAndBooksHints \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRegularFoldKeepsWindowCarriersButNeverHittableOutsideClip \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRegularFoldDropsSubPixelContentlessDecorationOnEveryBackend \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testPlainViewportPolicyFoldsWithoutAncestorCursor \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testPlainViewportPolicyDoesNotClipToScrollAncestor \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testEffectiveGeometryIntersectsViewportAndAncestorClip \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testEffectiveGeometryKeepsReportedOriginWhenFullyClipped \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRegularPresentationRoutesThroughVisibilityFold \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRegularPresentationPublishesGeometricActionabilityWithoutOcclusionOrTypeGate \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSnapshotAcquisitionDoesNotReintroduceRunnerOcclusionScan \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSnapshotScopePolicyMatchesGoldenParityTable \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testScrollContainerTypeNamesMatchElementTypeSet \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testPrivateAXDepthLimitedRequiresEveryFrontierResolved \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testDeepExtensionCountsMissedFrontiers \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testPreferredPrivateAXBackendPlansAsPenalized \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testPreferredTreeBackendPinsRegularPlanAndLeavesStructuredEvidence \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRawDiagnosticPlanCarriesOnlyBackendsThatCanServeRaw \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSnapshotBackendDeclarationsMatchCapabilityFixture \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testProjectionMismatchFailureIsStructuredAndNotAnAxFailure \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testPrivateAXRegularPresentationProjectsToViewportAndKeepsScrollHint \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testPrivateAXRawProjectionKeepsEveryAcquiredNode \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testPrivateAXRawProjectionAppliesRequestedTraversalDepth \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testPrivateAXPresentationKeepsOffscreenSubtreeExcludedWhenChildFramesAreClamped \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testPrivateAXGeometrylessSemanticsAreNeverActionableOrScrollContexts \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testPrivateAXAcquisitionDoesNotInterpretScope \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testPrivateAXInteractiveFiltersLoginLikeHiddenDrawer \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testDecodedPreferredBackendReachesOptionsAndApplicablePlan \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSparsePayloadReasonMatrix \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testStampedPayloadTruncationTracksCompletenessNotRecoveryProvenance \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRunnerScreenshotStabilitySettledNeedsEnoughSamples \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRunnerScreenshotStabilitySettledTrueWhenWindowMatches \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRunnerScreenshotStabilitySettledFalseOnMidWindowMismatch \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRunnerScreenshotStabilitySettledFalseOnFailedCapture \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRunnerScreenshotStabilitySettledOnlyLooksAtTheTrailingWindow \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRunnerScreenshotStabilitySettledRejectsDegenerateRequirement \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testCustomActionsRequestPinsPrivateAXBackend \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testPrivateAXNodesCarryAnnotatedCustomActions \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRequestPinnedBackendReportsItsOwnReason \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testCustomActionCoverageParsesOnlyCompletePairs \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testStampedPayloadCarriesDisclosuresOnlyInTheVerdict \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testActionNamesAreCappedPerElementAndReported \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testHungCustomActionReadIsContainedAndRecovers \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAbandonedTreeCaptureSkipsQuerySweepAndHonorsWarmupExemption \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testNonInteractiveQuerySweepStopsAtTheSliceItsCallerWaitsFor \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testQuerySweepThatEndsOnItsSliceDeadlinePenalizesChannelAndReachesPrivateAX \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testCoordinateTapRoutingPenalizesTheIdentityMainSettledOnWhileTheWriteWasPending \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testBlockingModalSnapshotLeavesWarmupExemptionForTheFirstCapturePlan \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testPrivateAXPinnedRegularDepthReachesAcquisitionAndPresentation 2>&1 | tee /tmp/agent-device-runner-regressions.log + "${IOS_PR_TEST_ARGS[@]}" 2>&1 | tee /tmp/agent-device-runner-regressions.log node --input-type=module -e ' import { readFileSync } from "node:fs"; const log = readFileSync("/tmp/agent-device-runner-regressions.log", "utf8"); @@ -329,6 +223,17 @@ jobs: pnpm clean:daemon node --experimental-strip-types src/bin.ts test test/integration/replays/ios/device/01-physical-lifecycle.ad --udid "$IOS_UDID" --retries 2 --artifacts-dir test/artifacts/replays-ios-device-smoke --report-junit test/artifacts/replays-ios-device-smoke.junit.xml + - name: Assert iOS XCTest selection was honored + if: always() + env: + SELECTED: ${{ steps.xctest-impact.outputs.run }} + OUTCOME: ${{ steps.ios-xctest.outcome }} + run: | + if [ "$SELECTED" = 'true' ] && [ "$OUTCOME" = 'skipped' ]; then + echo '::error::iOS runner XCTest was selected but its step was skipped.' + exit 1 + fi + - name: Upload iOS artifacts if: always() uses: ./.github/actions/upload-agent-device-artifacts diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 75194466b4..3720a4cbaa 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -153,25 +153,18 @@ jobs: test/artifacts/replays-macos.junit.xml # Clean-install verification may raise macOS local-network permission UI; keep it after replay. - - name: Verify clean-installed Simulator snapshot bridge preparation and the fold-helper -Werror gate + - name: Select Apple bridge proof if: github.event_name == 'pull_request' + id: bridge-impact env: BASE_SHA: ${{ github.event.pull_request.base.sha }} run: | - git fetch origin "$BASE_SHA" --depth=1 - if git diff --quiet "$BASE_SHA"...HEAD -- \ - apple/snapshot-bridge \ - apple/fold-helper \ - packages/platform-apple/src/snapshot-source \ - packages/platform-apple/src/foldable \ - scripts/check-package.ts \ - scripts/size-report-install.mjs \ - scripts/size-report-package.mjs \ - .github/workflows/ios.yml \ - .github/workflows/macos.yml; then - echo "Snapshot bridge and fold-helper sources are unchanged; skipping preparation proof." - exit 0 - fi + git fetch origin "$BASE_SHA" --depth=1 || echo 'Base fetch failed; bridge proof selection will fail open.' + node --experimental-strip-types scripts/apple-ci-impact.ts bridge + + - name: Verify clean-installed Simulator snapshot bridge preparation and the fold-helper -Werror gate + if: github.event_name == 'pull_request' && steps.bridge-impact.outputs.run != 'false' + run: | pnpm build pnpm exec vitest run \ packages/platform-apple/src/snapshot-source/native-runtime.test.ts \ diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ApplicationStateRawValueTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ApplicationStateRawValueTests.swift index 02375b2a99..84c8f2b472 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ApplicationStateRawValueTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ApplicationStateRawValueTests.swift @@ -7,8 +7,8 @@ extension RunnerTests { /// against a second handwritten table: the decoder maps the integer the runner stamps, and #2726 /// shipped `runningBackground` and `runningBackgroundSuspended` reversed. /// - /// The host lane runs this on every PR, and `.github/workflows/ios.yml` lists it for the simulator - /// lane so the suspended case — compiled out of the macOS build below — is pinned on every PR too. + /// The host lane runs this on every PR; the simulator lane derives it from the platform branch + /// below so the suspended case, compiled out of the macOS build, is pinned on every PR too. /// `packages/platform-apple/src/runner/__tests__/target-activation.test.ts` reads these calls back /// and compares them with its decode table, so a table that drifts from them fails on any host /// instead of waiting for a lane to report a mislabelled repair. diff --git a/docs/agents/testing.md b/docs/agents/testing.md index 0ef018e0ba..f0dbb77db8 100644 --- a/docs/agents/testing.md +++ b/docs/agents/testing.md @@ -45,7 +45,8 @@ provider, and coverage tests mock the typed HDC seam. Real validation is local h Apple runner changes run `pnpm check:xctest-selection` and build the affected target. The source `#if` guard is the XCTest lane classification — never maintain a second test-name list. Pure runner decisions use the macOS host lane; iOS/XCTest semantics need a simulator lane. -The iOS PR lane runs its XCTest list for native inputs or uncertain diffs; main and nightly remain broad. +The iOS PR lane derives platform-specific XCTests from Swift guards for runner changes; +nightly runs the full suite. Local host-lane XCTest may need signing and automation permission: diff --git a/scripts/__tests__/apple-ci-impact.test.ts b/scripts/__tests__/apple-ci-impact.test.ts new file mode 100644 index 0000000000..1effb8b52a --- /dev/null +++ b/scripts/__tests__/apple-ci-impact.test.ts @@ -0,0 +1,114 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { expect, test } from 'vitest'; +import { parse } from 'yaml'; +import { selectAppleBridgeProof, selectIosXctests } from '../apple-ci-impact.ts'; +import { selectChecks } from '../check-affected/model.ts'; + +const repoRoot = path.resolve(import.meta.dirname, '../..'); + +function cacheInputs(action: string): string[] { + const doc = parse(action) as { runs?: { steps?: Array<{ id?: string; run?: string }> } }; + const sourceHash = doc.runs?.steps?.find((step) => step.id === 'source-hash')?.run ?? ''; + const expressions = [...sourceHash.matchAll(/hashFiles\(([\s\S]*?)\)/g)]; + expect(expressions.length).toBeGreaterThan(0); + return expressions.flatMap((expression) => + [...expression[1]!.matchAll(/'([^']+)'/g)].map((match) => match[1]!), + ); +} + +test('every runner build-cache input triggers the PR XCTest lane', () => { + const action = fs.readFileSync( + path.join(repoRoot, '.github/actions/setup-apple-runner-build/action.yml'), + 'utf8', + ); + expect(cacheInputs(action)).toContain('packages/platform-apple/src/runner/**'); + expect(cacheInputs(action)).toContain('!packages/platform-apple/src/runner/__tests__/**'); + const uncovered = (text: string) => + cacheInputs(text) + .filter((input) => !input.startsWith('!')) + .filter((input) => { + const path = input.replace(/\*\*?$/, 'probe.ts'); + const plan = selectChecks({ changedFiles: [path] }); + return !plan.failOpen && !plan.checks.includes('swift-runner-ios'); + }); + expect(uncovered(action)).toEqual([]); + expect( + uncovered(action.replace('apple/runner/**', 'packages/platform-apple/src/snapshot-source/**')), + ).toEqual(['packages/platform-apple/src/snapshot-source/**']); + expect( + uncovered( + action.replace( + 'hashFiles(', + "hashFiles('packages/platform-apple/src/foldable/**',\n ", + ), + ), + ).toEqual(['packages/platform-apple/src/foldable/**']); +}); + +test('the PR workflow applies the impact decision to the XCTest step', () => { + const workflow = fs.readFileSync(path.join(repoRoot, '.github/workflows/ios.yml'), 'utf8'); + expect(workflow).toContain('node --experimental-strip-types scripts/apple-ci-impact.ts xctest'); + expect(workflow).toMatch( + /- name: Run targeted iOS runner XCTest regressions\n\s+id: ios-xctest\n\s+if: steps\.xctest-impact\.outputs\.run != 'false'/, + ); + expect(workflow).toContain('if [ "$SELECTED" = \'true\' ] && [ "$OUTCOME" = \'skipped\' ]'); +}); + +test('macOS clean-install proof follows live UI replay', () => { + const workflow = fs.readFileSync(path.join(repoRoot, '.github/workflows/macos.yml'), 'utf8'); + const replay = workflow.indexOf('- name: Run macOS integration test'); + const proof = workflow.indexOf( + '- name: Verify clean-installed Simulator snapshot bridge preparation', + ); + expect(replay).toBeGreaterThan(-1); + expect(proof).toBeGreaterThan(replay); + expect(workflow).toContain('node --experimental-strip-types scripts/apple-ci-impact.ts bridge'); + expect(workflow).toContain("steps.bridge-impact.outputs.run != 'false'"); +}); + +test('native runner and golden-table changes run XCTest; TypeScript runtime changes use live E2E', () => { + for (const file of [ + 'apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift', + 'apple/snapshot-presentation/Sources/Presenter.swift', + 'packages/platform-apple/src/runner/runner-icon.ts', + 'contracts/fixtures/scroll-gesture.json', + '.github/workflows/ios.yml', + 'scripts/apple-ci-impact.ts', + ]) { + expect(selectIosXctests('pull_request', [file]).run, file).toBe(true); + } + for (const file of [ + 'packages/platform-apple/src/snapshot-source/cache.ts', + 'packages/platform-apple/src/runner/__tests__/runner-icon.test.ts', + 'apple/fold-helper/fold-helper.c', + 'test/integration/ios-simulator-e2e/live-runner.ts', + ]) { + expect(selectIosXctests('pull_request', [file]), file).toMatchObject({ run: false }); + } +}); + +test('pushes and uncertain diffs keep the full XCTest selection', () => { + expect(selectIosXctests('push', ['src/index.ts']).run).toBe(true); + expect(selectIosXctests('pull_request', null).run).toBe(true); + expect(selectIosXctests('pull_request', []).run).toBe(true); + expect(selectIosXctests('pull_request', ['src/index.ts', 'package.json']).run).toBe(true); +}); + +test('bridge proof runs for its owning sources and uncertain tooling changes', () => { + for (const file of [ + 'apple/snapshot-bridge/Bridge.c', + 'apple/fold-helper/Helper.c', + 'apple/new-native-module/Source.m', + 'packages/platform-apple/src/snapshot-source/native-runtime.ts', + 'packages/platform-apple/src/foldable/fold-helper-cache.ts', + 'packages/platform-apple/src/new-module/runtime.ts', + 'scripts/check-package.ts', + '.github/workflows/macos.yml', + ]) { + expect(selectAppleBridgeProof([file]).run, file).toBe(true); + } + expect(selectAppleBridgeProof(['src/index.ts']).run).toBe(false); + expect(selectAppleBridgeProof(null).run).toBe(true); + expect(selectAppleBridgeProof([]).run).toBe(true); +}); diff --git a/scripts/__tests__/ios-xctest-impact.test.ts b/scripts/__tests__/ios-xctest-impact.test.ts deleted file mode 100644 index 673978a55a..0000000000 --- a/scripts/__tests__/ios-xctest-impact.test.ts +++ /dev/null @@ -1,63 +0,0 @@ -import fs from 'node:fs'; -import path from 'node:path'; -import { expect, test } from 'vitest'; -import { - affectsIosXctests, - selectIosXctests, - uncoveredRunnerCacheInputs, -} from '../ios-xctest-impact.ts'; - -test('every runner build-cache input triggers the PR XCTest lane', () => { - const action = fs.readFileSync( - path.resolve('.github/actions/setup-apple-runner-build/action.yml'), - 'utf8', - ); - expect(uncoveredRunnerCacheInputs(action)).toEqual([]); - expect( - uncoveredRunnerCacheInputs(action.replace('apple/runner/**', 'new-native-input/**')), - ).toEqual(['new-native-input/**']); -}); - -test('the PR workflow applies the impact decision to the XCTest step', () => { - const workflow = fs.readFileSync(path.resolve('.github/workflows/ios.yml'), 'utf8'); - expect(workflow).toContain('node --experimental-strip-types scripts/ios-xctest-impact.ts'); - expect(workflow).toMatch( - /- name: Run targeted iOS runner XCTest regressions\n\s+if: steps\.xctest-impact\.outputs\.run == 'true'/, - ); -}); - -test('macOS clean-install proof follows live UI replay', () => { - const workflow = fs.readFileSync(path.resolve('.github/workflows/macos.yml'), 'utf8'); - const replay = workflow.indexOf('- name: Run macOS integration test'); - const proof = workflow.indexOf( - '- name: Verify clean-installed Simulator snapshot bridge preparation', - ); - expect(replay).toBeGreaterThan(-1); - expect(proof).toBeGreaterThan(replay); -}); - -test('native runner and golden-table changes run XCTest; TypeScript runtime changes use live E2E', () => { - for (const file of [ - 'apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift', - 'apple/snapshot-presentation/Sources/Presenter.swift', - 'contracts/fixtures/scroll-gesture.json', - '.github/workflows/ios.yml', - 'scripts/ios-xctest-impact.ts', - ]) { - expect(affectsIosXctests(file), file).toBe(true); - } - for (const file of [ - 'packages/platform-apple/src/snapshot-source/cache.ts', - 'apple/fold-helper/fold-helper.c', - 'test/integration/ios-simulator-e2e/live-runner.ts', - ]) { - expect(selectIosXctests('pull_request', [file]), file).toMatchObject({ run: false }); - } -}); - -test('pushes and uncertain diffs keep the full XCTest selection', () => { - expect(selectIosXctests('push', ['src/index.ts']).run).toBe(true); - expect(selectIosXctests('pull_request', null).run).toBe(true); - expect(selectIosXctests('pull_request', []).run).toBe(true); - expect(selectIosXctests('pull_request', ['src/index.ts', 'package.json']).run).toBe(true); -}); diff --git a/scripts/__tests__/xctest-declarations.test.ts b/scripts/__tests__/xctest-declarations.test.ts index 70edbdd68b..e1f77fab14 100644 --- a/scripts/__tests__/xctest-declarations.test.ts +++ b/scripts/__tests__/xctest-declarations.test.ts @@ -67,6 +67,50 @@ describe('the declaration scan', () => { ).toEqual([`${TARGET}/RunnerTests/testGolden`]); }); + test('discovers tests in indented extensions and new files without a name list', () => { + expect( + parseDeclaredTests(TARGET, [ + { + file: 'UnitTests/NewScreenCaptureTests.swift', + text: + '#if os(iOS)\n' + + ' extension RunnerTests {\n' + + ' func testObservedScreenCapture() {}\n' + + ' }\n' + + '#endif\n', + }, + { + file: 'NewTests.swift', + text: ' extension RunnerTests {\n func testAnotherIndent() {}\n }\n', + }, + ]), + ).toEqual([ + `${TARGET}/RunnerTests/testAnotherIndent`, + `${TARGET}/RunnerTests/testObservedScreenCapture`, + ]); + }); + + test('fails closed when a test-shaped declaration cannot be classified', () => { + expect(() => parseDeclaredTests(TARGET, source('func testOutsideAType() {}\n'))).toThrow( + 'RunnerTests+Fixture.swift:1: unrecognized XCTest declaration', + ); + expect(() => + parseDeclaredTests( + TARGET, + source('extension RunnerTests {\n func testGeneric() {}\n}\n'), + ), + ).toThrow('RunnerTests+Fixture.swift:2: unrecognized XCTest declaration'); + expect(() => + parseDeclaredTests(TARGET, source('extension RunnerTests { func testInline() {} }\n')), + ).toThrow('RunnerTests+Fixture.swift:1: unrecognized XCTest declaration'); + expect(() => + parseDeclaredTests( + TARGET, + source('extension RunnerTests {\n @available(iOS 17, *) func testAttributed() {}\n}\n'), + ), + ).toThrow('RunnerTests+Fixture.swift:2: unrecognized XCTest declaration'); + }); + test('attributes each declared method to the platforms that compile it', () => { expect( parseDeclaredTestsByPlatform( diff --git a/scripts/__tests__/xctest-selection.test.ts b/scripts/__tests__/xctest-selection.test.ts index 20f282fdea..af59b84761 100644 --- a/scripts/__tests__/xctest-selection.test.ts +++ b/scripts/__tests__/xctest-selection.test.ts @@ -1,8 +1,8 @@ // The check that keeps the runner XCTest lanes honest is itself only as good as its // parsers, and all of its inputs are files nobody edits with this check in mind. So: the -// real tree must pass, a planted typo in the real workflow text must fail — in both flag -// directions, because an unknown `-skip-testing:` entry re-arms a whole-bundle lane's -// 24-hour hang on `RunnerTests/testCommand` — and a planted guard that compiles a test out +// real tree must pass, a broken generated PR selector and a planted typo in a whole-bundle +// `-skip-testing:` flag must fail (the latter re-arms the 24-hour `testCommand` hang), and a +// planted guard that compiles a test out // of every lane must fail as "dark". Synthetic sources cover the shapes the real tree // happens not to contain today. @@ -17,6 +17,7 @@ import { formatSummary, GUARDED_WORKFLOWS, HOST_WORKFLOW_FILE, + iosPrTestIdentifiers, LANES, loadReport, NIGHTLY_WORKFLOW_FILE, @@ -48,12 +49,18 @@ function realSources() { return readSwiftSources(path.join(repoRoot, RUNNER_TESTS_DIR)); } -/** Workflow texts that skip the entry point on both whole-bundle lanes and list `pr` on the PR lane. */ -function laneWorkflows(pr: readonly string[] = []): WorkflowSource[] { +/** Workflow texts that skip the entry point and consume the guard-derived PR selection. */ +function laneWorkflows(): WorkflowSource[] { return [ { workflow: HOST_WORKFLOW_FILE, text: `-skip-testing:${ENTRY_POINT}` }, { workflow: NIGHTLY_WORKFLOW_FILE, text: `-skip-testing:${ENTRY_POINT}` }, - { workflow: PR_WORKFLOW_FILE, text: pr.map((id) => `-only-testing:${id}`).join('\n') }, + { + workflow: PR_WORKFLOW_FILE, + text: + 'node --experimental-strip-types scripts/check-xctest-selection.ts --ios-pr-tests\n' + + 'IOS_PR_TEST_ARGS+=("-only-testing:$test_id")\n' + + 'xcodebuild "${IOS_PR_TEST_ARGS[@]}"', + }, ]; } @@ -69,8 +76,9 @@ describe('the real tree', () => { const { declared, host, pr, nightly, dark } = counts(report); // Not pinned to today's exact numbers; the invariants are the shape. The host lane // (macOS) and the nightly (iOS) both skip only the entry point, so together with the - // simulator-only guard they cover everything else; the PR list is a proper subset of - // the nightly; and the entry point is the only method outside every lane. + // simulator-only guard they cover everything else; the PR lane runs iOS-only methods + // and shared methods with platform-dependent bodies. The entry point is the only method + // outside every lane. expect(host).toBeGreaterThan(0); expect(nightly).toBeGreaterThan(pr); expect(pr).toBeGreaterThan(0); @@ -79,6 +87,19 @@ describe('the real tree', () => { expect(reachedAnywhere.size).toBe(declared - 1); expect(reachedAnywhere.has(ENTRY_POINT)).toBe(false); for (const id of report.reach.pr) expect(report.reach.nightly.has(id)).toBe(true); + expect(iosPrTestIdentifiers(report.declaredTests, report.sharedPlatformBranchIds)).toEqual([ + ...report.reach.pr, + ]); + expect( + report.reach.pr.has( + `${TARGET}/RunnerTests/testSnapshotTraversalIdentityPreservesSameOriginNodesWithDifferentBounds`, + ), + ).toBe(true); + expect( + report.reach.pr.has( + `${TARGET}/RunnerTests/testApplicationStateRawValuesMatchTheActivationDecoder`, + ), + ).toBe(true); }); test('the whole-bundle lanes skip the runner server entry point, which is not a test', () => { @@ -110,7 +131,8 @@ describe('the real tree', () => { test('the declared set covers every addressable method in the target directory', () => { // Derived independently of the check: the directory is globbed here, with this test's - // own regex, because the Xcode project uses a PBXFileSystemSynchronizedRootGroup — every + // own loose declaration count, because the Xcode project uses a + // PBXFileSystemSynchronizedRootGroup — every // .swift file in it is a member. Reusing the check's own file filter would make this // tautological, and a name-based filter is exactly the bug it caught // (RunnerTapPointPolicy.swift declared a test and does not start with "RunnerTests"). @@ -121,7 +143,9 @@ describe('the real tree', () => { if (entry.isDirectory()) return total + countAddressableMethods(entryPath); if (!entry.isFile() || !entry.name.endsWith('.swift')) return total; const text = fs.readFileSync(entryPath, 'utf8'); - return total + (text.match(/^ {2}(?:[\w@]+ )*func test/gm)?.length ?? 0); + return ( + total + (text.match(/^[ \t]*(?:[\w@]+[ \t]+)*func[ \t]+test\w*[ \t]*\(/gm)?.length ?? 0) + ); }, 0); const counted = countAddressableMethods(directory); @@ -164,24 +188,36 @@ describe('the real tree', () => { }); describe('a planted typo', () => { - test('a renamed test in the PR `-only-testing:` list is reported with its line', () => { - const workflow = fs.readFileSync(path.join(repoRoot, PR_WORKFLOW_FILE), 'utf8'); - const first = parseFlaggedTests(PR_WORKFLOW_FILE, workflow).find( - (entry) => entry.flag === 'only-testing', + test('a disconnected PR selector fails the workflow wiring check', () => { + const workflow = laneWorkflows().find((entry) => entry.workflow === PR_WORKFLOW_FILE); + if (!workflow?.text) throw new Error('Missing synthetic PR workflow'); + const report = buildReport( + TARGET, + source( + `${ENTRY_SOURCE}#if os(iOS)\nextension RunnerTests {\n func testIosOnly() {}\n}\n#endif\n`, + ), + laneWorkflows().map((entry) => + entry.workflow === PR_WORKFLOW_FILE + ? { ...entry, text: workflow.text?.replace('--ios-pr-tests', '--wrong-mode') ?? null } + : entry, + ), ); - if (!first) throw new Error('ios.yml has no -only-testing entries to plant a typo in'); - const typo = `${first.identifier}Renamed`; + expect(reportFailures(report).join('\n')).toContain('does not invoke'); + }); + test('a handwritten PR identifier cannot silently narrow the generated selection', () => { const report = buildReport( TARGET, - realSources(), - realWorkflows({ [PR_WORKFLOW_FILE]: workflow.replace(first.identifier, typo) }), + source( + `${ENTRY_SOURCE}#if os(iOS)\nextension RunnerTests {\n func testIosOnly() {}\n}\n#endif\n`, + ), + laneWorkflows().map((entry) => + entry.workflow === PR_WORKFLOW_FILE + ? { ...entry, text: `${entry.text}\n-only-testing:${TARGET}/RunnerTests/testIosOnly` } + : entry, + ), ); - - expect(report.unknown).toEqual([ - { workflow: PR_WORKFLOW_FILE, flag: 'only-testing', identifier: typo, line: first.line }, - ]); - expect(reportFailures(report).join('\n')).toContain(typo); + expect(reportFailures(report).join('\n')).toContain('hand-maintained XCTest selection'); }); test.each([ @@ -209,38 +245,77 @@ describe('a planted typo', () => { }, ); - test('a deleted test is reported even though the surviving list still passes', () => { - const kept = `${ENTRY_SOURCE}extension RunnerTests {\n func testKept() {}\n}\n`; - const workflows = laneWorkflows([ - `${TARGET}/RunnerTests/testKept`, - `${TARGET}/RunnerTests/testGone`, - ]); - + test('adding or removing an iOS-only method changes the PR selection automatically', () => { + const iosMethod = '#if os(iOS)\nextension RunnerTests {\n func testIosOnly() {}\n}\n#endif\n'; + const id = `${TARGET}/RunnerTests/testIosOnly`; expect( - buildReport( + buildReport(TARGET, source(`${ENTRY_SOURCE}${iosMethod}`), laneWorkflows()).reach.pr.has(id), + ).toBe(true); + expect(buildReport(TARGET, source(ENTRY_SOURCE), laneWorkflows()).reach.pr.has(id)).toBe(false); + }); + + test.each(['os(iOS)', 'canImport(UIKit)', 'targetEnvironment(simulator)'])( + 'a shared method with an in-body %s branch enters the generated PR selection', + (condition) => { + const report = buildReport( TARGET, - source(`${kept}extension RunnerTests {\n func testGone() {}\n}\n`), - workflows, - ).unknown, - ).toEqual([]); - expect( - buildReport(TARGET, source(kept), workflows).unknown.map((entry) => entry.identifier), - ).toEqual([`${TARGET}/RunnerTests/testGone`]); + source( + `${ENTRY_SOURCE}extension RunnerTests {\n func testSharedBranch() {\n#if ${condition}\n XCTAssertTrue(true)\n#else\n XCTAssertFalse(true)\n#endif\n }\n}\n`, + ), + laneWorkflows(), + ); + const id = `${TARGET}/RunnerTests/testSharedBranch`; + expect(report.declaredTests.find((test) => test.identifier === id)?.platforms).toEqual([ + 'iOS', + 'macOS', + 'tvOS', + ]); + expect(report.sharedPlatformBranchIds).toEqual([id]); + expect(report.reach.pr.has(id)).toBe(true); + expect(reportFailures(report)).toEqual([]); + }, + ); + + test('a platform-neutral in-body compile flag does not duplicate a host test', () => { + const report = buildReport( + TARGET, + source( + `${ENTRY_SOURCE}extension RunnerTests {\n func testSharedDebug() {\n#if DEBUG\n XCTAssertTrue(true)\n#endif\n }\n}\n`, + ), + laneWorkflows(), + ); + expect(report.sharedPlatformBranchIds).toEqual([]); + expect(report.reach.pr.has(`${TARGET}/RunnerTests/testSharedDebug`)).toBe(false); + }); + + test('ambiguous shared method names fail closed instead of losing platform branches', () => { + const report = buildReport( + TARGET, + source( + `${ENTRY_SOURCE}extension RunnerTests {\n func testSameName() {}\n}\nfinal class OtherTests: XCTestCase {\n func testSameName() {}\n}\n`, + ), + laneWorkflows(), + ); + expect(reportFailures(report).join('\n')).toContain( + 'ambiguous shared XCTest method testSameName', + ); }); - test('a listed test the PR lane platform never compiles is reported, not silently unmatched', () => { - // Declared, so the rename check passes — but ios.yml builds for iOS, and an - // `os(macOS)` guard means the identifier matches nothing there. + test('a skip flag naming a test that its lane cannot compile is reported', () => { const report = buildReport( TARGET, source( - `${ENTRY_SOURCE}extension RunnerTests {\n#if os(macOS)\n func testHostOnly() {}\n#endif\n}\n`, + `${ENTRY_SOURCE}#if os(iOS)\nextension RunnerTests {\n func testIosOnly() {}\n}\n#endif\n`, + ), + laneWorkflows().map((entry) => + entry.workflow === HOST_WORKFLOW_FILE + ? { ...entry, text: `${entry.text}\n-skip-testing:${TARGET}/RunnerTests/testIosOnly` } + : entry, ), - laneWorkflows([`${TARGET}/RunnerTests/testHostOnly`]), ); expect(report.unknown).toEqual([]); expect(report.uncompiled.map((entry) => entry.identifier)).toEqual([ - `${TARGET}/RunnerTests/testHostOnly`, + `${TARGET}/RunnerTests/testIosOnly`, ]); expect(reportFailures(report).join('\n')).toContain('never compiles'); }); @@ -276,7 +351,7 @@ describe('a planted guard', () => { const id = `${TARGET}/RunnerTests/testLaunchesApp`; expect(report.reach.host.has(id)).toBe(false); expect(report.reach.nightly.has(id)).toBe(true); - expect(report.reach.pr.has(id)).toBe(false); + expect(report.reach.pr.has(id)).toBe(true); expect(reportFailures(report)).toEqual([]); }); @@ -284,15 +359,19 @@ describe('a planted guard', () => { const report = buildReport(TARGET, source(ENTRY_SOURCE), [ { workflow: HOST_WORKFLOW_FILE, text: 'run: xcodebuild test-without-building' }, { workflow: NIGHTLY_WORKFLOW_FILE, text: `-skip-testing:${ENTRY_POINT}` }, - { workflow: PR_WORKFLOW_FILE, text: `-only-testing:${TARGET}/RunnerTests/testOther` }, + laneWorkflows().find((entry) => entry.workflow === PR_WORKFLOW_FILE)!, ]); expect(report.entryPointReachedBy).toEqual(['host']); expect(reportFailures(report).join('\n')).toContain('reachable by lane(s): host'); }); - test('the PR list naming the entry point is reported too', () => { - const report = buildReport(TARGET, source(ENTRY_SOURCE), laneWorkflows([ENTRY_POINT])); - expect(report.entryPointReachedBy).toEqual(['pr']); + test('the generated PR selection excludes the entry point even under an iOS-only guard', () => { + const report = buildReport( + TARGET, + source(`#if os(iOS)\n${ENTRY_SOURCE}#endif\n`), + laneWorkflows(), + ); + expect(report.reach.pr.has(ENTRY_POINT)).toBe(false); }); }); @@ -386,9 +465,7 @@ describe('the blind-parse guards', () => { test('a guarded workflow that no longer exists fails instead of leaving a stale claim', () => { const report = buildReport(TARGET, oneTest(), [ - ...laneWorkflows([`${TARGET}/RunnerTests/testOne`]).filter( - (entry) => entry.workflow !== NIGHTLY_WORKFLOW_FILE, - ), + ...laneWorkflows().filter((entry) => entry.workflow !== NIGHTLY_WORKFLOW_FILE), { workflow: NIGHTLY_WORKFLOW_FILE, text: null }, ]); expect(reportFailures(report).join('\n')).toContain(NIGHTLY_WORKFLOW_FILE); @@ -411,7 +488,7 @@ describe('the summary line', () => { const { declared, host, pr, nightly, dark } = counts(report); const summary = formatSummary(report); expect(summary).toContain(`${declared} declared`); - expect(summary).toContain(`reaches ${host}, PR list`); + expect(summary).toContain(`reaches ${host}, PR iOS-specific lane`); expect(summary).toContain(`selects ${pr}, nightly`); expect(summary).toContain(`reaches ${nightly};`); expect(summary).toContain(`${dark} reachable by no lane`); diff --git a/scripts/apple-ci-impact.ts b/scripts/apple-ci-impact.ts new file mode 100644 index 0000000000..4e79bfe3d6 --- /dev/null +++ b/scripts/apple-ci-impact.ts @@ -0,0 +1,88 @@ +import fs from 'node:fs'; +import { spawnSync } from 'node:child_process'; +import { pathToFileURL } from 'node:url'; +import { selectChecks } from './check-affected/model.ts'; + +type Selection = { run: boolean; reason: string }; + +function checkOwnership( + changedPaths: readonly string[] | null, +): { early: Selection; plan?: never } | { early?: never; plan: ReturnType } { + if (changedPaths === null) { + return { early: { run: true, reason: 'the PR change set could not be established' } }; + } + if (changedPaths.length === 0) { + return { early: { run: true, reason: 'the PR diff is empty' } }; + } + const plan = selectChecks({ changedFiles: changedPaths }); + if (plan.failOpen) { + return { + early: { + run: true, + reason: `affected-check ownership is uncertain: ${plan.failOpenReasons[0]?.path}`, + }, + }; + } + return { plan }; +} + +export function selectIosXctests( + eventName: string, + changedPaths: readonly string[] | null, +): Selection { + if (eventName !== 'pull_request') + return { run: true, reason: 'main and manual runs exercise XCTest' }; + const checked = checkOwnership(changedPaths); + if (checked.early) return checked.early; + const input = checked.plan.reasons.find((entry) => entry.check === 'swift-runner-ios'); + return input + ? { run: true, reason: `XCTest input changed: ${input.path}` } + : { run: false, reason: 'the affected-check model selected no iOS runner build' }; +} + +export function selectAppleBridgeProof(changedPaths: readonly string[] | null): Selection { + const checked = checkOwnership(changedPaths); + if (checked.early) return checked.early; + const input = changedPaths.find( + (file) => file.startsWith('apple/') || file.startsWith('packages/platform-apple/src/'), + ); + return input + ? { run: true, reason: `Apple implementation changed: ${input}` } + : { run: false, reason: 'the PR changed no Apple implementation or gate tooling' }; +} + +function changedPathsFromGit(baseSha: string): string[] | null { + if (!/^[a-f0-9]{40}$/.test(baseSha)) return null; + const result = spawnSync('git', ['diff', '--name-only', '-z', `${baseSha}...HEAD`], { + encoding: 'utf8', + }); + if (result.error || result.status !== 0 || result.stdout === null) return null; + return result.stdout.split('\0').filter(Boolean); +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) { + const target = process.argv[2]; + if (target !== 'xctest' && target !== 'bridge') { + process.stderr.write('Expected xctest or bridge target\n'); + process.exitCode = 2; + } else { + const eventName = process.env.GITHUB_EVENT_NAME ?? ''; + const changedPaths = + eventName === 'pull_request' ? changedPathsFromGit(process.env.BASE_SHA ?? '') : []; + const selection = + target === 'xctest' + ? selectIosXctests(eventName, changedPaths) + : selectAppleBridgeProof(changedPaths); + const label = target === 'xctest' ? 'iOS XCTest' : 'Apple bridge proof'; + process.stdout.write(`${label}: ${selection.run ? 'run' : 'skip'}; ${selection.reason}\n`); + if (process.env.GITHUB_OUTPUT) { + fs.appendFileSync(process.env.GITHUB_OUTPUT, `run=${selection.run}\n`); + } + if (process.env.GITHUB_STEP_SUMMARY) { + fs.appendFileSync( + process.env.GITHUB_STEP_SUMMARY, + `### ${label} selection\n\n${selection.run ? 'Run' : 'Skip'}: ${selection.reason}.\n`, + ); + } + } +} diff --git a/scripts/check-affected/model.ts b/scripts/check-affected/model.ts index d21e10de21..3cfeef6731 100644 --- a/scripts/check-affected/model.ts +++ b/scripts/check-affected/model.ts @@ -440,6 +440,12 @@ const daemonWireCompatOwnership: OwnershipRule = ({ file }) => { ]; }; +const ownsAppleRunnerBuildSource = (file: string): boolean => + file.startsWith('apple/runner/') || + file.startsWith('apple/snapshot-presentation/') || + (file.startsWith('packages/platform-apple/src/runner/') && !file.includes('/__tests__/')) || + file.endsWith('.swift'); + const BUILD_OWNERSHIP: ReadonlyArray<{ check: CheckId; rule: string; @@ -455,19 +461,19 @@ const BUILD_OWNERSHIP: ReadonlyArray<{ file.startsWith('apple/snapshot-presentation/') || file === 'contracts/fixtures/ios-snapshot-engine-conformance.json', }, - // Both platform builds compile the same runner sources, and each is a separate - // gate in a separate lane, so a Swift change owns both. + // The native runner cache hashes these source trees for both Apple targets. + // Keep the build owner broader than the current file extensions. { check: 'swift-runner-ios', rule: 'own:swift', detail: 'Swift runner sources require the iOS XCUITest build', - owns: (file) => file.startsWith('apple/runner/') || file.endsWith('.swift'), + owns: ownsAppleRunnerBuildSource, }, { check: 'swift-runner-macos', rule: 'own:swift', detail: 'Swift runner sources require the macOS XCUITest build', - owns: (file) => file.startsWith('apple/runner/') || file.endsWith('.swift'), + owns: ownsAppleRunnerBuildSource, }, // The PR lane names each runner XCTest method it runs, so renaming or deleting one // silently shrinks that lane. Selected here so the drift shows up on the change that diff --git a/scripts/check-xctest-selection.ts b/scripts/check-xctest-selection.ts index be6226ed9a..b6c60b53c2 100644 --- a/scripts/check-xctest-selection.ts +++ b/scripts/check-xctest-selection.ts @@ -9,7 +9,8 @@ // `-skip-testing:` — the pure runner-decision tests, whose guard is // `#if AGENT_DEVICE_RUNNER_UNIT_TESTS` alone. // - pr ios.yml, iOS Simulator, when runner inputs change on a PR (and on every main -// push): the hand-written `-only-testing:` list. +// push): methods compiled only for iOS, plus shared methods with platform- +// dependent bodies, derived from Swift guards. // - nightly xctest-nightly.yml, iOS Simulator, scheduled: the whole bundle as compiled for // iOS, minus `-skip-testing:` — includes the simulator-only tests, whose guard is // `… && os(iOS)` (they launch the host app, route through SpringBoard, or assert an @@ -19,12 +20,10 @@ // platform rather than treating a source-level `func test…` as running everywhere. What it // holds: // -// 1. Every `-only-testing:`/`-skip-testing:` identifier names a declared method that -// compiles for that lane's platform. `xcodebuild` treats an identifier matching nothing -// as an empty set rather than an error, in BOTH directions: an unknown `-only-testing:` -// drops a test from the PR lane silently, and an unknown `-skip-testing:` re-admits -// `RunnerTests/testCommand` — not a test but the runner's server entry point, which opens -// an NWListener and waits 24 hours — into a whole-bundle lane and hangs it. +// 1. Every `-skip-testing:` identifier names a declared method that compiles for its lane, +// and the PR workflow consumes the generated list. `xcodebuild` treats a skip identifier +// matching nothing as an empty set, which re-admits `RunnerTests/testCommand` — not a test +// but the runner's server entry point, which opens an NWListener and waits 24 hours. // 2. Every declared method is reachable by at least one lane. A test gated to a platform // no lane runs (the tvOS-only pair this check found) is dark from the day it is written. // 3. The entry point is reachable by no lane at all. @@ -37,7 +36,7 @@ import fs from 'node:fs'; import path from 'node:path'; import { pathToFileURL } from 'node:url'; import { runCmdSync } from '@agent-device/host-kit/command'; -import type { Platform } from './swift-conditional-compilation.ts'; +import { activeSource, type Platform } from './swift-conditional-compilation.ts'; import { parseDeclaredTestsByPlatform, readSwiftSources, @@ -52,7 +51,7 @@ const packageAppleRunnerScript = path.join(repoRoot, 'scripts/package-apple-runn /** The macOS host lane, which runs the whole macOS-compiled bundle on every PR. */ export const HOST_WORKFLOW_FILE = '.github/workflows/macos.yml'; -/** The PR lane, whose `-only-testing:` list decides what selected pull requests run on the simulator. */ +/** The PR lane, whose generated iOS-specific list runs on selected pull requests. */ export const PR_WORKFLOW_FILE = '.github/workflows/ios.yml'; /** The nightly lane, whose `-skip-testing:` list decides what the full simulator suite leaves out. */ @@ -64,8 +63,8 @@ export type Lane = { readonly id: LaneId; readonly workflow: string; readonly platform: Platform; - /** `whole`: everything compiled minus `-skip-testing:`; `list`: the `-only-testing:` entries. */ - readonly selection: 'whole' | 'list'; + /** `whole`: everything compiled minus `-skip-testing:`; `ios-specific`: derived from guards. */ + readonly selection: 'whole' | 'ios-specific'; /** The job-summary heading the lane's reporter prints. */ readonly title: string; }; @@ -82,8 +81,8 @@ export const LANES: readonly Lane[] = [ id: 'pr', workflow: PR_WORKFLOW_FILE, platform: 'iOS', - selection: 'list', - title: 'iOS runner PR XCTest list', + selection: 'ios-specific', + title: 'iOS-specific runner PR XCTest lane', }, { id: 'nightly', @@ -159,6 +158,12 @@ export type SelectionReport = { readonly dark: readonly string[]; /** Lanes that reach the entry point — a failure (a 24-hour hang). */ readonly entryPointReachedBy: readonly LaneId[]; + /** Whether the PR workflow runs and consumes the generated iOS-specific list. */ + readonly prWorkflowWiringFailures: readonly string[]; + /** Shared methods whose bodies differ between iOS and macOS builds. */ + readonly sharedPlatformBranchIds: readonly string[]; + /** Shared method bodies whose platform behavior could not be classified. */ + readonly sharedPlatformBranchFailures: readonly string[]; }; export type WorkflowSource = { readonly workflow: string; readonly text: string | null }; @@ -194,23 +199,141 @@ function laneReach( entry: Lane, declaredTests: readonly DeclaredTest[], flagged: readonly FlaggedTest[], + sharedPlatformBranchIds: readonly string[], ): Set { - const only = identifiers(flagged, entry.workflow, 'only-testing'); + if (entry.selection === 'ios-specific') + return new Set(iosPrTestIdentifiers(declaredTests, sharedPlatformBranchIds)); const skipped = identifiers(flagged, entry.workflow, 'skip-testing'); return new Set( declaredTests .filter((test) => test.platforms.includes(entry.platform)) .map((test) => test.identifier) - .filter((id) => (entry.selection === 'whole' || only.has(id)) && !skipped.has(id)), + .filter((id) => !skipped.has(id)), ); } +/** Run iOS-only methods and shared methods with platform-dependent bodies. */ +export function iosPrTestIdentifiers( + declaredTests: readonly DeclaredTest[], + sharedPlatformBranchIds: readonly string[], +): string[] { + const branchSensitive = new Set(sharedPlatformBranchIds); + return declaredTests + .filter( + (test) => + test.platforms.includes('iOS') && + (!test.platforms.includes('macOS') || branchSensitive.has(test.identifier)), + ) + .map((test) => test.identifier) + .filter((id) => !id.endsWith(`/${ENTRY_POINT_METHOD}`)) + .sort(); +} + +type SharedMethodBody = { id: string; body: string; file: string }; +type SharedMethodScan = { bodies: SharedMethodBody[]; failures: string[] }; + +function sharedMethodsByName(shared: readonly DeclaredTest[]): Map { + const byMethod = new Map(); + for (const test of shared) { + const method = test.identifier.split('/').at(-1)!; + byMethod.set(method, [...(byMethod.get(method) ?? []), test.identifier]); + } + return byMethod; +} + +function methodBody(lines: readonly string[], index: number, indent: number): string | undefined { + const start = lines[index]!; + const end = /\{[ \t]*}[ \t]*(?:\/\/.*)?$/.test(start) + ? index + : lines.findIndex( + (candidate, candidateIndex) => + candidateIndex > index && + (candidate.match(/^[ \t]*/)?.[0].length ?? 0) === indent && + /^[ \t]*}[ \t]*(?:\/\/.*)?$/.test(candidate), + ); + return end < index ? undefined : lines.slice(index, end + 1).join('\n'); +} + +function inspectSharedMethods( + source: SwiftSource, + byMethod: ReadonlyMap, +): SharedMethodScan { + const lines = source.text.split('\n'); + const bodies: SharedMethodBody[] = []; + const failures: string[] = []; + for (const [index, line] of lines.entries()) { + const declaration = /^([ \t]*)(?:[\w@]+[ \t]+)*func[ \t]+(test\w*)[ \t]*\(/.exec(line); + if (!declaration) continue; + const matching = byMethod.get(declaration[2]!); + if (!matching) continue; + if (matching.length !== 1) { + failures.push( + `${source.file}:${index + 1}: ambiguous shared XCTest method ${declaration[2]}.`, + ); + continue; + } + const body = methodBody(lines, index, declaration[1]!.length); + if (body === undefined) { + failures.push( + `${source.file}:${index + 1}: cannot inspect shared XCTest method ${matching[0]}.`, + ); + continue; + } + bodies.push({ id: matching[0]!, body, file: source.file }); + } + return { bodies, failures }; +} + +function scanSharedPlatformBranches( + sources: readonly SwiftSource[], + declaredTests: readonly DeclaredTest[], +): { ids: string[]; failures: string[] } { + const shared = declaredTests.filter( + (test) => test.platforms.includes('iOS') && test.platforms.includes('macOS'), + ); + const byMethod = sharedMethodsByName(shared); + const scans = sources.map((source) => inspectSharedMethods(source, byMethod)); + const bodies = scans.flatMap((scan) => scan.bodies); + const failures = scans.flatMap((scan) => scan.failures); + for (const test of shared) { + if (bodies.filter((body) => body.id === test.identifier).length !== 1) { + failures.push(`${test.identifier}: expected one inspectable shared XCTest method body.`); + } + } + const ids = bodies + .filter( + ({ body, file }) => activeSource(body, 'iOS', file) !== activeSource(body, 'macOS', file), + ) + .map(({ id }) => id); + return { ids: [...new Set(ids)].sort(), failures }; +} + +function prWorkflowWiringFailures(text: string | null): string[] { + if (text === null) return []; + const active = text + .split('\n') + .filter((line) => !YAML_COMMENT.test(line)) + .join('\n'); + const failures: string[] = []; + if (!active.includes('scripts/check-xctest-selection.ts --ios-pr-tests')) { + failures.push('PR workflow does not invoke the generated iOS-specific XCTest selector.'); + } + if (!active.includes('-only-testing:$test_id') || !active.includes('"${IOS_PR_TEST_ARGS[@]}"')) { + failures.push('PR workflow does not pass each generated XCTest identifier to xcodebuild.'); + } + if (parseFlaggedTests(PR_WORKFLOW_FILE, active).length > 0) { + failures.push('PR workflow still contains hand-maintained XCTest selection identifiers.'); + } + return failures; +} + export function buildReport( target: string, sources: readonly SwiftSource[], workflows: readonly WorkflowSource[], ): SelectionReport { const declaredTests = parseDeclaredTestsByPlatform(target, sources); + const sharedBranches = scanSharedPlatformBranches(sources, declaredTests); const declared = declaredTests.map((test) => test.identifier); const known = new Map(declaredTests.map((test) => [test.identifier, test])); const flagged = workflows.flatMap((entry) => @@ -221,10 +344,11 @@ export function buildReport( // and cannot speak for anything else a workflow might select. const owned = flagged.filter((entry) => entry.identifier.startsWith(`${target}/`)); const reach = Object.fromEntries( - LANES.map((entry) => [entry.id, laneReach(entry, declaredTests, flagged)]), + LANES.map((entry) => [entry.id, laneReach(entry, declaredTests, flagged, sharedBranches.ids)]), ) as Record>; const entryPoint = `${target}/${ENTRY_POINT_METHOD}`; const reachedAnywhere = new Set(LANES.flatMap((entry) => [...reach[entry.id]])); + const prWorkflow = workflows.find((entry) => entry.workflow === PR_WORKFLOW_FILE); return { target, declared, @@ -242,6 +366,9 @@ export function buildReport( entryPointReachedBy: LANES.filter((entry) => reach[entry.id].has(entryPoint)).map( (entry) => entry.id, ), + prWorkflowWiringFailures: prWorkflowWiringFailures(prWorkflow?.text ?? null), + sharedPlatformBranchIds: sharedBranches.ids, + sharedPlatformBranchFailures: sharedBranches.failures, }; } @@ -323,14 +450,16 @@ export function reportFailures(report: SelectionReport): string[] { 'widen the guard.', ); } + failures.push(...report.prWorkflowWiringFailures); + failures.push(...report.sharedPlatformBranchFailures); if (report.dark.length > 0) { failures.push( `${report.dark.length} declared XCTest method(s) are reachable by no lane:`, ...report.dark.map((identifier) => ` - ${identifier}`), 'The host lane runs everything the macOS build compiles, the nightly everything the iOS', - 'build compiles, and the PR list names its methods; a method outside all three — usually', + 'build compiles, and the PR lane selects iOS-specific methods; a method outside all three — usually', 'a guard naming a platform no lane runs — is dark from the day it is written. Widen the', - 'guard, list it, or delete it.', + 'guard or delete it.', ); } if (report.entryPointReachedBy.length > 0) { @@ -339,7 +468,7 @@ export function reportFailures(report: SelectionReport): string[] { `${report.entryPointReachedBy.join(', ')}.`, 'It is not a test: it opens an NWListener and waits 24 hours for a client, so a lane that', 'runs it hangs until timeout-minutes. Whole-bundle lanes must keep their -skip-testing:', - 'entry for it; the PR list must not name it.', + 'entry for it; the PR selector must exclude it.', ); } return failures; @@ -349,7 +478,7 @@ export function formatSummary(report: SelectionReport): string { const { declared, host, pr, nightly, dark } = counts(report); return ( `xctest selection: ${declared} declared ${report.target} methods — host lane ` + - `(${HOST_WORKFLOW_FILE}, macOS, every PR) reaches ${host}, PR list (${PR_WORKFLOW_FILE}, ` + + `(${HOST_WORKFLOW_FILE}, macOS, every PR) reaches ${host}, PR iOS-specific lane (${PR_WORKFLOW_FILE}, ` + `iOS Simulator, selected PRs and main) selects ${pr}, nightly (${NIGHTLY_WORKFLOW_FILE}, iOS Simulator) ` + `reaches ${nightly}; ${dark} reachable by no lane; ${ENTRY_POINT_METHOD} skipped everywhere.\n` ); @@ -369,6 +498,17 @@ export function runnerPackageSourceFailures(root: string = repoRoot): string[] { function main(): number { const report = loadReport(); + if (process.argv.slice(2).includes('--ios-pr-tests')) { + const failures = reportFailures(report); + if (report.reach.pr.size === 0) + failures.push('The PR lane selected no iOS-specific XCTest methods.'); + if (failures.length > 0) { + process.stderr.write(`${failures.join('\n')}\n`); + return 1; + } + process.stdout.write(`${[...report.reach.pr].sort().join('\n')}\n`); + return 0; + } const failures = [...reportFailures(report), ...runnerPackageSourceFailures()]; process.stdout.write(formatSummary(report)); if (failures.length === 0) return 0; diff --git a/scripts/ios-xctest-impact.ts b/scripts/ios-xctest-impact.ts deleted file mode 100644 index 65d55ea3e6..0000000000 --- a/scripts/ios-xctest-impact.ts +++ /dev/null @@ -1,76 +0,0 @@ -import fs from 'node:fs'; -import { spawnSync } from 'node:child_process'; - -const RUNNER_INPUTS = [ - 'apple/runner/', - 'apple/snapshot-presentation/', - 'contracts/fixtures/', - 'packages/platform-apple/src/runner/', - '.github/actions/setup-apple-runner-build/', - '.github/workflows/ios.yml', - 'package.json', - 'pnpm-lock.yaml', - 'scripts/build-xcuitest-apple.sh', - 'scripts/patch-xcuitest-runner-icon.ts', - 'scripts/swift-toolchain-tmpdir.ts', - 'scripts/write-xcuitest-cache-metadata.mjs', - 'scripts/check-xctest-selection.ts', - 'scripts/xctest-declarations.ts', - 'scripts/swift-conditional-compilation.ts', - 'scripts/ios-xctest-impact.ts', -] as const; - -export function affectsIosXctests(file: string): boolean { - return RUNNER_INPUTS.some((input) => - input.endsWith('/') ? file.startsWith(input) : file === input, - ); -} - -export function selectIosXctests( - eventName: string, - changedPaths: readonly string[] | null, -): { run: boolean; reason: string } { - if (eventName !== 'pull_request') - return { run: true, reason: 'main and manual runs exercise XCTest' }; - if (changedPaths === null || changedPaths.length === 0) { - return { run: true, reason: 'the PR change set could not be established' }; - } - const input = changedPaths.find(affectsIosXctests); - return input - ? { run: true, reason: `XCTest input changed: ${input}` } - : { run: false, reason: 'the PR changed no XCTest runner, test, or golden-table input' }; -} - -export function uncoveredRunnerCacheInputs(action: string): string[] { - const hashFiles = action.match(/hashFiles\(([^\n]+)\)/)?.[1]; - if (!hashFiles) return ['runner cache hashFiles declaration']; - return [...hashFiles.matchAll(/'([^']+)'/g)] - .map((match) => match[1] as string) - .filter((input) => !affectsIosXctests(input.replace(/\*\*?$/, 'probe.swift'))); -} - -function changedPathsFromGit(baseSha: string): string[] | null { - if (!/^[a-f0-9]{40}$/.test(baseSha)) return null; - const result = spawnSync('git', ['diff', '--name-only', '-z', baseSha, 'HEAD'], { - encoding: 'utf8', - }); - if (result.error || result.status !== 0 || result.stdout === null) return null; - return result.stdout.split('\0').filter(Boolean); -} - -if (process.argv[1]?.endsWith('/ios-xctest-impact.ts')) { - const eventName = process.env.GITHUB_EVENT_NAME ?? ''; - const changedPaths = - eventName === 'pull_request' ? changedPathsFromGit(process.env.BASE_SHA ?? '') : []; - const selection = selectIosXctests(eventName, changedPaths); - process.stdout.write(`iOS XCTest: ${selection.run ? 'run' : 'skip'}; ${selection.reason}\n`); - if (process.env.GITHUB_OUTPUT) { - fs.appendFileSync(process.env.GITHUB_OUTPUT, `run=${selection.run}\n`); - } - if (process.env.GITHUB_STEP_SUMMARY) { - fs.appendFileSync( - process.env.GITHUB_STEP_SUMMARY, - `### iOS XCTest selection\n\n${selection.run ? 'Run' : 'Skip'}: ${selection.reason}.\n`, - ); - } -} diff --git a/scripts/xctest-declarations.ts b/scripts/xctest-declarations.ts index 59ef490648..d7e7fbc4d8 100644 --- a/scripts/xctest-declarations.ts +++ b/scripts/xctest-declarations.ts @@ -18,15 +18,13 @@ export const RUNNER_TESTS_DIR = 'apple/runner/AgentDeviceRunner/AgentDeviceRunne // file's name says nothing about whether it declares addressable tests. const SWIFT_SOURCE = /\.swift$/; -// One ordered pass over the source. A column-0 type declaration moves the enclosing type; -// a `func test…` indented exactly one level binds to it. Position carries the meaning -// rather than brace counting, which would have to know which `{` sits inside a string -// literal. It is also the more precise rule: only a method declared directly in a -// top-level `class`/`extension` block is addressable as `Target/Class/method`, so a -// helper type nested inside a test body (`final class ResultBox` — several of these -// exist) contributes no test identifiers, and neither does a closure-local `func test…`. -const DECLARATION = - /^(?:[\w@]+[ \t]+)*(?:class|extension|struct|enum|actor|protocol)[ \t]+([A-Za-z_]\w*)|^ {2}(?:[\w@]+[ \t]+)*func[ \t]+(test\w*)[ \t]*\(/gm; +// Swift ignores indentation, but the runner tests use it consistently to distinguish +// methods from functions local to a method. The type and method may each be indented by +// a surrounding #if block; requiring column zero or exactly two spaces misses valid tests. +const TYPE_DECLARATION = + /^([ \t]*)(?:[\w@]+[ \t]+)*(?:class|extension|struct|enum|actor|protocol)[ \t]+([A-Za-z_]\w*)\b/; +const FUNCTION_DECLARATION = /^([ \t]*)(?:[\w@]+[ \t]+)*func[ \t]+([A-Za-z_]\w*)/; +const TEST_DECLARATION = /^([ \t]*)(?:[\w@]+[ \t]+)*func[ \t]+(test\w*)[ \t]*\(/; export type SwiftSource = { readonly file: string; readonly text: string }; @@ -59,15 +57,91 @@ function collectSwiftSourcePaths(directory: string, relativeDirectory: string, f /** Every `Target/Class/method` identifier the sources declare, sorted, guards ignored. */ export function parseDeclaredTests(target: string, sources: readonly SwiftSource[]): string[] { - const declared = new Set(); - for (const source of sources) { - let enclosing = ''; - for (const [, type, method] of source.text.matchAll(DECLARATION)) { - if (type !== undefined) enclosing = type; - else if (method !== undefined && enclosing) declared.add(`${target}/${enclosing}/${method}`); - } + return [...new Set(sources.flatMap((source) => declarationsInSource(target, source)))].sort(); +} + +type DeclarationScope = { + enclosing?: { name: string; indent: number }; + nestedTypeIndent?: number; + functionIndent?: number; +}; + +function declarationsInSource(target: string, source: SwiftSource): string[] { + const scope: DeclarationScope = {}; + const declared: string[] = []; + for (const [index, line] of source.text.split('\n').entries()) { + if (/^[ \t]*(?:\/\/|\/\*|\*)/.test(line)) continue; + const indent = line.match(/^[ \t]*/)?.[0].length ?? 0; + const error = () => unrecognizedDeclaration(source.file, index + 1, line); + if (/\bfunc[ \t]+test\w*/.test(line) && !FUNCTION_DECLARATION.test(line)) throw error(); + closeScopes(scope, line, indent); + if (readTypeDeclaration(scope, line, indent)) continue; + const method = readTestDeclaration(scope, line, indent, error); + if (method) declared.push(`${target}/${method}`); } - return [...declared].sort(); + return declared; +} + +function unrecognizedDeclaration(file: string, lineNumber: number, line: string): Error { + return new Error(`${file}:${lineNumber}: unrecognized XCTest declaration: ${line.trim()}`); +} + +function closeScopes(scope: DeclarationScope, line: string, indent: number): void { + if (!/^[ \t]*}/.test(line)) return; + if (scope.functionIndent !== undefined && indent <= scope.functionIndent) + scope.functionIndent = undefined; + if (scope.nestedTypeIndent !== undefined && indent <= scope.nestedTypeIndent) + scope.nestedTypeIndent = undefined; + if (scope.enclosing && indent <= scope.enclosing.indent) scope.enclosing = undefined; +} + +function readTypeDeclaration(scope: DeclarationScope, line: string, indent: number): boolean { + const type = TYPE_DECLARATION.exec(line); + if (!type) return false; + if (!scope.enclosing) scope.enclosing = { name: type[2]!, indent }; + else if (indent > scope.enclosing.indent && scope.functionIndent === undefined) + scope.nestedTypeIndent = indent; + return true; +} + +function readTestDeclaration( + scope: DeclarationScope, + line: string, + indent: number, + error: () => Error, +): string | undefined { + const func = FUNCTION_DECLARATION.exec(line); + if (!func) return undefined; + const local = isLocalFunction(scope, indent); + markFunctionScope(scope, indent, local); + const name = func[2]!; + if (local || !name.startsWith('test')) return undefined; + return checkedTestName(scope, line, indent, name, error); +} + +function isLocalFunction(scope: DeclarationScope, indent: number): boolean { + return ( + (scope.functionIndent !== undefined && indent > scope.functionIndent) || + (scope.nestedTypeIndent !== undefined && indent > scope.nestedTypeIndent) + ); +} + +function markFunctionScope(scope: DeclarationScope, indent: number, local: boolean): void { + if (!scope.enclosing || indent <= scope.enclosing.indent) return; + if (scope.functionIndent !== undefined || local) return; + scope.functionIndent = indent; +} + +function checkedTestName( + scope: DeclarationScope, + line: string, + indent: number, + name: string, + error: () => Error, +): string { + const enclosing = scope.enclosing; + if (!enclosing || indent <= enclosing.indent || !TEST_DECLARATION.test(line)) throw error(); + return `${enclosing.name}/${name}`; } /** The declared identifiers, each with the platforms whose unit-test build compiles it. */ diff --git a/vitest.config.ts b/vitest.config.ts index 202217d18f..bcddac82e9 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -125,10 +125,11 @@ export default defineConfig({ 'scripts/__tests__/agent-setup-startup-contract.test.ts', 'scripts/__tests__/npm-skills-exclusion.test.ts', 'scripts/__tests__/simulator-skills-contract.test.ts', - // Parses ios.yml and the runner's Swift sources: no Xcode, no simulator, and - // the check it guards is what keeps the PR lane's `-only-testing:` list honest. + // Parse Swift guards and declarations before deriving the simulator selection. + 'scripts/__tests__/swift-conditional-compilation.test.ts', + 'scripts/__tests__/xctest-declarations.test.ts', 'scripts/__tests__/xctest-selection.test.ts', - 'scripts/__tests__/ios-xctest-impact.test.ts', + 'scripts/__tests__/apple-ci-impact.test.ts', // The nightly XCTest lane's reporter/liveness check, which otherwise only ever // executes on a macOS runner at 04:30. 'scripts/__tests__/xctest-run-summary.test.ts', From 0baea4109cc79075ef2a7211d82bd3c4e8636bc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 24 Sep 2026 17:29:40 +0200 Subject: [PATCH 6/6] chore(gates): resolve shallow PR impact and iOS screenshot coverage --- .../RunnerTests+LifecycleTests.swift | 5 +++ scripts/__tests__/apple-ci-impact.test.ts | 42 +++++++++++++++++++ scripts/apple-ci-impact.ts | 2 +- scripts/check-xctest-selection.ts | 15 ++----- 4 files changed, 51 insertions(+), 13 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LifecycleTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LifecycleTests.swift index a4e75fe198..352cdbce1c 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LifecycleTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LifecycleTests.swift @@ -97,6 +97,11 @@ extension RunnerTests { // into the runner's temporary directory. let fileName = URL(fileURLWithPath: message).lastPathComponent XCTAssertTrue(message.hasSuffix(fileName), message) +#if os(iOS) + XCTAssertTrue(message.hasPrefix("tmp/"), message) +#elseif os(macOS) + XCTAssertTrue(message.hasPrefix("/"), message) +#endif let storedPath = (NSTemporaryDirectory() as NSString).appendingPathComponent(fileName) XCTAssertEqual(try Data(contentsOf: URL(fileURLWithPath: storedPath)), pngData) XCTAssertEqual(response.data?.screenshotMetadata?.pixelsPerPoint, 3) diff --git a/scripts/__tests__/apple-ci-impact.test.ts b/scripts/__tests__/apple-ci-impact.test.ts index 1effb8b52a..ea0834195d 100644 --- a/scripts/__tests__/apple-ci-impact.test.ts +++ b/scripts/__tests__/apple-ci-impact.test.ts @@ -1,4 +1,6 @@ import fs from 'node:fs'; +import { execFileSync } from 'node:child_process'; +import os from 'node:os'; import path from 'node:path'; import { expect, test } from 'vitest'; import { parse } from 'yaml'; @@ -112,3 +114,43 @@ test('bridge proof runs for its owning sources and uncertain tooling changes', ( expect(selectAppleBridgeProof(null).run).toBe(true); expect(selectAppleBridgeProof([]).run).toBe(true); }); + +test('a shallow PR merge still yields a known change set for both selectors', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'apple-ci-impact-')); + const source = path.join(root, 'source'); + const shallow = path.join(root, 'shallow'); + fs.mkdirSync(source); + const git = (cwd: string, ...args: string[]) => + execFileSync('git', args, { cwd, encoding: 'utf8' }).trim(); + try { + git(source, 'init', '-q', '-b', 'main'); + git(source, 'config', 'user.email', 'test@example.com'); + git(source, 'config', 'user.name', 'Test'); + git(source, 'commit', '--allow-empty', '-qm', 'base'); + const base = git(source, 'rev-parse', 'HEAD'); + git(source, 'checkout', '-qb', 'feature'); + fs.mkdirSync(path.join(source, 'src')); + fs.writeFileSync(path.join(source, 'src', 'feature.ts'), 'export const feature = true;\n'); + git(source, 'add', '.'); + git(source, 'commit', '-qm', 'feature'); + git(source, 'checkout', '-q', 'main'); + git(source, 'merge', '-q', '--no-ff', '-m', 'merge', 'feature'); + git(root, 'clone', '-q', '--depth=1', '--branch', 'main', `file://${source}`, shallow); + git(shallow, 'fetch', '-q', 'origin', base, '--depth=1'); + expect(git(shallow, 'rev-parse', '--is-shallow-repository')).toBe('true'); + const select = (target: string) => + execFileSync( + process.execPath, + ['--experimental-strip-types', path.join(repoRoot, 'scripts/apple-ci-impact.ts'), target], + { + cwd: shallow, + encoding: 'utf8', + env: { ...process.env, BASE_SHA: base, GITHUB_EVENT_NAME: 'pull_request' }, + }, + ); + expect(select('xctest')).toContain('skip;'); + expect(select('bridge')).toContain('skip;'); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/scripts/apple-ci-impact.ts b/scripts/apple-ci-impact.ts index 4e79bfe3d6..e696acdd83 100644 --- a/scripts/apple-ci-impact.ts +++ b/scripts/apple-ci-impact.ts @@ -53,7 +53,7 @@ export function selectAppleBridgeProof(changedPaths: readonly string[] | null): function changedPathsFromGit(baseSha: string): string[] | null { if (!/^[a-f0-9]{40}$/.test(baseSha)) return null; - const result = spawnSync('git', ['diff', '--name-only', '-z', `${baseSha}...HEAD`], { + const result = spawnSync('git', ['diff', '--name-only', '-z', baseSha, 'HEAD'], { encoding: 'utf8', }); if (result.error || result.status !== 0 || result.stdout === null) return null; diff --git a/scripts/check-xctest-selection.ts b/scripts/check-xctest-selection.ts index b6c60b53c2..f8c8a2b523 100644 --- a/scripts/check-xctest-selection.ts +++ b/scripts/check-xctest-selection.ts @@ -8,9 +8,7 @@ // - host macos.yml, macOS host, every PR: the whole bundle as compiled for macOS, minus // `-skip-testing:` — the pure runner-decision tests, whose guard is // `#if AGENT_DEVICE_RUNNER_UNIT_TESTS` alone. -// - pr ios.yml, iOS Simulator, when runner inputs change on a PR (and on every main -// push): methods compiled only for iOS, plus shared methods with platform- -// dependent bodies, derived from Swift guards. +// - pr ios.yml, iOS Simulator: tests with iOS-specific Swift bodies, when selected. // - nightly xctest-nightly.yml, iOS Simulator, scheduled: the whole bundle as compiled for // iOS, minus `-skip-testing:` — includes the simulator-only tests, whose guard is // `… && os(iOS)` (they launch the host app, route through SpringBoard, or assert an @@ -20,10 +18,8 @@ // platform rather than treating a source-level `func test…` as running everywhere. What it // holds: // -// 1. Every `-skip-testing:` identifier names a declared method that compiles for its lane, -// and the PR workflow consumes the generated list. `xcodebuild` treats a skip identifier -// matching nothing as an empty set, which re-admits `RunnerTests/testCommand` — not a test -// but the runner's server entry point, which opens an NWListener and waits 24 hours. +// 1. Every `-skip-testing:` identifier compiles for its lane. An unknown skip re-admits +// `RunnerTests/testCommand`, the server entry point that waits 24 hours. // 2. Every declared method is reachable by at least one lane. A test gated to a platform // no lane runs (the tvOS-only pair this check found) is dark from the day it is written. // 3. The entry point is reachable by no lane at all. @@ -51,7 +47,6 @@ const packageAppleRunnerScript = path.join(repoRoot, 'scripts/package-apple-runn /** The macOS host lane, which runs the whole macOS-compiled bundle on every PR. */ export const HOST_WORKFLOW_FILE = '.github/workflows/macos.yml'; -/** The PR lane, whose generated iOS-specific list runs on selected pull requests. */ export const PR_WORKFLOW_FILE = '.github/workflows/ios.yml'; /** The nightly lane, whose `-skip-testing:` list decides what the full simulator suite leaves out. */ @@ -158,11 +153,8 @@ export type SelectionReport = { readonly dark: readonly string[]; /** Lanes that reach the entry point — a failure (a 24-hour hang). */ readonly entryPointReachedBy: readonly LaneId[]; - /** Whether the PR workflow runs and consumes the generated iOS-specific list. */ readonly prWorkflowWiringFailures: readonly string[]; - /** Shared methods whose bodies differ between iOS and macOS builds. */ readonly sharedPlatformBranchIds: readonly string[]; - /** Shared method bodies whose platform behavior could not be classified. */ readonly sharedPlatformBranchFailures: readonly string[]; }; @@ -212,7 +204,6 @@ function laneReach( ); } -/** Run iOS-only methods and shared methods with platform-dependent bodies. */ export function iosPrTestIdentifiers( declaredTests: readonly DeclaredTest[], sharedPlatformBranchIds: readonly string[],