From 656e75d2c9eb2b5d9efd99d27a6dac007e672712 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 24 Sep 2026 14:55:58 +0200 Subject: [PATCH 1/2] test(ios): wait out the launch an accepted deep-link confirmation releases MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The smoke automation scenario accepted SpringBoard's "Open in …?" prompt and then gave the route one 10 s wait. On a loaded host the released launch reached the foreground 20.7 s after the Open tap (CI run 35991523779; 27.4 s locally), so every read answered the retriable APP_NOT_RUNNING and the step failed although the app later rendered the deep-linked route. The deep-link confirmation helper moves to its own module and keys every step on the typed details.runnerErrorCode: only a destination wait that ends on APP_NOT_RUNNING probes for the confirmation or earns another wait, up to four 15 s waits. Any other miss returns to the caller's own destination assertion, and the alert probe never queries a rendered screen. Refs #2491 --- ...mulator-e2e-deep-link-confirmation.test.ts | 124 ++++++++++++++++++ .../live-automation-scenario.ts | 44 +------ .../live-deep-link-confirmation.ts | 76 +++++++++++ .../live-snapshot-depth-frontier.ts | 2 +- .../live-webview-remote-content.ts | 2 +- 5 files changed, 203 insertions(+), 45 deletions(-) create mode 100644 test/integration/ios-simulator-e2e-deep-link-confirmation.test.ts create mode 100644 test/integration/ios-simulator-e2e/live-deep-link-confirmation.ts diff --git a/test/integration/ios-simulator-e2e-deep-link-confirmation.test.ts b/test/integration/ios-simulator-e2e-deep-link-confirmation.test.ts new file mode 100644 index 0000000000..43fd98583e --- /dev/null +++ b/test/integration/ios-simulator-e2e-deep-link-confirmation.test.ts @@ -0,0 +1,124 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import type { CliJsonResult } from './cli-json.ts'; +import { + answerDeepLinkConfirmation, + type DeepLinkConfirmationDevice, +} from './ios-simulator-e2e/live-deep-link-confirmation.ts'; + +function result(status: number, json?: unknown): CliJsonResult { + return { json, status, stderr: '', stdout: '' }; +} + +const LANDED = result(0, { success: true }); +const LAUNCH_PENDING = result(1, { + error: { + code: 'COMMAND_FAILED', + details: { reason: 'wait_capture_stalled', runnerErrorCode: 'APP_NOT_RUNNING' }, + }, +}); +const WRONG_ROUTE = result(1, { + error: { code: 'COMMAND_FAILED', details: { reason: 'wait_target_absent' } }, +}); +const OPEN_PROMPT = result(0, { + data: { message: 'Open in “Agent Device Tester”?', items: ['Cancel', 'Open'] }, +}); +const NO_ALERT = result(1, { error: { code: 'COMMAND_FAILED' } }); + +/** A simulator whose destination waits and alert probes answer in the order given. */ +function simulator(destinationWaits: CliJsonResult[], alerts: CliJsonResult[] = [OPEN_PROMPT]) { + const log: string[] = []; + const device: DeepLinkConfirmationDevice = { + waitForDestination: async (step) => { + log.push(step); + const next = destinationWaits.shift(); + assert.ok(next, `unexpected destination wait: ${step}`); + return next; + }, + inspectAlert: async () => { + log.push('alert get'); + const next = alerts.shift(); + assert.ok(next, 'unexpected alert probe'); + return next; + }, + acceptAlert: async () => { + log.push('alert accept'); + }, + }; + return { device, log }; +} + +const waits = (log: string[]) => log.filter((step) => step.startsWith('wait for')).length; + +test('a destination that arrives never probes for the confirmation', async () => { + const { device, log } = simulator([LANDED]); + + await answerDeepLinkConfirmation(device); + + assert.deepEqual(log, ['wait for the deep-link destination (1/4)']); +}); + +test('the launch an accepted confirmation releases is waited for until it lands', async () => { + // CI run 35991523779: the app reached the foreground 20.7 s after `alert accept` tapped Open. + const { device, log } = simulator([LAUNCH_PENDING, LAUNCH_PENDING, LAUNCH_PENDING, LANDED]); + + await answerDeepLinkConfirmation(device); + + assert.deepEqual(log, [ + 'wait for the deep-link destination (1/4)', + 'alert get', + 'alert accept', + 'wait for the deep-link destination (2/4)', + 'wait for the deep-link destination (3/4)', + 'wait for the deep-link destination (4/4)', + ]); +}); + +test('a miss that is not a pending launch neither probes nor waits again', async () => { + const { device, log } = simulator([WRONG_ROUTE]); + + await answerDeepLinkConfirmation(device); + + assert.deepEqual(log, ['wait for the deep-link destination (1/4)']); +}); + +test('after the accept, a miss that is not a pending launch earns no further wait', async () => { + const { device, log } = simulator([LAUNCH_PENDING, WRONG_ROUTE]); + + await answerDeepLinkConfirmation(device); + + assert.equal(waits(log), 2); +}); + +test('a confirmation that appears late is still answered once', async () => { + const { device, log } = simulator( + [LAUNCH_PENDING, LAUNCH_PENDING, LAUNCH_PENDING, LANDED], + [NO_ALERT, OPEN_PROMPT], + ); + + await answerDeepLinkConfirmation(device); + + assert.deepEqual( + log.filter((step) => step.startsWith('alert')), + ['alert get', 'alert get', 'alert accept'], + ); +}); + +test('the wait budget is bounded when the app never starts', async () => { + const { device, log } = simulator(Array.from({ length: 5 }, () => LAUNCH_PENDING)); + + await answerDeepLinkConfirmation(device); + + assert.equal(waits(log), 4); +}); + +test('a prompt that is not the deep-link confirmation is never accepted', async () => { + const { device, log } = simulator( + [LAUNCH_PENDING], + [result(0, { data: { message: 'Allow notifications?', items: ['Allow'] } })], + ); + + await assert.rejects(answerDeepLinkConfirmation(device)); + assert.equal(log.includes('alert accept'), false); +}); diff --git a/test/integration/ios-simulator-e2e/live-automation-scenario.ts b/test/integration/ios-simulator-e2e/live-automation-scenario.ts index dfe77d2eac..6b3168a0f3 100644 --- a/test/integration/ios-simulator-e2e/live-automation-scenario.ts +++ b/test/integration/ios-simulator-e2e/live-automation-scenario.ts @@ -11,19 +11,13 @@ import { assertJsonContains, assertWaitText, } from './live-assertions.ts'; +import { acceptDeepLinkConfirmationIfPresent } from './live-deep-link-confirmation.ts'; import { clearStateLaunchUrlMaestroFlow } from './live-fixtures.ts'; import { type LiveContext, runStep, verifyBehavior, verifyCommand } from './live-harness.ts'; const C = PUBLIC_COMMANDS; const ALERT_WAIT_TIMEOUT = String(DEFAULT_ALERT_TIMEOUT_MS); const FIXTURE_HOME_TITLE = 'Agent Device Tester'; -/** - * Deliberately generous. This budget decides whether the alert probe below runs at all, so it must - * outlast the slowest honest route mount on a cold CI simulator — the WebView lab took over 2.5 s - * there while rendering correctly. Waiting longer costs nothing when a confirmation really is up, - * because the route never renders until it is accepted; being too short costs the whole scenario. - */ -const DEEP_LINK_DESTINATION_WAIT_MS = '15000'; /** The Automation lab's own first landmark; see `acceptDeepLinkConfirmationIfPresent`. */ const AUTOMATION_LAB_LANDMARK = ['text', 'Automation lab'] as const; const AUTOMATION_DEEP_LINK = @@ -253,42 +247,6 @@ async function assertClearStateLaunchUrl(context: LiveContext): Promise { await assertElementText(context, 'id="automation-event-payload"', '{"source":"deep-link"}'); } -/** - * iOS sometimes puts an "Open in ?" confirmation in front of a custom-scheme deep link, so a - * scenario that launched one must accept it before asserting anything. `destination` is the `wait` - * predicate for the route's own first landmark, and it decides whether the alert probe runs at all: - * a landmark that arrived proves no confirmation is in the way. Each caller passes its own, because - * a shared landmark never matches off its route and sends every caller into the probe — and - * `alert get` against a live WKWebView screen is the XCTest query that exceeds the runner's - * execution watchdog, leaving every later command refused as `RUNNER_BUSY` (#2484 follow-up). The - * landmark must therefore be a native node the route renders before its content, and the budget - * above must outlast a cold mount, so the probe is reached only when something really is blocking. - */ -export async function acceptDeepLinkConfirmationIfPresent( - context: LiveContext, - destination: readonly string[], -): Promise { - const arrived = await runStep( - context, - 'wait for deep-link destination before inspecting system UI', - ['wait', ...destination, DEEP_LINK_DESTINATION_WAIT_MS], - { allowFailure: true }, - ); - if (arrived.status === 0) return; - - const alert = await runStep(context, 'inspect delayed deep-link system alert', ['alert', 'get'], { - allowFailure: true, - }); - if (alert.status !== 0) return; - const alertInfo = alert.json?.data; - assert.match(String(alertInfo?.message), /^Open in\b/, JSON.stringify(alert.json)); - assert.ok( - Array.isArray(alertInfo?.items) && alertInfo.items.includes('Open'), - JSON.stringify(alert.json), - ); - await runStep(context, 'accept deep-link confirmation', ['alert', 'accept']); -} - async function openAutomationDeepLink(context: LiveContext, step: string): Promise { await runStep(context, step, [ 'open', diff --git a/test/integration/ios-simulator-e2e/live-deep-link-confirmation.ts b/test/integration/ios-simulator-e2e/live-deep-link-confirmation.ts new file mode 100644 index 0000000000..7e169a1321 --- /dev/null +++ b/test/integration/ios-simulator-e2e/live-deep-link-confirmation.ts @@ -0,0 +1,76 @@ +import assert from 'node:assert/strict'; + +import type { CliJsonResult } from '../cli-json.ts'; +import { type LiveContext, runStep } from './live-harness.ts'; + +/** One destination wait; generous because the WebView lab took over 2.5 s to mount on cold CI. */ +const DEEP_LINK_DESTINATION_WAIT_MS = '15000'; +/** + * One wait finds the confirmation holding the launch; three more cover the launch its answer + * releases. That launch usually reaches the foreground within 3 s of the tap, but a loaded host has + * held it for 20.7 s (CI run 35991523779) and 27.4 s (a local run), and the route renders after that. + */ +const DESTINATION_WAITS = 4; +/** `details.runnerErrorCode` of a read the runner refused because the session app is not running. */ +const APP_NOT_RUNNING = 'APP_NOT_RUNNING'; + +export type DeepLinkConfirmationDevice = { + waitForDestination: (step: string) => Promise; + inspectAlert: () => Promise; + acceptAlert: () => Promise; +}; + +/** + * iOS can hold a custom-scheme deep link behind an "Open in ?" confirmation. `destination` is + * the `wait` predicate for the route's own first landmark, a native node the route renders before + * its content. + */ +export function acceptDeepLinkConfirmationIfPresent( + context: LiveContext, + destination: readonly string[], +): Promise { + return answerDeepLinkConfirmation({ + waitForDestination: (step) => + runStep(context, step, ['wait', ...destination, DEEP_LINK_DESTINATION_WAIT_MS], { + allowFailure: true, + }), + inspectAlert: () => + runStep(context, 'inspect delayed deep-link system alert', ['alert', 'get'], { + allowFailure: true, + }), + acceptAlert: () => runStep(context, 'accept deep-link confirmation', ['alert', 'accept']), + }); +} + +/** + * Only a destination wait that ends on `APP_NOT_RUNNING` — the launch is still pending — leads to + * the alert probe or to another wait. Any other outcome returns, so an arrived route skips the probe + * and a wrong route or a stalled capture fails on the caller's own destination assertion. The probe + * therefore never queries a rendered screen, where `alert get` against a live WKWebView exceeds the + * runner's execution watchdog and leaves later commands refused as `RUNNER_BUSY` (#2484 follow-up). + */ +export async function answerDeepLinkConfirmation( + device: DeepLinkConfirmationDevice, +): Promise { + let answered = false; + for (let wait = 1; wait <= DESTINATION_WAITS; wait += 1) { + const arrived = await device.waitForDestination( + `wait for the deep-link destination (${wait}/${DESTINATION_WAITS})`, + ); + if (arrived.json?.error?.details?.runnerErrorCode !== APP_NOT_RUNNING) return; + if (!answered) answered = await acceptOpenConfirmation(device); + } +} + +async function acceptOpenConfirmation(device: DeepLinkConfirmationDevice): Promise { + const alert = await device.inspectAlert(); + if (alert.status !== 0) return false; + const alertInfo = alert.json?.data; + assert.match(String(alertInfo?.message), /^Open in\b/, JSON.stringify(alert.json)); + assert.ok( + Array.isArray(alertInfo?.items) && alertInfo.items.includes('Open'), + JSON.stringify(alert.json), + ); + await device.acceptAlert(); + return true; +} diff --git a/test/integration/ios-simulator-e2e/live-snapshot-depth-frontier.ts b/test/integration/ios-simulator-e2e/live-snapshot-depth-frontier.ts index 84997a70d9..ce21123612 100644 --- a/test/integration/ios-simulator-e2e/live-snapshot-depth-frontier.ts +++ b/test/integration/ios-simulator-e2e/live-snapshot-depth-frontier.ts @@ -6,7 +6,7 @@ import { type LiveSnapshotNode as SnapshotNode, snapshotNodes, } from './live-assertions.ts'; -import { acceptDeepLinkConfirmationIfPresent } from './live-automation-scenario.ts'; +import { acceptDeepLinkConfirmationIfPresent } from './live-deep-link-confirmation.ts'; import { type LiveContext, runStep, verifyBehavior } from './live-harness.ts'; const VISIBLE_DEPTH_DEEP_LINK = 'agent-device-test-app:///snapshot-depth'; diff --git a/test/integration/ios-simulator-e2e/live-webview-remote-content.ts b/test/integration/ios-simulator-e2e/live-webview-remote-content.ts index 711f4fab6f..484ab49c0c 100644 --- a/test/integration/ios-simulator-e2e/live-webview-remote-content.ts +++ b/test/integration/ios-simulator-e2e/live-webview-remote-content.ts @@ -1,7 +1,7 @@ import assert from 'node:assert/strict'; import { assertWaitText, snapshotNodes } from './live-assertions.ts'; -import { acceptDeepLinkConfirmationIfPresent } from './live-automation-scenario.ts'; +import { acceptDeepLinkConfirmationIfPresent } from './live-deep-link-confirmation.ts'; import { type LiveContext, runStep, verifyBehavior } from './live-harness.ts'; const WEBVIEW_LAB_DEEP_LINK = 'agent-device-test-app:///webview'; From 3922d06915cb00ac8e8a4502aaed41fd5fa93411 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 24 Sep 2026 16:03:06 +0200 Subject: [PATCH 2/2] fix(ios-e2e): recover readable deep-link timeouts --- ...mulator-e2e-deep-link-confirmation.test.ts | 70 +++++++++++++++---- .../live-deep-link-confirmation.ts | 26 ++++--- 2 files changed, 74 insertions(+), 22 deletions(-) diff --git a/test/integration/ios-simulator-e2e-deep-link-confirmation.test.ts b/test/integration/ios-simulator-e2e-deep-link-confirmation.test.ts index 43fd98583e..a07a38db8e 100644 --- a/test/integration/ios-simulator-e2e-deep-link-confirmation.test.ts +++ b/test/integration/ios-simulator-e2e-deep-link-confirmation.test.ts @@ -21,6 +21,12 @@ const LAUNCH_PENDING = result(1, { const WRONG_ROUTE = result(1, { error: { code: 'COMMAND_FAILED', details: { reason: 'wait_target_absent' } }, }); +const READABLE_TIMEOUT = result(1, { + error: { + code: 'COMMAND_FAILED', + details: { reason: 'wait_deadline_exceeded', readableCaptures: 5, captureTruncated: true }, + }, +}); const OPEN_PROMPT = result(0, { data: { message: 'Open in “Agent Device Tester”?', items: ['Cancel', 'Open'] }, }); @@ -56,7 +62,47 @@ test('a destination that arrives never probes for the confirmation', async () => await answerDeepLinkConfirmation(device); - assert.deepEqual(log, ['wait for the deep-link destination (1/4)']); + assert.deepEqual(log, ['wait for the deep-link destination (1/5)']); +}); + +test('a readable destination timeout still answers a real Open confirmation', async () => { + const { device, log } = simulator([READABLE_TIMEOUT, LANDED]); + + await answerDeepLinkConfirmation(device); + + assert.deepEqual(log, [ + 'wait for the deep-link destination (1/5)', + 'alert get', + 'alert accept', + 'wait for the deep-link destination (2/5)', + ]); +}); + +test('a readable no-match that answers Open waits for the released launch', async () => { + const { device, log } = simulator([WRONG_ROUTE, LAUNCH_PENDING, LANDED]); + + await answerDeepLinkConfirmation(device); + + assert.deepEqual(log, [ + 'wait for the deep-link destination (1/5)', + 'alert get', + 'alert accept', + 'wait for the deep-link destination (2/5)', + 'wait for the deep-link destination (3/5)', + ]); +}); + +test('a truncated capture retries for four bounded waits without accepting a missing alert', async () => { + const { device, log } = simulator( + [READABLE_TIMEOUT, READABLE_TIMEOUT, READABLE_TIMEOUT, READABLE_TIMEOUT, LANDED], + [NO_ALERT, NO_ALERT, NO_ALERT, NO_ALERT], + ); + + await answerDeepLinkConfirmation(device); + + assert.equal(waits(log), 5); + assert.equal(log.filter((step) => step === 'alert get').length, 4); + assert.equal(log.includes('alert accept'), false); }); test('the launch an accepted confirmation releases is waited for until it lands', async () => { @@ -66,29 +112,29 @@ test('the launch an accepted confirmation releases is waited for until it lands' await answerDeepLinkConfirmation(device); assert.deepEqual(log, [ - 'wait for the deep-link destination (1/4)', + 'wait for the deep-link destination (1/5)', 'alert get', 'alert accept', - 'wait for the deep-link destination (2/4)', - 'wait for the deep-link destination (3/4)', - 'wait for the deep-link destination (4/4)', + 'wait for the deep-link destination (2/5)', + 'wait for the deep-link destination (3/5)', + 'wait for the deep-link destination (4/5)', ]); }); -test('a miss that is not a pending launch neither probes nor waits again', async () => { - const { device, log } = simulator([WRONG_ROUTE]); +test('a readable miss probes once and leaves a wrong route to the caller', async () => { + const { device, log } = simulator([WRONG_ROUTE], [NO_ALERT]); await answerDeepLinkConfirmation(device); - assert.deepEqual(log, ['wait for the deep-link destination (1/4)']); + assert.deepEqual(log, ['wait for the deep-link destination (1/5)', 'alert get']); }); -test('after the accept, a miss that is not a pending launch earns no further wait', async () => { - const { device, log } = simulator([LAUNCH_PENDING, WRONG_ROUTE]); +test('after the accept, a readable no-match still gets a bounded launch wait', async () => { + const { device, log } = simulator([LAUNCH_PENDING, WRONG_ROUTE, LANDED]); await answerDeepLinkConfirmation(device); - assert.equal(waits(log), 2); + assert.equal(waits(log), 3); }); test('a confirmation that appears late is still answered once', async () => { @@ -110,7 +156,7 @@ test('the wait budget is bounded when the app never starts', async () => { await answerDeepLinkConfirmation(device); - assert.equal(waits(log), 4); + assert.equal(waits(log), 5); }); test('a prompt that is not the deep-link confirmation is never accepted', async () => { diff --git a/test/integration/ios-simulator-e2e/live-deep-link-confirmation.ts b/test/integration/ios-simulator-e2e/live-deep-link-confirmation.ts index 7e169a1321..cbeb4a4e64 100644 --- a/test/integration/ios-simulator-e2e/live-deep-link-confirmation.ts +++ b/test/integration/ios-simulator-e2e/live-deep-link-confirmation.ts @@ -6,11 +6,11 @@ import { type LiveContext, runStep } from './live-harness.ts'; /** One destination wait; generous because the WebView lab took over 2.5 s to mount on cold CI. */ const DEEP_LINK_DESTINATION_WAIT_MS = '15000'; /** - * One wait finds the confirmation holding the launch; three more cover the launch its answer - * releases. That launch usually reaches the foreground within 3 s of the tap, but a loaded host has - * held it for 20.7 s (CI run 35991523779) and 27.4 s (a local run), and the route renders after that. + * One wait can find the confirmation holding the launch; four more cover its release or a stalled + * runner restart. The answered launch usually reaches the foreground within 3 s, but a loaded host + * held it for 20.7 s (CI run 35991523779) and 27.4 s (a local run). */ -const DESTINATION_WAITS = 4; +const DESTINATION_WAITS = 5; /** `details.runnerErrorCode` of a read the runner refused because the session app is not running. */ const APP_NOT_RUNNING = 'APP_NOT_RUNNING'; @@ -43,11 +43,9 @@ export function acceptDeepLinkConfirmationIfPresent( } /** - * Only a destination wait that ends on `APP_NOT_RUNNING` — the launch is still pending — leads to - * the alert probe or to another wait. Any other outcome returns, so an arrived route skips the probe - * and a wrong route or a stalled capture fails on the caller's own destination assertion. The probe - * therefore never queries a rendered screen, where `alert get` against a live WKWebView exceeds the - * runner's execution watchdog and leaves later commands refused as `RUNNER_BUSY` (#2484 follow-up). + * A readable destination timeout can still leave the launch behind a system confirmation. Probe + * once per miss until it is answered; a truncated or stalled capture then gets another bounded + * wait, while a readable wrong-route miss without a prompt goes to the caller's assertion. */ export async function answerDeepLinkConfirmation( device: DeepLinkConfirmationDevice, @@ -57,8 +55,16 @@ export async function answerDeepLinkConfirmation( const arrived = await device.waitForDestination( `wait for the deep-link destination (${wait}/${DESTINATION_WAITS})`, ); - if (arrived.json?.error?.details?.runnerErrorCode !== APP_NOT_RUNNING) return; + if (arrived.status === 0) return; + const details = arrived.json?.error?.details; + const launchPending = details?.runnerErrorCode === APP_NOT_RUNNING; + const reason = details?.reason; + const readableMiss = reason === 'wait_target_absent' || reason === 'wait_deadline_exceeded'; + const interruptedCapture = + reason === 'wait_capture_stalled' || reason === 'wait_runner_restart_exhausted'; + if (!launchPending && !readableMiss && !interruptedCapture) return; if (!answered) answered = await acceptOpenConfirmation(device); + if (!answered && reason === 'wait_target_absent') return; } }