From bdaf06a340793c13a9a59c4d628ef7ad58534bc6 Mon Sep 17 00:00:00 2001 From: billsbooth <111548547+billsbooth@users.noreply.github.com> Date: Sun, 6 Sep 2026 16:44:47 -0400 Subject: [PATCH 1/2] test(e2e): capture scenario assertion failure evidence --- .../live-device-e2e-runtime.test.ts | 178 ++++++++++++++++++ test/integration/live-device-e2e/runtime.ts | 52 ++++- 2 files changed, 221 insertions(+), 9 deletions(-) create mode 100644 test/integration/live-device-e2e-runtime.test.ts diff --git a/test/integration/live-device-e2e-runtime.test.ts b/test/integration/live-device-e2e-runtime.test.ts new file mode 100644 index 0000000000..bf7b964a65 --- /dev/null +++ b/test/integration/live-device-e2e-runtime.test.ts @@ -0,0 +1,178 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import test from 'node:test'; + +import { mkdtempForTestSync } from '../../src/__tests__/test-utils/tmp-dir.ts'; +import type { CliJsonResult } from './cli-json.ts'; +import { createLiveDeviceContext, createLiveDeviceHarness } from './live-device-e2e/runtime.ts'; + +function fixture(options: { captureThrows?: boolean; deviceEvidence?: string } = {}) { + const context = createLiveDeviceContext({ + artifactRoot: mkdtempForTestSync('scenario-failure-evidence-'), + session: 'owned-fixture', + }); + const calls: string[][] = []; + let reports = 0; + const harness = createLiveDeviceHarness({ + behaviorsForScenario: () => [], + commandsForScenario: () => [], + commonFlags: (current, args) => [...args, '--session', current.session, '--json'], + runCli: async (args): Promise => { + calls.push(args); + if (args[0] === 'screenshot' || args[0] === 'snapshot') { + if (options.captureThrows) throw new Error('capture unavailable'); + if (args[0] === 'screenshot') fs.writeFileSync(args[1]!, 'fixture-png'); + return { status: 0, stdout: '', stderr: '', json: { success: true, data: { nodes: [] } } }; + } + return { status: 1, stdout: '', stderr: '', json: { success: false } }; + }, + ...(options.deviceEvidence === undefined + ? {} + : { deviceEvidence: async () => options.deviceEvidence }), + writeCoverageReport: () => { + reports += 1; + }, + }); + return { context, harness, calls, reports: () => reports }; +} + +test('a scenario assertion after allowed misses captures evidence before caller cleanup', async () => { + const { context, harness, calls, reports } = fixture(); + const failure = new assert.AssertionError({ message: 'canary never became visible' }); + await assert.rejects( + harness.runScenario(context, { + id: 'visibility', + run: async () => { + await harness.runStep(context, 'probe', ['is', 'visible', 'id="canary"'], { + allowFailure: true, + }); + throw failure; + }, + }), + (error: unknown) => error === failure, + ); + + assert.deepEqual( + calls.map((args) => args[0]), + ['is', 'screenshot', 'snapshot'], + ); + for (const args of calls) + assert.deepEqual(args.slice(-3), ['--session', 'owned-fixture', '--json']); + assert.ok(fs.existsSync(path.join(context.artifactDir, 'failed-step-1.png'))); + assert.ok(fs.existsSync(path.join(context.artifactDir, 'failed-step-1-snapshot.json'))); + const report = fs.readFileSync(path.join(context.artifactDir, 'failed-step.txt'), 'utf8'); + assert.match(report, /scenario: visibility/); + assert.match(report, /canary never became visible/); + assert.match(report, /failed-step-1-snapshot.json/); + assert.deepEqual(context.completedScenarios, []); + assert.equal(reports(), 1); +}); + +test('a scenario assertion before any command also captures evidence', async () => { + const { context, harness, calls } = fixture(); + const failure = new Error('fixture assertion'); + await assert.rejects( + harness.runScenario(context, { + id: 'assertion', + run: async () => { + throw failure; + }, + }), + (error: unknown) => error === failure, + ); + assert.deepEqual( + calls.map((args) => args[0]), + ['screenshot', 'snapshot'], + ); + assert.ok(fs.existsSync(path.join(context.artifactDir, 'failed-step-0-snapshot.json'))); +}); + +test('scenario failure includes platform-owned device evidence', async () => { + const { context, harness } = fixture({ deviceEvidence: 'fixture process exited' }); + const failure = new Error('missing canary'); + await assert.rejects( + harness.runScenario(context, { + id: 'device-facts', + run: async () => { + throw failure; + }, + }), + (error: unknown) => error === failure, + ); + const devicePath = path.join(context.artifactDir, 'failed-step-0-device.txt'); + assert.equal(fs.readFileSync(devicePath, 'utf8'), 'fixture process exited'); + assert.match( + fs.readFileSync(path.join(context.artifactDir, 'failed-step.txt'), 'utf8'), + /failed-step-0-device\.txt/, + ); +}); + +test('an already captured command failure is not captured again by its scenario', async () => { + const { context, harness, calls } = fixture(); + await assert.rejects( + harness.runScenario(context, { + id: 'command', + run: async () => { + await harness.runStep(context, 'read canary', ['get', 'text', 'id="canary"']); + }, + }), + /step: read canary/, + ); + assert.deepEqual( + calls.map((args) => args[0]), + ['get', 'screenshot', 'snapshot'], + ); + assert.match( + fs.readFileSync(path.join(context.artifactDir, 'failed-step.txt'), 'utf8'), + /step: read canary/, + ); +}); + +test('failed capture preserves the scenario error and still writes coverage', async () => { + const { context, harness, calls, reports } = fixture({ captureThrows: true }); + const failure = new Error('original assertion'); + await assert.rejects( + harness.runScenario(context, { + id: 'capture-failure', + run: async () => { + throw failure; + }, + }), + (error: unknown) => error === failure, + ); + assert.deepEqual( + calls.map((args) => args[0]), + ['screenshot', 'snapshot'], + ); + assert.equal(reports(), 1); + assert.match( + fs.readFileSync(path.join(context.artifactDir, 'failed-step.txt'), 'utf8'), + /capture failed/, + ); +}); + +test('a successful scenario has no diagnostic capture', async () => { + const { context, harness, calls, reports } = fixture(); + await harness.runScenario(context, { id: 'success', run: async () => undefined }); + assert.deepEqual(calls, []); + assert.deepEqual(context.completedScenarios, ['success']); + assert.equal(reports(), 1); + assert.equal(fs.existsSync(path.join(context.artifactDir, 'failed-step.txt')), false); +}); + +test('unwritable artifact output cannot replace the scenario failure', async () => { + const { context, harness, reports } = fixture(); + fs.renameSync(context.artifactDir, `${context.artifactDir}-moved`); + const failure = new Error('original failure before artifact I/O'); + await assert.rejects( + harness.runScenario(context, { + id: 'write-failure', + run: async () => { + throw failure; + }, + }), + (error: unknown) => error === failure, + ); + assert.equal(reports(), 1); +}); diff --git a/test/integration/live-device-e2e/runtime.ts b/test/integration/live-device-e2e/runtime.ts index d3d2e05f94..ed1cb7bf92 100644 --- a/test/integration/live-device-e2e/runtime.ts +++ b/test/integration/live-device-e2e/runtime.ts @@ -94,6 +94,8 @@ export function createLiveDeviceHarness< Context extends LiveDeviceContext, BehaviorId extends string, >(options: HarnessOptions) { + const reportedFailures = new WeakSet(); + async function runScenario(context: Context, scenario: LiveScenario): Promise { context.currentScenario = scenario.id; const commandCounts = evidenceCounts( @@ -120,12 +122,46 @@ export function createLiveDeviceHarness< behaviorCounts, ); context.completedScenarios.push(scenario.id); + } catch (error) { + await recordScenarioFailure(context, error); + throw error; } finally { context.timings.push({ durationMs: Date.now() - startedAt, id: scenario.id }); options.writeCoverageReport(context); } } + async function recordScenarioFailure(context: Context, error: unknown): Promise { + if (error instanceof Error && reportedFailures.has(error)) return; + try { + const evidence = await captureFailedStepEvidence(context); + writeFailureReport( + context, + error instanceof Error ? (error.stack ?? error.message) : String(error), + evidence, + ); + } catch { + // Artifact I/O is best-effort and must not replace the scenario failure. + } + } + + function writeFailureReport( + context: Context, + description: string, + evidence: FailedStepEvidence, + ): string { + const message = [ + description, + `scenario: ${context.currentScenario}`, + `artifacts: ${context.artifactDir}`, + `screenshot: ${evidence.screenshotPath ?? '(capture failed)'}`, + `snapshot: ${evidence.snapshotPath ?? '(capture failed)'}`, + `device: ${evidence.devicePath ?? '(not collected)'}`, + ].join('\n'); + fs.writeFileSync(path.join(context.artifactDir, 'failed-step.txt'), message); + return message; + } + async function runStep( context: Context, step: string, @@ -179,16 +215,14 @@ export function createLiveDeviceHarness< result.status !== 0 && !failedAsExpected && stepOptions.allowFailure !== true; if (unexpectedFailure) { const evidence = await captureFailedStepEvidence(context); - const message = [ + const message = writeFailureReport( + context, formatResultDebug(step, fullArgs, result), - `scenario: ${context.currentScenario}`, - `artifacts: ${context.artifactDir}`, - `screenshot: ${evidence.screenshotPath ?? '(capture failed)'}`, - `snapshot: ${evidence.snapshotPath ?? '(capture failed)'}`, - `device: ${evidence.devicePath ?? '(not collected)'}`, - ].join('\n'); - fs.writeFileSync(path.join(context.artifactDir, 'failed-step.txt'), message); - assert.fail(message); + evidence, + ); + const failure = new assert.AssertionError({ message }); + reportedFailures.add(failure); + throw failure; } if (stepOptions.expectFailure === true && result.status === 0) { assert.fail(`${step} unexpectedly succeeded\ncommand: agent-device ${fullArgs.join(' ')}`); From 15b7d9dcb5e1fc4c81023ac83b11d70980483dde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 24 Sep 2026 18:54:25 +0200 Subject: [PATCH 2/2] test(e2e): verify live iOS assertion artifacts --- .../ios-simulator-e2e/live-runner.ts | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/test/integration/ios-simulator-e2e/live-runner.ts b/test/integration/ios-simulator-e2e/live-runner.ts index 5e41ebf42e..02fd5a39be 100644 --- a/test/integration/ios-simulator-e2e/live-runner.ts +++ b/test/integration/ios-simulator-e2e/live-runner.ts @@ -94,9 +94,15 @@ export async function runIosSimulatorE2E(): Promise { } async function executeLiveScenarios(context: LiveContext): Promise { + let assertionCaptureVerified = false; for (const scenario of LIVE_SCENARIOS.filter((candidate) => candidate.tier === 'smoke')) { await runScenario(context, scenario); + if (!assertionCaptureVerified && context.sessionOpen) { + await assertLiveAssertionCapture(context); + assertionCaptureVerified = true; + } } + assert.ok(assertionCaptureVerified, 'iOS smoke did not open a session for assertion capture'); if (context.tier === 'full') { await runStep(context, 'reopen fixture for full tier', ['open', context.appId, '--relaunch']); for (const scenario of LIVE_SCENARIOS.filter((candidate) => candidate.tier === 'full')) { @@ -106,6 +112,33 @@ async function executeLiveScenarios(context: LiveContext): Promise { assertCoverageComplete(context); } +async function assertLiveAssertionCapture(context: LiveContext): Promise { + const startedAt = Date.now(); + const stem = `failed-step-${context.stepHistory.length}`; + const failure = new assert.AssertionError({ message: 'deliberate live assertion capture' }); + await assert.rejects( + runScenario(context, { + id: 'smoke:failure-evidence-canary', + run: async () => { + throw failure; + }, + }), + (error: unknown) => error === failure, + ); + + const screenshotPath = path.join(context.artifactDir, `${stem}.png`); + const snapshotPath = path.join(context.artifactDir, `${stem}-snapshot.json`); + const reportPath = path.join(context.artifactDir, 'failed-step.txt'); + assertPngFile(screenshotPath); + const snapshot = JSON.parse(fs.readFileSync(snapshotPath, 'utf8')) as { success?: unknown }; + assert.equal(snapshot.success, true); + const report = fs.readFileSync(reportPath, 'utf8'); + assert.match(report, /deliberate live assertion capture/); + assert.ok(report.includes(screenshotPath)); + assert.ok(report.includes(snapshotPath)); + console.log(`iOS live assertion capture: ${Date.now() - startedAt}ms; ${context.artifactDir}`); +} + async function finalizeLiveRun(context: LiveContext): Promise { let cleanupError = await finalizeSessionCleanup(context, sessionExists, cleanupSession); try {