test(e2e): capture scenario assertion failure evidence

test/integration/ios-simulator-e2e/live-runner.ts

@@ -94,9 +94,15 @@ export async function runIosSimulatorE2E(): Promise<void> {
 }
 
 async function executeLiveScenarios(context: LiveContext): Promise<void> {
+  let assertionCaptureVerified = false;
   for (const scenario of LIVE_SCENARIOS.filter((candidate) => candidate.tier === 'smoke')) {
     await runScenario(context, scenario);
+    if (!assertionCaptureVerified && context.sessionOpen) {
+      await assertLiveAssertionCapture(context);
+      assertionCaptureVerified = true;
+    }
   }
+  assert.ok(assertionCaptureVerified, 'iOS smoke did not open a session for assertion capture');
   if (context.tier === 'full') {
     await runStep(context, 'reopen fixture for full tier', ['open', context.appId, '--relaunch']);
     for (const scenario of LIVE_SCENARIOS.filter((candidate) => candidate.tier === 'full')) {
@@ -106,6 +112,33 @@ async function executeLiveScenarios(context: LiveContext): Promise<void> {
   assertCoverageComplete(context);
 }
 
+async function assertLiveAssertionCapture(context: LiveContext): Promise<void> {
+  const startedAt = Date.now();
+  const stem = `failed-step-${context.stepHistory.length}`;
+  const failure = new assert.AssertionError({ message: 'deliberate live assertion capture' });
+  await assert.rejects(
+    runScenario(context, {
+      id: 'smoke:failure-evidence-canary',
+      run: async () => {
+        throw failure;
+      },
+    }),
+    (error: unknown) => error === failure,
+  );
+
+  const screenshotPath = path.join(context.artifactDir, `${stem}.png`);
+  const snapshotPath = path.join(context.artifactDir, `${stem}-snapshot.json`);
+  const reportPath = path.join(context.artifactDir, 'failed-step.txt');
+  assertPngFile(screenshotPath);
+  const snapshot = JSON.parse(fs.readFileSync(snapshotPath, 'utf8')) as { success?: unknown };
+  assert.equal(snapshot.success, true);
+  const report = fs.readFileSync(reportPath, 'utf8');
+  assert.match(report, /deliberate live assertion capture/);
+  assert.ok(report.includes(screenshotPath));
+  assert.ok(report.includes(snapshotPath));
+  console.log(`iOS live assertion capture: ${Date.now() - startedAt}ms; ${context.artifactDir}`);
+}
+
 async function finalizeLiveRun(context: LiveContext): Promise<unknown> {
   let cleanupError = await finalizeSessionCleanup(context, sessionExists, cleanupSession);
   try {

test/integration/live-device-e2e-runtime.test.ts

@@ -0,0 +1,178 @@
+import assert from 'node:assert/strict';
+import fs from 'node:fs';
+import path from 'node:path';
+import test from 'node:test';
+
+import { mkdtempForTestSync } from '../../src/__tests__/test-utils/tmp-dir.ts';
+import type { CliJsonResult } from './cli-json.ts';
+import { createLiveDeviceContext, createLiveDeviceHarness } from './live-device-e2e/runtime.ts';
+
+function fixture(options: { captureThrows?: boolean; deviceEvidence?: string } = {}) {
+  const context = createLiveDeviceContext<string>({
+    artifactRoot: mkdtempForTestSync('scenario-failure-evidence-'),
+    session: 'owned-fixture',
+  });
+  const calls: string[][] = [];
+  let reports = 0;
+  const harness = createLiveDeviceHarness<typeof context, string>({
+    behaviorsForScenario: () => [],
+    commandsForScenario: () => [],
+    commonFlags: (current, args) => [...args, '--session', current.session, '--json'],
+    runCli: async (args): Promise<CliJsonResult> => {
+      calls.push(args);
+      if (args[0] === 'screenshot' || args[0] === 'snapshot') {
+        if (options.captureThrows) throw new Error('capture unavailable');
+        if (args[0] === 'screenshot') fs.writeFileSync(args[1]!, 'fixture-png');
+        return { status: 0, stdout: '', stderr: '', json: { success: true, data: { nodes: [] } } };
+      }
+      return { status: 1, stdout: '', stderr: '', json: { success: false } };
+    },
+    ...(options.deviceEvidence === undefined
+      ? {}
+      : { deviceEvidence: async () => options.deviceEvidence }),
+    writeCoverageReport: () => {
+      reports += 1;
+    },
+  });
+  return { context, harness, calls, reports: () => reports };
+}
+
+test('a scenario assertion after allowed misses captures evidence before caller cleanup', async () => {
+  const { context, harness, calls, reports } = fixture();
+  const failure = new assert.AssertionError({ message: 'canary never became visible' });
+  await assert.rejects(
+    harness.runScenario(context, {
+      id: 'visibility',
+      run: async () => {
+        await harness.runStep(context, 'probe', ['is', 'visible', 'id="canary"'], {
+          allowFailure: true,
+        });
+        throw failure;
+      },
+    }),
+    (error: unknown) => error === failure,
+  );
+
+  assert.deepEqual(
+    calls.map((args) => args[0]),
+    ['is', 'screenshot', 'snapshot'],
+  );
+  for (const args of calls)
+    assert.deepEqual(args.slice(-3), ['--session', 'owned-fixture', '--json']);
+  assert.ok(fs.existsSync(path.join(context.artifactDir, 'failed-step-1.png')));
+  assert.ok(fs.existsSync(path.join(context.artifactDir, 'failed-step-1-snapshot.json')));
+  const report = fs.readFileSync(path.join(context.artifactDir, 'failed-step.txt'), 'utf8');
+  assert.match(report, /scenario: visibility/);
+  assert.match(report, /canary never became visible/);
+  assert.match(report, /failed-step-1-snapshot.json/);
+  assert.deepEqual(context.completedScenarios, []);
+  assert.equal(reports(), 1);
+});
+
+test('a scenario assertion before any command also captures evidence', async () => {
+  const { context, harness, calls } = fixture();
+  const failure = new Error('fixture assertion');
+  await assert.rejects(
+    harness.runScenario(context, {
+      id: 'assertion',
+      run: async () => {
+        throw failure;
+      },
+    }),
+    (error: unknown) => error === failure,
+  );
+  assert.deepEqual(
+    calls.map((args) => args[0]),
+    ['screenshot', 'snapshot'],
+  );
+  assert.ok(fs.existsSync(path.join(context.artifactDir, 'failed-step-0-snapshot.json')));
+});
+
+test('scenario failure includes platform-owned device evidence', async () => {
+  const { context, harness } = fixture({ deviceEvidence: 'fixture process exited' });
+  const failure = new Error('missing canary');
+  await assert.rejects(
+    harness.runScenario(context, {
+      id: 'device-facts',
+      run: async () => {
+        throw failure;
+      },
+    }),
+    (error: unknown) => error === failure,
+  );
+  const devicePath = path.join(context.artifactDir, 'failed-step-0-device.txt');
+  assert.equal(fs.readFileSync(devicePath, 'utf8'), 'fixture process exited');
+  assert.match(
+    fs.readFileSync(path.join(context.artifactDir, 'failed-step.txt'), 'utf8'),
+    /failed-step-0-device\.txt/,
+  );
+});
+
+test('an already captured command failure is not captured again by its scenario', async () => {
+  const { context, harness, calls } = fixture();
+  await assert.rejects(
+    harness.runScenario(context, {
+      id: 'command',
+      run: async () => {
+        await harness.runStep(context, 'read canary', ['get', 'text', 'id="canary"']);
+      },
+    }),
+    /step: read canary/,
+  );
+  assert.deepEqual(
+    calls.map((args) => args[0]),
+    ['get', 'screenshot', 'snapshot'],
+  );
+  assert.match(
+    fs.readFileSync(path.join(context.artifactDir, 'failed-step.txt'), 'utf8'),
+    /step: read canary/,
+  );
+});
+
+test('failed capture preserves the scenario error and still writes coverage', async () => {
+  const { context, harness, calls, reports } = fixture({ captureThrows: true });
+  const failure = new Error('original assertion');
+  await assert.rejects(
+    harness.runScenario(context, {
+      id: 'capture-failure',
+      run: async () => {
+        throw failure;
+      },
+    }),
+    (error: unknown) => error === failure,
+  );
+  assert.deepEqual(
+    calls.map((args) => args[0]),
+    ['screenshot', 'snapshot'],
+  );
+  assert.equal(reports(), 1);
+  assert.match(
+    fs.readFileSync(path.join(context.artifactDir, 'failed-step.txt'), 'utf8'),
+    /capture failed/,
+  );
+});
+
+test('a successful scenario has no diagnostic capture', async () => {
+  const { context, harness, calls, reports } = fixture();
+  await harness.runScenario(context, { id: 'success', run: async () => undefined });
+  assert.deepEqual(calls, []);
+  assert.deepEqual(context.completedScenarios, ['success']);
+  assert.equal(reports(), 1);
+  assert.equal(fs.existsSync(path.join(context.artifactDir, 'failed-step.txt')), false);
+});
+
+test('unwritable artifact output cannot replace the scenario failure', async () => {
+  const { context, harness, reports } = fixture();
+  fs.renameSync(context.artifactDir, `${context.artifactDir}-moved`);
+  const failure = new Error('original failure before artifact I/O');
+  await assert.rejects(
+    harness.runScenario(context, {
+      id: 'write-failure',
+      run: async () => {
+        throw failure;
+      },
+    }),
+    (error: unknown) => error === failure,
+  );
+  assert.equal(reports(), 1);
+});

test/integration/live-device-e2e/runtime.ts

@@ -94,6 +94,8 @@ export function createLiveDeviceHarness<
   Context extends LiveDeviceContext<BehaviorId>,
   BehaviorId extends string,
 >(options: HarnessOptions<Context, BehaviorId>) {
+  const reportedFailures = new WeakSet<Error>();
+
   async function runScenario(context: Context, scenario: LiveScenario<Context>): Promise<void> {
     context.currentScenario = scenario.id;
     const commandCounts = evidenceCounts(
@@ -120,12 +122,46 @@ export function createLiveDeviceHarness<
         behaviorCounts,
       );
       context.completedScenarios.push(scenario.id);
+    } catch (error) {
+      await recordScenarioFailure(context, error);
+      throw error;
     } finally {
       context.timings.push({ durationMs: Date.now() - startedAt, id: scenario.id });
       options.writeCoverageReport(context);
     }
   }
 
+  async function recordScenarioFailure(context: Context, error: unknown): Promise<void> {
+    if (error instanceof Error && reportedFailures.has(error)) return;
+    try {
+      const evidence = await captureFailedStepEvidence(context);
+      writeFailureReport(
+        context,
+        error instanceof Error ? (error.stack ?? error.message) : String(error),
+        evidence,
+      );
+    } catch {
+      // Artifact I/O is best-effort and must not replace the scenario failure.
+    }
+  }
+
+  function writeFailureReport(
+    context: Context,
+    description: string,
+    evidence: FailedStepEvidence,
+  ): string {
+    const message = [
+      description,
+      `scenario: ${context.currentScenario}`,
+      `artifacts: ${context.artifactDir}`,
+      `screenshot: ${evidence.screenshotPath ?? '(capture failed)'}`,
+      `snapshot: ${evidence.snapshotPath ?? '(capture failed)'}`,
+      `device: ${evidence.devicePath ?? '(not collected)'}`,
+    ].join('\n');
+    fs.writeFileSync(path.join(context.artifactDir, 'failed-step.txt'), message);
+    return message;
+  }
+
   async function runStep(
     context: Context,
     step: string,
@@ -179,16 +215,14 @@ export function createLiveDeviceHarness<
       result.status !== 0 && !failedAsExpected && stepOptions.allowFailure !== true;
     if (unexpectedFailure) {
       const evidence = await captureFailedStepEvidence(context);
-      const message = [
+      const message = writeFailureReport(
+        context,
         formatResultDebug(step, fullArgs, result),
-        `scenario: ${context.currentScenario}`,
-        `artifacts: ${context.artifactDir}`,
-        `screenshot: ${evidence.screenshotPath ?? '(capture failed)'}`,
-        `snapshot: ${evidence.snapshotPath ?? '(capture failed)'}`,
-        `device: ${evidence.devicePath ?? '(not collected)'}`,
-      ].join('\n');
-      fs.writeFileSync(path.join(context.artifactDir, 'failed-step.txt'), message);
-      assert.fail(message);
+        evidence,
+      );
+      const failure = new assert.AssertionError({ message });
+      reportedFailures.add(failure);
+      throw failure;
     }
     if (stepOptions.expectFailure === true && result.status === 0) {
       assert.fail(`${step} unexpectedly succeeded\ncommand: agent-device ${fullArgs.join(' ')}`);