fix(ios-runner): resend read-only commands across the RUNNER_BUSY drain window

packages/platform-apple/src/runner/__tests__/runner-command-busy-resend.test.ts

@@ -0,0 +1,213 @@
+import { beforeEach, test, vi } from 'vitest';
+import assert from 'node:assert/strict';
+import { IOS_SIMULATOR } from './device-fixtures.ts';
+import { createTestRequestCancellation, makeRunnerSession } from './runner-session-fixtures.ts';
+import { AppError, isRequestCanceledError } from '@agent-device/kernel/errors';
+import { appleRunnerTestHost } from '../test-host.ts';
+
+// The read-only resend policy around a `RUNNER_BUSY` refusal. The runner refuses fast while
+// watchdog-abandoned XCTest work drains, so the daemon's resend has to outlast that drain, keep
+// the old three-attempt budget for transport failures, and stay cancellable across the seconds it
+// may now wait.
+
+const {
+  mockEnsureRunnerSession,
+  mockExecuteRunnerCommandWithSession,
+  mockReadRunnerSessionLiveness,
+  mockInvalidateRunnerSession,
+} = vi.hoisted(() => ({
+  mockEnsureRunnerSession: vi.fn(),
+  mockExecuteRunnerCommandWithSession: vi.fn(),
+  mockReadRunnerSessionLiveness: vi.fn(),
+  mockInvalidateRunnerSession: vi.fn(),
+}));
+
+vi.mock('../runner-session.ts', async () => {
+  const actual =
+    await vi.importActual<typeof import('../runner-session.ts')>('../runner-session.ts');
+  return {
+    ...actual,
+    ensureRunnerSession: mockEnsureRunnerSession,
+    executeRunnerCommandWithSession: mockExecuteRunnerCommandWithSession,
+    readRunnerSessionLiveness: mockReadRunnerSessionLiveness,
+    invalidateRunnerSession: mockInvalidateRunnerSession,
+  };
+});
+
+import { runAppleRunnerCommand } from '../runner-client.ts';
+import { buildRunnerResponseError } from '../runner-contract.ts';
+import { resetRunnerRecycleLedgerForTests } from '../runner-recycle-ledger.ts';
+
+const requestCancellation = createTestRequestCancellation();
+
+beforeEach(() => {
+  vi.resetAllMocks();
+  vi.useRealTimers();
+  resetRunnerRecycleLedgerForTests();
+  requestCancellation.reset();
+  appleRunnerTestHost.update({
+    emitDiagnostic: vi.fn(),
+    isRequestCanceled: requestCancellation.isRequestCanceled,
+    getRequestSignal: () => undefined,
+  });
+  const session = makeRunnerSession({ state: 'ready' });
+  mockEnsureRunnerSession.mockResolvedValue(session);
+  // A live session, or `executeRunnerCommand` reads the resend as a recycle boot and charges the
+  // per-request recycle budget instead of resending.
+  mockReadRunnerSessionLiveness.mockReturnValue({
+    sessionId: session.sessionId,
+    liveness: 'ready',
+  });
+});
+
+/** The runner's live refusal, exactly as the transport decodes it. */
+function busyRefusal(): AppError {
+  return buildRunnerResponseError({
+    ok: false,
+    error: { code: 'RUNNER_BUSY', message: 'The iOS runner is still finishing a previous command' },
+  });
+}
+
+function sentCommands(): string[] {
+  return mockExecuteRunnerCommandWithSession.mock.calls.map((call) => call[2].command);
+}
+
+test('read-only commands wait out RUNNER_BUSY past the transport resend backoff', async () => {
+  // Three busy answers spaced 200/400/800ms apart, then the runner answers. The default three
+  // attempts stopped after the second delay, before the abandoned work had drained.
+  vi.useFakeTimers();
+  mockExecuteRunnerCommandWithSession
+    .mockRejectedValueOnce(busyRefusal())
+    .mockRejectedValueOnce(busyRefusal())
+    .mockRejectedValueOnce(busyRefusal())
+    .mockResolvedValueOnce({ nodes: [], truncated: false });
+
+  const pending = runAppleRunnerCommand(IOS_SIMULATOR, { command: 'snapshot' });
+  await vi.advanceTimersByTimeAsync(6_000);
+  const result = await pending;
+
+  assert.deepEqual(result, { nodes: [], truncated: false });
+  assert.equal(mockInvalidateRunnerSession.mock.calls.length, 0);
+  // A structured refusal already says the command did not run, so no status probe sits between
+  // the resends.
+  assert.deepEqual(sentCommands(), ['snapshot', 'snapshot', 'snapshot', 'snapshot']);
+});
+
+test('a RUNNER_BUSY resend still ends when the refusals outlast the window', async () => {
+  vi.useFakeTimers();
+  mockExecuteRunnerCommandWithSession.mockImplementation(async () => {
+    throw busyRefusal();
+  });
+
+  const pending = runAppleRunnerCommand(IOS_SIMULATOR, { command: 'snapshot' });
+  const settled = pending.then(
+    () => undefined,
+    (error: unknown) => error,
+  );
+  await vi.advanceTimersByTimeAsync(20_000);
+  const error = await settled;
+
+  assert.ok(error instanceof AppError);
+  assert.equal(error.details?.runnerErrorCode, 'RUNNER_BUSY');
+  assert.equal(error.details?.recovery, undefined, 'the raw refusal reaches the caller unwrapped');
+  assert.deepEqual(sentCommands(), Array<string>(8).fill('snapshot'));
+});
+
+test('read-only transport failures keep the three-attempt resend budget', async () => {
+  mockExecuteRunnerCommandWithSession
+    .mockRejectedValueOnce(new AppError('COMMAND_FAILED', 'fetch failed'))
+    .mockResolvedValueOnce({ lifecycleState: 'started' })
+    .mockRejectedValueOnce(new AppError('COMMAND_FAILED', 'fetch failed'))
+    .mockResolvedValueOnce({ lifecycleState: 'started' })
+    .mockRejectedValueOnce(new AppError('COMMAND_FAILED', 'fetch failed'))
+    .mockResolvedValueOnce({ lifecycleState: 'started' })
+    .mockResolvedValueOnce({ nodes: [], truncated: false });
+
+  await assert.rejects(
+    () => runAppleRunnerCommand(IOS_SIMULATOR, { command: 'snapshot' }),
+    (error: unknown) => {
+      assert.ok(error instanceof AppError);
+      assert.equal(error.message, 'fetch failed');
+      return true;
+    },
+  );
+
+  assert.equal(sentCommands().filter((command) => command === 'snapshot').length, 3);
+});
+
+test('a wait deadline landing mid-window rethrows the last RUNNER_BUSY refusal, not a cancel', async () => {
+  // `wait` bounds each poll with its own abort signal (runWithinWaitDeadline) and keeps the last
+  // typed refusal as the wait's cause. The refusal must therefore survive the deadline: a bare
+  // cancellation would make the wait report a stalled capture and drop the runner's own code.
+  vi.useFakeTimers();
+  const deadline = new AbortController();
+  mockExecuteRunnerCommandWithSession
+    .mockRejectedValueOnce(busyRefusal())
+    .mockResolvedValue({ nodes: [], truncated: false });
+
+  const pending = runAppleRunnerCommand(
+    IOS_SIMULATOR,
+    { command: 'snapshot' },
+    { requestId: 'req-wait', signal: deadline.signal },
+  );
+  const settled = pending.then(
+    () => undefined,
+    (error: unknown) => error,
+  );
+  await vi.advanceTimersByTimeAsync(50);
+  deadline.abort(new DOMException('Wait deadline exceeded', 'TimeoutError'));
+  await vi.advanceTimersByTimeAsync(0);
+  const error = await settled;
+
+  assert.ok(error instanceof AppError, `expected the refusal, got ${String(error)}`);
+  assert.equal(error.details?.runnerErrorCode, 'RUNNER_BUSY');
+  assert.equal(error.details?.retriable, true);
+  assert.deepEqual(sentCommands(), ['snapshot']);
+});
+
+test('a cancelled request wakes the RUNNER_BUSY delay and reports the cancellation', async () => {
+  vi.useFakeTimers();
+  const request = new AbortController();
+  appleRunnerTestHost.update({ getRequestSignal: () => request.signal });
+  mockExecuteRunnerCommandWithSession
+    .mockRejectedValueOnce(busyRefusal())
+    .mockResolvedValue({ nodes: [], truncated: false });
+
+  const pending = runAppleRunnerCommand(
+    IOS_SIMULATOR,
+    { command: 'snapshot' },
+    { requestId: 'req-cancelled' },
+  );
+  const settled = pending.then(
+    () => undefined,
+    (error: unknown) => error,
+  );
+  // Inside the first 200ms delay; the next attempt would otherwise succeed at the timer.
+  await vi.advanceTimersByTimeAsync(50);
+  requestCancellation.markRequestCanceled('req-cancelled');
+  request.abort();
+  await vi.advanceTimersByTimeAsync(0);
+  const error = await settled;
+
+  assert.ok(isRequestCanceledError(error), `expected a canceled request, got ${String(error)}`);
+  assert.deepEqual(sentCommands(), ['snapshot']);
+});
+
+test('a mutating command meets RUNNER_BUSY once, with no status probe and no resend', async () => {
+  // The status-recovery bypass for structured replies reaches mutating commands too: the refusal
+  // says the tap did not run, so there is nothing to recover and nothing is replayed.
+  mockExecuteRunnerCommandWithSession.mockRejectedValueOnce(busyRefusal());
+
+  await assert.rejects(
+    () => runAppleRunnerCommand(IOS_SIMULATOR, { command: 'tap', x: 120, y: 240 }),
+    (error: unknown) => {
+      assert.ok(error instanceof AppError);
+      assert.equal(error.details?.runnerErrorCode, 'RUNNER_BUSY');
+      assert.equal(error.details?.recovery, undefined);
+      return true;
+    },
+  );
+
+  assert.deepEqual(sentCommands(), ['tap']);
+  assert.equal(mockInvalidateRunnerSession.mock.calls.length, 0);
+});

packages/platform-apple/src/runner/__tests__/runner-error-classification.test.ts

@@ -8,6 +8,7 @@ import {
 import {
   RUNNER_ERROR_RULES,
   isRetryableRunnerError,
+  isRunnerBusyError,
   resolveRunnerFatalErrorReason,
   shouldRebuildCachedRunnerArtifact,
   shouldRestartRunnerAfterReadinessPreflight,
@@ -301,3 +302,27 @@ test('a refused screen capture keeps the runner reason and stays off the wire co
     assert.equal(classified.details.retriable, undefined);
   }
 });
+
+// --- busy refusal (isRunnerBusyError) ---
+
+test('only the typed RUNNER_BUSY refusal reads as busy', () => {
+  assert.equal(
+    isRunnerBusyError(commandFailed('runner is busy', { runnerErrorCode: 'RUNNER_BUSY' })),
+    true,
+  );
+  // The stalling command's own timeout already spent its wait: not a refusal to resend.
+  assert.equal(
+    isRunnerBusyError(
+      commandFailed('main thread execution timed out', { runnerErrorCode: 'MAIN_THREAD_TIMEOUT' }),
+    ),
+    false,
+  );
+  assert.equal(
+    isRunnerBusyError(
+      commandFailed('The iOS runner is still finishing a previous command', { retriable: true }),
+    ),
+    false,
+  );
+  assert.equal(isRunnerBusyError(commandFailed('RUNNER_BUSY')), false);
+  assert.equal(isRunnerBusyError(new Error('RUNNER_BUSY')), false);
+});

packages/platform-apple/src/runner/runner-client.ts

@@ -1,5 +1,6 @@
-import { retryWithPolicy, emitDiagnostic } from './host.ts';
+import { retryWithPolicy, emitDiagnostic, getRequestSignal, isRequestCanceled } from './host.ts';
 import { isIosFamily, type DeviceInfo } from '@agent-device/kernel/device';
+import { isRequestCanceledError } from '@agent-device/kernel/errors';
 import {
   ensureRunnerSession,
   readRunnerSessionLiveness,
@@ -9,10 +10,11 @@ import {
 } from './runner-session.ts';
 import {
   assertRunnerRequestActive,
+  resolveRunnerRequestSignal,
   withRunnerCommandId,
   type RunnerCommand,
 } from './runner-contract.ts';
-import { isRetryableRunnerError } from './runner-error-classification.ts';
+import { isRetryableRunnerError, isRunnerBusyError } from './runner-error-classification.ts';
 import { isReadOnlyRunnerCommand } from './runner-command-traits.ts';
 import {
   createLocalAppleRunnerProvider,
@@ -32,6 +34,41 @@ import { RUNNER_COMMAND_TIMEOUT_MS } from './runner-transport.ts';
 
 // --- Runner command execution ---
 
+/**
+ * Attempts a read-only command may spend on one error class; 1 means no resend. A `RUNNER_BUSY`
+ * refusal is the runner refusing fast on purpose while it drains abandoned XCTest work (#1105), so
+ * that budget has to outlast the drain. The window is a heuristic, not a measurement: 200ms doubling
+ * to a 1s cap, no jitter, is 5.4s of delay across eight attempts before round trips, and a drain
+ * that outlives it still surfaces as `RUNNER_BUSY` (the runner's own `abandonedForSeconds=` marker
+ * in runner.log is the evidence for tuning it). Transport failures keep the pre-existing three
+ * attempts. The budget is positional: the error on each attempt sets how many attempts the loop may
+ * reach in total, so three busy refusals followed by a transport failure resend no further.
+ */
+const RUNNER_BUSY_RESEND_ATTEMPTS = 8;
+const TRANSPORT_RESEND_ATTEMPTS = 3;
+const READ_ONLY_RESEND_POLICY = {
+  maxAttempts: RUNNER_BUSY_RESEND_ATTEMPTS,
+  baseDelayMs: 200,
+  maxDelayMs: 1_000,
+  jitter: 0,
+};
+
+function readOnlyResendBudget(error: unknown): number {
+  if (isRunnerBusyError(error)) return RUNNER_BUSY_RESEND_ATTEMPTS;
+  return isRetryableRunnerError(error) ? TRANSPORT_RESEND_ATTEMPTS : 1;
+}
+
+/**
+ * Whether the request itself was cancelled, as opposed to one caller's deadline on this command.
+ * A `wait` bounds each poll with its own abort signal; that deadline landing inside the resend
+ * window must not erase the refusal the runner already gave.
+ */
+function isRunnerRequestCancelled(options: AppleRunnerCommandOptions): boolean {
+  return (
+    isRequestCanceled(options.requestId) || getRequestSignal(options.requestId)?.aborted === true
+  );
+}
+
 export async function runAppleRunnerCommand(
   device: DeviceInfo,
   command: RunnerCommand,
@@ -41,21 +78,37 @@ export async function runAppleRunnerCommand(
   assertRunnerRequestActive(options.requestId);
   const runnerCommand = withRunnerCommandId(command);
   const provider = resolveAppleRunnerRuntime(device, options);
-  if (isReadOnlyRunnerCommand(runnerCommand)) {
-    return retryWithPolicy(
+  if (!isReadOnlyRunnerCommand(runnerCommand)) {
+    return provider.runCommand(device, runnerCommand, options);
+  }
+  let lastBusyRefusal: unknown;
+  try {
+    return await retryWithPolicy(
       () => {
         assertRunnerRequestActive(options.requestId);
         return provider.runCommand(device, runnerCommand, options);
       },
       {
-        shouldRetry: (error) => {
+        ...READ_ONLY_RESEND_POLICY,
+        shouldRetry: (error, attempt) => {
           assertRunnerRequestActive(options.requestId);
-          return isRetryableRunnerError(error);
+          if (isRunnerBusyError(error)) lastBusyRefusal = error;
+          return attempt < readOnlyResendBudget(error);
         },
       },
+      // The busy window is seconds long, so an abort must wake the delay instead of sleeping it
+      // out and sending one more attempt.
+      { signal: resolveRunnerRequestSignal(options) },
     );
+  } catch (error) {
+    // A caller's deadline (a `wait` poll bounding this capture) that lands mid-window still has an
+    // answer: the runner refused, and that typed refusal is what the caller can act on. Only a
+    // cancelled request reports as a bare cancellation.
+    if (isRequestCanceledError(error) && lastBusyRefusal && !isRunnerRequestCancelled(options)) {
+      throw lastBusyRefusal;
+    }
+    throw error;
   }
-  return provider.runCommand(device, runnerCommand, options);
 }
 
 export async function notifyIosRunnerAppRelaunched(