fix(interaction): report an unsettled post-gesture surface instead of a definite miss

CHANGELOG.md

@@ -2,6 +2,14 @@
 
 ## Unreleased
 
+- Fixed (mobile): a read taken right after a `scroll`, `swipe`, or `gesture swipe` no longer reports
+  a definite miss when the surface never settled. When post-gesture stabilization ran out of budget
+  on a surface still moving, `is visible` answered a plain `selector_not_found` and `is absent`
+  passed. That capture now carries `unsettledGesture`: `is`, `get`, `find`, and `wait` report it (in
+  `error.details` or `data`) with an appended warning, `snapshot` appends the warning, `is absent`
+  refuses with `observation: "unsettled"`, `wait absent` keeps polling, and the next read captures
+  afresh. Click, press, and fill by selector do not disclose it yet. A failed read now also carries
+  `targetActivation` in `error.details`, the same place as `unsettledGesture`.
 - Fixed (ios): `open` on a local Simulator now waits for the launched app's discovery before it
   decides whether the app is observable. On a loaded host `simctl spawn launchctl list` outlasts one
   1.5 s discovery wait slice, and the launch observation read that slice as an unobservable app, so

packages/capture-kit/src/post-gesture-stability.ts

@@ -1,5 +1,6 @@
 import { emitDiagnostic } from '@agent-device/host-kit/diagnostics';
 import { sleep } from '@agent-device/host-kit/retry';
+import type { PostGestureAction } from '@agent-device/kernel/snapshot';
 
 /**
  * Pure post-gesture stability mechanics: the quiet-window polling loop and the
@@ -58,6 +59,8 @@ export type PostGestureStabilityHooks<T, S extends readonly unknown[]> = {
 
 export type PostGestureStabilityOutcome<T> = {
   value: T;
+  /** Present when the deadline expired while the last two captures still disagreed. */
+  unsettledGesture?: PostGestureAction;
   /**
    * Present ONLY when the accept-stale verdict is corroborated by full-surface
    * evidence (`surfacesIdentical`). The bare verdict is NOT enough — it is
@@ -66,9 +69,33 @@ export type PostGestureStabilityOutcome<T> = {
    * Callers surface this to the agent: a diagnostics-only signal let one
    * benchmark run burn 40 calls re-issuing scrolls that moved nothing (#1600).
    */
-  gestureNoEffect?: { action: string; positionals: string[] };
+  gestureNoEffect?: PostGestureAction;
 };
 
+function describePostGestureAction(gesture: PostGestureAction): string {
+  return [gesture.action, ...gesture.positionals].join(' ').trim();
+}
+
+/**
+ * The agent-facing wording for a proven no-effect gesture. Names the exact
+ * gesture, admits the honest ambiguity (at-edge is a legitimate no-op the
+ * platform cannot distinguish), and hands over the one escape hatch that
+ * moved a stuck list when synthesized scrolls did not (#1600, element-18:
+ * raw `swipe` worked where scroll/fling/pan all silently no-opped).
+ */
+export function formatGestureNoEffectWarning(action: string, positionals: string[]): string {
+  return (
+    `${describePostGestureAction({ action, positionals })} produced no visible change: the tree still matches its pre-gesture state. ` +
+    'Either the container is already at its edge, or it ignores synthesized scrolls — ' +
+    'a raw drag moves such lists: swipe x1 y1 x2 y2 (start inside the list).'
+  );
+}
+
+/** A miss on a tree read while a gesture's surface was still changing is not proof of absence. */
+export function formatGestureUnsettledWarning(gesture: PostGestureAction): string {
+  return `The surface was still changing after ${describePostGestureAction(gesture)} when this was read, so an element missing from it may still be on screen. Read again before treating it as absent.`;
+}
+
 /**
  * Verdict for a quiet match that has already been observed. `'ambiguous'`
  * baseline evidence (no comparable content) falls through to `trust`, same as
@@ -114,12 +141,16 @@ export async function runPostGestureStabilityLoop<T, S extends readonly unknown[
   // Extended past STABILIZATION_DEADLINE_MS only when the distrust verdict
   // fires below; the ordinary (non-distrust) timeout path is unaffected.
   let effectiveDeadlineMs = STABILIZATION_DEADLINE_MS;
+  // A rebase or a distrust verdict keeps polling on a pair that DID agree, so
+  // the deadline can expire on a surface that is already at rest.
+  let lastPairAgreed = false;
 
   while (attempts < STABILIZATION_MIN_ATTEMPTS || Date.now() - startedAt < effectiveDeadlineMs) {
     await sleep(STABILIZATION_INTERVAL_MS);
     attempts += 1;
     const current = await captureSurface(hooks);
-    if (hooks.signaturesStable(previous.signature, current.signature)) {
+    lastPairAgreed = hooks.signaturesStable(previous.signature, current.signature);
+    if (lastPairAgreed) {
       const elapsedMs = Date.now() - startedAt;
       // A capture plan may fall back or be pre-empted by the XCTest-channel
       // penalty at any time, so the backend can change mid-poll. Backends do
@@ -164,9 +195,14 @@ export async function runPostGestureStabilityLoop<T, S extends readonly unknown[
       action: pending.action,
       attempts,
       durationMs: Date.now() - startedAt,
+      lastPairAgreed,
     },
   });
-  return { value: previous.value };
+  if (lastPairAgreed) return { value: previous.value };
+  return {
+    value: previous.value,
+    unsettledGesture: { action: pending.action, positionals: pending.positionals },
+  };
 }
 
 type CapturedSurface<T, S> = {

packages/kernel/src/snapshot.ts

@@ -500,8 +500,13 @@ export type SnapshotState = {
    * the foreground instead (#2682). Consumers that surface this tree disclose the repair.
    */
   targetActivation?: IosTargetActivation;
+  /** The gesture whose surface was still changing when stabilization gave up on this capture. */
+  unsettledGesture?: PostGestureAction;
 } & SnapshotStateProvenance;
 
+/** The gesture a post-gesture outcome fact names: the command and its positionals. */
+export type PostGestureAction = { action: string; positionals: string[] };
+
 export type SnapshotUnchanged = {
   ageMs: number;
   nodeCount: number;

packages/selectors/src/absence-observation-errors.ts

@@ -1,6 +1,8 @@
 import { asAppError, AppError } from '@agent-device/kernel/errors';
 import { INTERACTION_ERROR_REASONS } from './interaction-error.ts';
 import {
+  UNPROVABLE_ABSENCE_CAUSES,
+  type UnprovableAbsenceKind,
   absenceCaptureOptionMessage,
   type AbsenceCaptureOption,
   type AbsenceObservation,
@@ -47,9 +49,7 @@ export function absenceObservationError(
   }
   return new AppError(
     'COMMAND_FAILED',
-    `${command} absent could not prove absence for selector ${selector}: ${
-      observation.kind === 'sparse' ? 'capture was sparse' : 'capture was truncated'
-    }`,
+    `${command} absent could not prove absence for selector ${selector}: ${UNPROVABLE_ABSENCE_CAUSES[observation.kind as UnprovableAbsenceKind]}`,
     {
       ...details,
       hint: 'Retry after the accessibility capture is complete.',

packages/selectors/src/absence-observation.ts

@@ -32,7 +32,21 @@ export type AbsenceObservation =
   | { kind: 'absent'; matches: 0 }
   | { kind: 'present'; matches: number; firstMatch: AbsenceFirstMatch }
   | { kind: 'sparse'; matches: number; firstMatch?: AbsenceFirstMatch; quality: SparseQuality }
-  | { kind: 'truncated'; matches: number; firstMatch?: AbsenceFirstMatch };
+  | { kind: 'truncated'; matches: number; firstMatch?: AbsenceFirstMatch }
+  | { kind: 'unsettled'; matches: 0 };
+
+/** Why a capture with no match still cannot prove absence. */
+export const UNPROVABLE_ABSENCE_CAUSES = {
+  sparse: 'capture was sparse',
+  truncated: 'capture was truncated',
+  unsettled: 'the surface was still changing after a gesture',
+} as const;
+
+export type UnprovableAbsenceKind = keyof typeof UNPROVABLE_ABSENCE_CAUSES;
+
+export function isUnprovableAbsence(kind: unknown): kind is UnprovableAbsenceKind {
+  return typeof kind === 'string' && Object.hasOwn(UNPROVABLE_ABSENCE_CAUSES, kind);
+}
 
 export type AbsenceCaptureOption = 'depth' | 'scope';
 
@@ -56,7 +70,10 @@ export function absenceCaptureOptionMessage(
 }
 
 export function classifyAbsenceObservation(
-  snapshot: Pick<SnapshotState, 'backend' | 'nodes' | 'snapshotQuality' | 'truncated'>,
+  snapshot: Pick<
+    SnapshotState,
+    'backend' | 'nodes' | 'snapshotQuality' | 'truncated' | 'unsettledGesture'
+  >,
   matches: readonly SnapshotNode[],
 ): AbsenceObservation {
   const firstMatch = matches[0] ? stableFirstMatch(matches[0]) : undefined;
@@ -82,7 +99,9 @@ export function classifyAbsenceObservation(
       },
     };
   }
-  if (matchCount === 0) return { kind: 'absent', matches: 0 };
+  if (matchCount === 0) {
+    return { kind: snapshot.unsettledGesture ? 'unsettled' : 'absent', matches: 0 };
+  }
   return { kind: 'present', matches: matchCount, firstMatch: firstMatch! };
 }
 

src/commands/capture/runtime/snapshot.test.ts

@@ -13,6 +13,7 @@ import {
   type CommandSessionStore,
 } from '../../../runtime.ts';
 import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures';
+import { formatGestureUnsettledWarning } from '@agent-device/capture-kit/post-gesture-stability';
 
 test('runtime snapshot captures nodes and updates the session baseline', async () => {
   let stored: Parameters<CommandSessionStore['set']>[0] | undefined;
@@ -786,3 +787,19 @@ test('runtime snapshot leaves the keyboard band off when the backend measured no
 
   assert.equal('keyboard' in result, false);
 });
+
+test('runtime snapshot warns when its tree was read on a surface still moving after a gesture', async () => {
+  const gesture = { action: 'scroll', positionals: ['down'] };
+  const device = createSnapshotOnlyDevice({
+    snapshot: {
+      ...makeSnapshotState([{ index: 0, depth: 0, type: 'Window', label: 'Home' }], {
+        backend: 'xctest',
+      }),
+      unsettledGesture: gesture,
+    },
+  });
+
+  const result = await device.capture.snapshot({ session: 'default' });
+
+  assert.deepEqual(result.warnings, [formatGestureUnsettledWarning(gesture)]);
+});

src/commands/capture/runtime/snapshot.ts

@@ -26,6 +26,7 @@ import {
   renderSnapshotQualityWarnings,
   truncatedCaptureWarning,
 } from '@agent-device/capture-kit/quality-warnings';
+import { formatGestureUnsettledWarning } from '@agent-device/capture-kit/post-gesture-stability';
 import { buildSnapshotVisibility } from '@agent-device/capture-kit/snapshot-visibility';
 import { ANDROID_SYSTEM_SURFACE_DISCLOSURE } from '@agent-device/contracts/android-system-surface-disclosure';
 import { formatReactNativeOverlayWarning } from '../../react-native/overlay.ts';
@@ -260,6 +261,9 @@ function buildSnapshotWarnings(params: {
     );
   }
   warnings.push(...truncatedCaptureWarning(snapshotTruncationForResult(params.snapshot)));
+  if (params.snapshot.unsettledGesture) {
+    warnings.push(formatGestureUnsettledWarning(params.snapshot.unsettledGesture));
+  }
   warnings.push(...buildEmptyAndroidInteractiveWarnings(params));
   if (!params.annotations.quality) {
     // Legacy runners without a structured verdict keep the old daemon-side heuristics.

src/commands/interaction/metadata.ts

@@ -70,7 +70,7 @@ const interactionCommandDescriptions = {
   scroll:
     'Scroll in a direction, or toward the top/bottom edge of scrollable content. Set until to a selector to reach an off-screen target in one command rather than a scroll-and-check loop. The optional amount is the finger-path fraction of the viewport axis, honored up to 0.8 of it; directional scrolls reduce release momentum, while app scroll physics determine the final content offset. A visible keyboard shortens the swiped band instead of being dismissed; when too little is left, the command refuses with scroll_keyboard_occludes_surface. A directional scroll also reports the movement it observed as movement: moved, at-edge, unchanged, or unobserved when the two reads could not back a claim either way; an unchanged surface inside a container that still hides content in that direction refuses with scroll_no_progress rather than repeating the requested distance. The movement field is absent where a tier verifies per pass (top/bottom, until), where the runtime cannot read a screen, or where a settle observation or a replay already owns that observation.',
   get: 'Read text or accessibility attributes from a snapshot ref or selector without changing the app. Use format text for visible content or attrs for the element attribute map.',
-  is: 'Check whether a selector satisfies a UI predicate such as visible, hidden, exists, absent, editable, selected, focused, or text. `absent` passes only when one readable, complete, unscoped, full-depth accessibility capture has zero matches. Use wait when the condition may appear asynchronously.',
+  is: 'Check whether a selector satisfies a UI predicate such as visible, hidden, exists, absent, editable, selected, focused, or text. `absent` passes only when one readable, complete, settled, unscoped, full-depth accessibility capture has zero matches. Use wait when the condition may appear asynchronously.',
   find: 'Find by text/label/value/role/id and run action',
   gesture:
     'Perform a structured pan, fling, swipe, pinch, rotate, transform, or drag gesture. Select the gesture kind, then provide only the inputs that apply to that kind.',

src/commands/interaction/runtime/wait-absent.test.ts

@@ -141,6 +141,23 @@ test('wait absent rides out sparse and truncated captures without counting them
   assert.equal(result.waitedMs >= 600, true);
 });
 
+test('wait absent does not take a miss on a surface still moving after a gesture as absence', async () => {
+  const unsettled = {
+    snapshot: {
+      ...makeSnapshotState([]),
+      unsettledGesture: { action: 'scroll', positionals: ['down'] },
+    },
+  };
+  const device = absentDevice([unsettled, snapshot('Removed')]);
+
+  await assert.rejects(waitAbsent(device, 500), (error: unknown) => {
+    assert.ok(error instanceof AppError);
+    assert.equal(error.details?.reason, 'wait_target_present');
+    assert.equal(error.details?.readableCaptures, 1);
+    return true;
+  });
+});
+
 test('wait absent excludes sparse and truncated polls from deadline readable-capture evidence', async () => {
   const sparse = snapshot(undefined, {
     snapshotQuality: { state: 'sparse', backend: 'private-ax', reasonCode: 'sparse-tree' },

src/commands/interaction/runtime/wait-absent.ts

@@ -3,6 +3,7 @@ import { isUnreadableCaptureContentError } from '@agent-device/contracts/android
 import { AppError } from '@agent-device/kernel/errors';
 import {
   absenceCaptureOptionRefusal,
+  isUnprovableAbsence,
   type AbsenceObservation,
 } from '@agent-device/selectors/absence-observation';
 import {
@@ -63,7 +64,7 @@ export async function waitForAbsent<Runtime extends SelectorWaitRuntime>(
           selectorExpression,
           runtime.backend.platform,
         );
-        if (observation.kind === 'sparse' || observation.kind === 'truncated') {
+        if (isUnprovableAbsence(observation.kind)) {
           throw absenceObservationError(selectorExpression, observation, 'wait');
         }
         return observation;
@@ -130,8 +131,6 @@ function isWaitAbsentUnreadableError(error: unknown): boolean {
   return (
     details?.command === 'wait' &&
     details.predicate === 'absent' &&
-    (details.observation === 'sparse' ||
-      details.observation === 'truncated' ||
-      details.observation === 'unreadable')
+    (isUnprovableAbsence(details.observation) || details.observation === 'unreadable')
   );
 }

src/daemon/__tests__/is-runtime.test.ts

@@ -1,4 +1,4 @@
-import { beforeEach, expect, test, vi } from 'vitest';
+import { afterEach, beforeEach, expect, test, vi } from 'vitest';
 import type { SnapshotResult } from '@agent-device/contracts/snapshot-runtime';
 import { buildSnapshotPresentationKey } from '@agent-device/kernel/snapshot';
 import { ANDROID_EMULATOR, IOS_SIMULATOR } from '../../__tests__/test-utils/device-fixtures.ts';
@@ -12,6 +12,8 @@ import { withTestDeviceInventory } from '../../__tests__/test-utils/device-inven
 import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures';
 import type { DaemonRequest } from '../daemon-request.ts';
 import { selectorCaptureFixture } from './selector-capture-fixture.ts';
+import { markDeferredInteractionOutcome } from '../deferred-interaction-outcome.ts';
+import { formatGestureUnsettledWarning } from '@agent-device/capture-kit/post-gesture-stability';
 
 const { mockRunAppleRunnerCommand } = vi.hoisted(() => ({ mockRunAppleRunnerCommand: vi.fn() }));
 
@@ -28,6 +30,10 @@ beforeEach(() => {
   mockRunAppleRunnerCommand.mockResolvedValue({});
 });
 
+afterEach(() => {
+  vi.useRealTimers();
+});
+
 // `is` answers every one of its eight predicates from the resolved capture — `isCommand` never
 // reaches `backend.readText`. So its whole platform execution is the request-bound capture, and
 // these cases bind at `inspectFacts` / `bindDevice`, never at `-device/device-selection/dispatch-resolve`.
@@ -416,3 +422,49 @@ test('a failing predicate answers COMMAND_FAILED from the bound capture', async
   // The bound capture is what answered it.
   expect(fixture.captures.length).toBeGreaterThan(0);
 });
+
+test('a miss on a surface that never settled carries the unsettled fact, and the re-read captures afresh', async () => {
+  vi.useFakeTimers();
+  // Every capture shows the row at a new offset, so no two consecutive reads agree.
+  const fixture = selectorCaptureFixture({
+    snapshot: (_input, index) => ({
+      nodes: [
+        {
+          index: 0,
+          type: 'Cell',
+          identifier: 'row',
+          rect: { x: 0, y: 200 - index * 37, width: 390, height: 60 },
+        },
+      ],
+      backend: 'xctest',
+      producer: 'apple-runner',
+    }),
+  });
+  const sessionStore = makeSessionStore();
+  const session = makeIosAppSession('is-unsettled');
+  markDeferredInteractionOutcome({ session, command: 'scroll', positionals: [], flags: {} });
+  sessionStore.set('is-unsettled', session);
+  const isVisible = () =>
+    dispatchIsViaRuntime({
+      req: isRequest('is-unsettled', ['visible', 'id=target']),
+      sessionName: 'is-unsettled',
+      sessionStore,
+      inspectFacts: fixture.inspectFacts,
+      bindDevice: fixture.bindDevice,
+    });
+  const pending = isVisible();
+  // Just past the 1.5s stabilization deadline, so the re-read lands inside the cache window.
+  await vi.advanceTimersByTimeAsync(1_700);
+  const gesture = { action: 'scroll', positionals: [] };
+
+  const response = await pending;
+  expect(response?.ok === false && response.error.details).toMatchObject({
+    reason: 'selector_not_found',
+    unsettledGesture: gesture,
+    hint: expect.stringContaining(formatGestureUnsettledWarning(gesture)),
+  });
+  const captures = fixture.captures.length;
+  const reread = await isVisible();
+  expect(fixture.captures.length).toBe(captures + 1);
+  expect(reread?.ok === false && reread.error.details?.unsettledGesture).toBeUndefined();
+});

src/daemon/__tests__/post-gesture-no-effect-claim.test.ts

@@ -17,7 +17,7 @@ import {
   capturePostGestureStabilizedResult,
   markDeferredInteractionOutcome,
 } from '../deferred-interaction-outcome.ts';
-import { formatGestureNoEffectWarning } from '../gesture-no-effect.ts';
+import { formatGestureNoEffectWarning } from '@agent-device/capture-kit/post-gesture-stability';
 import type { SessionState } from '../session-state.ts';
 import {
   chromeWithListSnapshot,