diff --git a/CHANGELOG.md b/CHANGELOG.md index 2cdcb0b8eb..187b1f3762 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,14 @@ ## Unreleased +- Fixed (mobile): a read taken right after a `scroll`, `swipe`, or `gesture swipe` no longer reports + a definite miss when the surface never settled. When post-gesture stabilization ran out of budget + on a surface still moving, `is visible` answered a plain `selector_not_found` and `is absent` + passed. That capture now carries `unsettledGesture`: `is`, `get`, `find`, and `wait` report it (in + `error.details` or `data`) with an appended warning, `snapshot` appends the warning, `is absent` + refuses with `observation: "unsettled"`, `wait absent` keeps polling, and the next read captures + afresh. Click, press, and fill by selector do not disclose it yet. A failed read now also carries + `targetActivation` in `error.details`, the same place as `unsettledGesture`. - Fixed (ios): `open` on a local Simulator now waits for the launched app's discovery before it decides whether the app is observable. On a loaded host `simctl spawn launchctl list` outlasts one 1.5 s discovery wait slice, and the launch observation read that slice as an unobservable app, so diff --git a/packages/capture-kit/src/post-gesture-stability.ts b/packages/capture-kit/src/post-gesture-stability.ts index cad874f7c8..438a674bbb 100644 --- a/packages/capture-kit/src/post-gesture-stability.ts +++ b/packages/capture-kit/src/post-gesture-stability.ts @@ -1,5 +1,6 @@ import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; import { sleep } from '@agent-device/host-kit/retry'; +import type { PostGestureAction } from '@agent-device/kernel/snapshot'; /** * Pure post-gesture stability mechanics: the quiet-window polling loop and the @@ -58,6 +59,8 @@ export type PostGestureStabilityHooks = { export type PostGestureStabilityOutcome = { value: T; + /** Present when the deadline expired while the last two captures still disagreed. */ + unsettledGesture?: PostGestureAction; /** * Present ONLY when the accept-stale verdict is corroborated by full-surface * evidence (`surfacesIdentical`). The bare verdict is NOT enough — it is @@ -66,9 +69,33 @@ export type PostGestureStabilityOutcome = { * Callers surface this to the agent: a diagnostics-only signal let one * benchmark run burn 40 calls re-issuing scrolls that moved nothing (#1600). */ - gestureNoEffect?: { action: string; positionals: string[] }; + gestureNoEffect?: PostGestureAction; }; +function describePostGestureAction(gesture: PostGestureAction): string { + return [gesture.action, ...gesture.positionals].join(' ').trim(); +} + +/** + * The agent-facing wording for a proven no-effect gesture. Names the exact + * gesture, admits the honest ambiguity (at-edge is a legitimate no-op the + * platform cannot distinguish), and hands over the one escape hatch that + * moved a stuck list when synthesized scrolls did not (#1600, element-18: + * raw `swipe` worked where scroll/fling/pan all silently no-opped). + */ +export function formatGestureNoEffectWarning(action: string, positionals: string[]): string { + return ( + `${describePostGestureAction({ action, positionals })} produced no visible change: the tree still matches its pre-gesture state. ` + + 'Either the container is already at its edge, or it ignores synthesized scrolls — ' + + 'a raw drag moves such lists: swipe x1 y1 x2 y2 (start inside the list).' + ); +} + +/** A miss on a tree read while a gesture's surface was still changing is not proof of absence. */ +export function formatGestureUnsettledWarning(gesture: PostGestureAction): string { + return `The surface was still changing after ${describePostGestureAction(gesture)} when this was read, so an element missing from it may still be on screen. Read again before treating it as absent.`; +} + /** * Verdict for a quiet match that has already been observed. `'ambiguous'` * baseline evidence (no comparable content) falls through to `trust`, same as @@ -114,12 +141,16 @@ export async function runPostGestureStabilityLoop = { diff --git a/packages/kernel/src/snapshot.ts b/packages/kernel/src/snapshot.ts index b8ec116309..33eeafed20 100644 --- a/packages/kernel/src/snapshot.ts +++ b/packages/kernel/src/snapshot.ts @@ -500,8 +500,13 @@ export type SnapshotState = { * the foreground instead (#2682). Consumers that surface this tree disclose the repair. */ targetActivation?: IosTargetActivation; + /** The gesture whose surface was still changing when stabilization gave up on this capture. */ + unsettledGesture?: PostGestureAction; } & SnapshotStateProvenance; +/** The gesture a post-gesture outcome fact names: the command and its positionals. */ +export type PostGestureAction = { action: string; positionals: string[] }; + export type SnapshotUnchanged = { ageMs: number; nodeCount: number; diff --git a/packages/selectors/src/absence-observation-errors.ts b/packages/selectors/src/absence-observation-errors.ts index 24f977fe83..196f0d0737 100644 --- a/packages/selectors/src/absence-observation-errors.ts +++ b/packages/selectors/src/absence-observation-errors.ts @@ -1,6 +1,8 @@ import { asAppError, AppError } from '@agent-device/kernel/errors'; import { INTERACTION_ERROR_REASONS } from './interaction-error.ts'; import { + UNPROVABLE_ABSENCE_CAUSES, + type UnprovableAbsenceKind, absenceCaptureOptionMessage, type AbsenceCaptureOption, type AbsenceObservation, @@ -47,9 +49,7 @@ export function absenceObservationError( } return new AppError( 'COMMAND_FAILED', - `${command} absent could not prove absence for selector ${selector}: ${ - observation.kind === 'sparse' ? 'capture was sparse' : 'capture was truncated' - }`, + `${command} absent could not prove absence for selector ${selector}: ${UNPROVABLE_ABSENCE_CAUSES[observation.kind as UnprovableAbsenceKind]}`, { ...details, hint: 'Retry after the accessibility capture is complete.', diff --git a/packages/selectors/src/absence-observation.ts b/packages/selectors/src/absence-observation.ts index ac2f811966..02964761ef 100644 --- a/packages/selectors/src/absence-observation.ts +++ b/packages/selectors/src/absence-observation.ts @@ -32,7 +32,21 @@ export type AbsenceObservation = | { kind: 'absent'; matches: 0 } | { kind: 'present'; matches: number; firstMatch: AbsenceFirstMatch } | { kind: 'sparse'; matches: number; firstMatch?: AbsenceFirstMatch; quality: SparseQuality } - | { kind: 'truncated'; matches: number; firstMatch?: AbsenceFirstMatch }; + | { kind: 'truncated'; matches: number; firstMatch?: AbsenceFirstMatch } + | { kind: 'unsettled'; matches: 0 }; + +/** Why a capture with no match still cannot prove absence. */ +export const UNPROVABLE_ABSENCE_CAUSES = { + sparse: 'capture was sparse', + truncated: 'capture was truncated', + unsettled: 'the surface was still changing after a gesture', +} as const; + +export type UnprovableAbsenceKind = keyof typeof UNPROVABLE_ABSENCE_CAUSES; + +export function isUnprovableAbsence(kind: unknown): kind is UnprovableAbsenceKind { + return typeof kind === 'string' && Object.hasOwn(UNPROVABLE_ABSENCE_CAUSES, kind); +} export type AbsenceCaptureOption = 'depth' | 'scope'; @@ -56,7 +70,10 @@ export function absenceCaptureOptionMessage( } export function classifyAbsenceObservation( - snapshot: Pick, + snapshot: Pick< + SnapshotState, + 'backend' | 'nodes' | 'snapshotQuality' | 'truncated' | 'unsettledGesture' + >, matches: readonly SnapshotNode[], ): AbsenceObservation { const firstMatch = matches[0] ? stableFirstMatch(matches[0]) : undefined; @@ -82,7 +99,9 @@ export function classifyAbsenceObservation( }, }; } - if (matchCount === 0) return { kind: 'absent', matches: 0 }; + if (matchCount === 0) { + return { kind: snapshot.unsettledGesture ? 'unsettled' : 'absent', matches: 0 }; + } return { kind: 'present', matches: matchCount, firstMatch: firstMatch! }; } diff --git a/src/commands/capture/runtime/snapshot.test.ts b/src/commands/capture/runtime/snapshot.test.ts index 378c3596a1..d24f27a490 100644 --- a/src/commands/capture/runtime/snapshot.test.ts +++ b/src/commands/capture/runtime/snapshot.test.ts @@ -13,6 +13,7 @@ import { type CommandSessionStore, } from '../../../runtime.ts'; import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; +import { formatGestureUnsettledWarning } from '@agent-device/capture-kit/post-gesture-stability'; test('runtime snapshot captures nodes and updates the session baseline', async () => { let stored: Parameters[0] | undefined; @@ -786,3 +787,19 @@ test('runtime snapshot leaves the keyboard band off when the backend measured no assert.equal('keyboard' in result, false); }); + +test('runtime snapshot warns when its tree was read on a surface still moving after a gesture', async () => { + const gesture = { action: 'scroll', positionals: ['down'] }; + const device = createSnapshotOnlyDevice({ + snapshot: { + ...makeSnapshotState([{ index: 0, depth: 0, type: 'Window', label: 'Home' }], { + backend: 'xctest', + }), + unsettledGesture: gesture, + }, + }); + + const result = await device.capture.snapshot({ session: 'default' }); + + assert.deepEqual(result.warnings, [formatGestureUnsettledWarning(gesture)]); +}); diff --git a/src/commands/capture/runtime/snapshot.ts b/src/commands/capture/runtime/snapshot.ts index bfa2a8c4ea..21f058fe53 100644 --- a/src/commands/capture/runtime/snapshot.ts +++ b/src/commands/capture/runtime/snapshot.ts @@ -26,6 +26,7 @@ import { renderSnapshotQualityWarnings, truncatedCaptureWarning, } from '@agent-device/capture-kit/quality-warnings'; +import { formatGestureUnsettledWarning } from '@agent-device/capture-kit/post-gesture-stability'; import { buildSnapshotVisibility } from '@agent-device/capture-kit/snapshot-visibility'; import { ANDROID_SYSTEM_SURFACE_DISCLOSURE } from '@agent-device/contracts/android-system-surface-disclosure'; import { formatReactNativeOverlayWarning } from '../../react-native/overlay.ts'; @@ -260,6 +261,9 @@ function buildSnapshotWarnings(params: { ); } warnings.push(...truncatedCaptureWarning(snapshotTruncationForResult(params.snapshot))); + if (params.snapshot.unsettledGesture) { + warnings.push(formatGestureUnsettledWarning(params.snapshot.unsettledGesture)); + } warnings.push(...buildEmptyAndroidInteractiveWarnings(params)); if (!params.annotations.quality) { // Legacy runners without a structured verdict keep the old daemon-side heuristics. diff --git a/src/commands/interaction/metadata.ts b/src/commands/interaction/metadata.ts index b497ceb4eb..6faba1ad95 100644 --- a/src/commands/interaction/metadata.ts +++ b/src/commands/interaction/metadata.ts @@ -70,7 +70,7 @@ const interactionCommandDescriptions = { scroll: 'Scroll in a direction, or toward the top/bottom edge of scrollable content. Set until to a selector to reach an off-screen target in one command rather than a scroll-and-check loop. The optional amount is the finger-path fraction of the viewport axis, honored up to 0.8 of it; directional scrolls reduce release momentum, while app scroll physics determine the final content offset. A visible keyboard shortens the swiped band instead of being dismissed; when too little is left, the command refuses with scroll_keyboard_occludes_surface. A directional scroll also reports the movement it observed as movement: moved, at-edge, unchanged, or unobserved when the two reads could not back a claim either way; an unchanged surface inside a container that still hides content in that direction refuses with scroll_no_progress rather than repeating the requested distance. The movement field is absent where a tier verifies per pass (top/bottom, until), where the runtime cannot read a screen, or where a settle observation or a replay already owns that observation.', get: 'Read text or accessibility attributes from a snapshot ref or selector without changing the app. Use format text for visible content or attrs for the element attribute map.', - is: 'Check whether a selector satisfies a UI predicate such as visible, hidden, exists, absent, editable, selected, focused, or text. `absent` passes only when one readable, complete, unscoped, full-depth accessibility capture has zero matches. Use wait when the condition may appear asynchronously.', + is: 'Check whether a selector satisfies a UI predicate such as visible, hidden, exists, absent, editable, selected, focused, or text. `absent` passes only when one readable, complete, settled, unscoped, full-depth accessibility capture has zero matches. Use wait when the condition may appear asynchronously.', find: 'Find by text/label/value/role/id and run action', gesture: 'Perform a structured pan, fling, swipe, pinch, rotate, transform, or drag gesture. Select the gesture kind, then provide only the inputs that apply to that kind.', diff --git a/src/commands/interaction/runtime/wait-absent.test.ts b/src/commands/interaction/runtime/wait-absent.test.ts index c9d3f833a8..2c5bf6f35f 100644 --- a/src/commands/interaction/runtime/wait-absent.test.ts +++ b/src/commands/interaction/runtime/wait-absent.test.ts @@ -141,6 +141,23 @@ test('wait absent rides out sparse and truncated captures without counting them assert.equal(result.waitedMs >= 600, true); }); +test('wait absent does not take a miss on a surface still moving after a gesture as absence', async () => { + const unsettled = { + snapshot: { + ...makeSnapshotState([]), + unsettledGesture: { action: 'scroll', positionals: ['down'] }, + }, + }; + const device = absentDevice([unsettled, snapshot('Removed')]); + + await assert.rejects(waitAbsent(device, 500), (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.details?.reason, 'wait_target_present'); + assert.equal(error.details?.readableCaptures, 1); + return true; + }); +}); + test('wait absent excludes sparse and truncated polls from deadline readable-capture evidence', async () => { const sparse = snapshot(undefined, { snapshotQuality: { state: 'sparse', backend: 'private-ax', reasonCode: 'sparse-tree' }, diff --git a/src/commands/interaction/runtime/wait-absent.ts b/src/commands/interaction/runtime/wait-absent.ts index 65ec917126..afcd08970f 100644 --- a/src/commands/interaction/runtime/wait-absent.ts +++ b/src/commands/interaction/runtime/wait-absent.ts @@ -3,6 +3,7 @@ import { isUnreadableCaptureContentError } from '@agent-device/contracts/android import { AppError } from '@agent-device/kernel/errors'; import { absenceCaptureOptionRefusal, + isUnprovableAbsence, type AbsenceObservation, } from '@agent-device/selectors/absence-observation'; import { @@ -63,7 +64,7 @@ export async function waitForAbsent( selectorExpression, runtime.backend.platform, ); - if (observation.kind === 'sparse' || observation.kind === 'truncated') { + if (isUnprovableAbsence(observation.kind)) { throw absenceObservationError(selectorExpression, observation, 'wait'); } return observation; @@ -130,8 +131,6 @@ function isWaitAbsentUnreadableError(error: unknown): boolean { return ( details?.command === 'wait' && details.predicate === 'absent' && - (details.observation === 'sparse' || - details.observation === 'truncated' || - details.observation === 'unreadable') + (isUnprovableAbsence(details.observation) || details.observation === 'unreadable') ); } diff --git a/src/daemon/__tests__/is-runtime.test.ts b/src/daemon/__tests__/is-runtime.test.ts index 6e044371fd..722948d9ab 100644 --- a/src/daemon/__tests__/is-runtime.test.ts +++ b/src/daemon/__tests__/is-runtime.test.ts @@ -1,4 +1,4 @@ -import { beforeEach, expect, test, vi } from 'vitest'; +import { afterEach, beforeEach, expect, test, vi } from 'vitest'; import type { SnapshotResult } from '@agent-device/contracts/snapshot-runtime'; import { buildSnapshotPresentationKey } from '@agent-device/kernel/snapshot'; import { ANDROID_EMULATOR, IOS_SIMULATOR } from '../../__tests__/test-utils/device-fixtures.ts'; @@ -12,6 +12,8 @@ import { withTestDeviceInventory } from '../../__tests__/test-utils/device-inven import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; import type { DaemonRequest } from '../daemon-request.ts'; import { selectorCaptureFixture } from './selector-capture-fixture.ts'; +import { markDeferredInteractionOutcome } from '../deferred-interaction-outcome.ts'; +import { formatGestureUnsettledWarning } from '@agent-device/capture-kit/post-gesture-stability'; const { mockRunAppleRunnerCommand } = vi.hoisted(() => ({ mockRunAppleRunnerCommand: vi.fn() })); @@ -28,6 +30,10 @@ beforeEach(() => { mockRunAppleRunnerCommand.mockResolvedValue({}); }); +afterEach(() => { + vi.useRealTimers(); +}); + // `is` answers every one of its eight predicates from the resolved capture — `isCommand` never // reaches `backend.readText`. So its whole platform execution is the request-bound capture, and // these cases bind at `inspectFacts` / `bindDevice`, never at `-device/device-selection/dispatch-resolve`. @@ -416,3 +422,49 @@ test('a failing predicate answers COMMAND_FAILED from the bound capture', async // The bound capture is what answered it. expect(fixture.captures.length).toBeGreaterThan(0); }); + +test('a miss on a surface that never settled carries the unsettled fact, and the re-read captures afresh', async () => { + vi.useFakeTimers(); + // Every capture shows the row at a new offset, so no two consecutive reads agree. + const fixture = selectorCaptureFixture({ + snapshot: (_input, index) => ({ + nodes: [ + { + index: 0, + type: 'Cell', + identifier: 'row', + rect: { x: 0, y: 200 - index * 37, width: 390, height: 60 }, + }, + ], + backend: 'xctest', + producer: 'apple-runner', + }), + }); + const sessionStore = makeSessionStore(); + const session = makeIosAppSession('is-unsettled'); + markDeferredInteractionOutcome({ session, command: 'scroll', positionals: [], flags: {} }); + sessionStore.set('is-unsettled', session); + const isVisible = () => + dispatchIsViaRuntime({ + req: isRequest('is-unsettled', ['visible', 'id=target']), + sessionName: 'is-unsettled', + sessionStore, + inspectFacts: fixture.inspectFacts, + bindDevice: fixture.bindDevice, + }); + const pending = isVisible(); + // Just past the 1.5s stabilization deadline, so the re-read lands inside the cache window. + await vi.advanceTimersByTimeAsync(1_700); + const gesture = { action: 'scroll', positionals: [] }; + + const response = await pending; + expect(response?.ok === false && response.error.details).toMatchObject({ + reason: 'selector_not_found', + unsettledGesture: gesture, + hint: expect.stringContaining(formatGestureUnsettledWarning(gesture)), + }); + const captures = fixture.captures.length; + const reread = await isVisible(); + expect(fixture.captures.length).toBe(captures + 1); + expect(reread?.ok === false && reread.error.details?.unsettledGesture).toBeUndefined(); +}); diff --git a/src/daemon/__tests__/post-gesture-no-effect-claim.test.ts b/src/daemon/__tests__/post-gesture-no-effect-claim.test.ts index 6314be67f5..bfe0ddca12 100644 --- a/src/daemon/__tests__/post-gesture-no-effect-claim.test.ts +++ b/src/daemon/__tests__/post-gesture-no-effect-claim.test.ts @@ -17,7 +17,7 @@ import { capturePostGestureStabilizedResult, markDeferredInteractionOutcome, } from '../deferred-interaction-outcome.ts'; -import { formatGestureNoEffectWarning } from '../gesture-no-effect.ts'; +import { formatGestureNoEffectWarning } from '@agent-device/capture-kit/post-gesture-stability'; import type { SessionState } from '../session-state.ts'; import { chromeWithListSnapshot, diff --git a/src/daemon/__tests__/post-gesture-stabilization.test.ts b/src/daemon/__tests__/post-gesture-stabilization.test.ts index 18f532a523..d8c99751f8 100644 --- a/src/daemon/__tests__/post-gesture-stabilization.test.ts +++ b/src/daemon/__tests__/post-gesture-stabilization.test.ts @@ -330,6 +330,43 @@ test('capturePostGestureStabilizedResult keeps the ordinary never-quiet timeout ); }); +test('a deadline that expires right after a rebased quiet pair is not reported as unsettled', async () => { + vi.useFakeTimers(); + const session = makeSession('ios'); + session.snapshot = makeSnapshotState(pickupSnapshot(500).nodes, { + snapshotQuality: { state: 'healthy', backend: 'tree' }, + }); + markPostGestureStabilization(session, 'scroll'); + // Moving on the baseline backend until the final poll pair, which agrees on another backend: + // the loop rebases on it and then runs out of time with the surface at rest. + let call = 0; + const capture = vi.fn(async () => { + call += 1; + return call >= 8 + ? makeSnapshotState(pickupSnapshot(640).nodes, { + snapshotQuality: { state: 'healthy', backend: 'private-ax' }, + }) + : makeSnapshotState(pickupSnapshot(100 + call * 40).nodes, { + snapshotQuality: { state: 'healthy', backend: 'tree' }, + }); + }); + + const resultPromise = withDiagnosticsScope({}, async () => ({ + result: await capturePostGestureStabilizedResult({ + session, + capture, + readSnapshot: (snapshot) => snapshot, + }), + rebased: countDiagnosticEventsByPhase(['post_gesture_snapshot_baseline_rebased']), + timeouts: countDiagnosticEventsByPhase(['post_gesture_snapshot_stabilization_timeout']), + })); + await vi.advanceTimersByTimeAsync(1_700); + const { result, rebased, timeouts } = await resultPromise; + + assert.deepEqual([rebased, timeouts], [1, 1]); + assert.equal(result.unsettledGesture, undefined); +}); + test('capturePostGestureStabilizedResult catches a frozen target even when the baseline came from a broader-scope capture than the post-gesture reads (iOS, live regression)', async () => { // Live shape (checkout-form.ad): the pre-gesture baseline is whatever // `session.snapshot` held from an earlier broad capture (e.g. a text-search diff --git a/src/daemon/capture-disclosure.ts b/src/daemon/capture-disclosure.ts index 87aa1bb3fd..1e594f1613 100644 --- a/src/daemon/capture-disclosure.ts +++ b/src/daemon/capture-disclosure.ts @@ -2,12 +2,13 @@ import type { SnapshotState } from '@agent-device/kernel/snapshot'; import { systemSurfaceDisclosure } from '@agent-device/contracts/android-system-surface-disclosure'; import { iosSystemSurfaceDisclosure } from '@agent-device/contracts/ios-system-surface'; import { iosTargetActivationDisclosure } from '@agent-device/contracts/ios-target-activation'; +import { formatGestureUnsettledWarning } from '@agent-device/capture-kit/post-gesture-stability'; import type { DaemonResponse } from './daemon-request.ts'; -/** The capture provenance a response must be disclosed against (#2438, #2682). */ +/** The capture provenance a response must be disclosed against (#2438, #2682, unsettled gestures). */ export type CaptureProvenance = Pick< SnapshotState, - 'systemSurfaceOnly' | 'iosSystemSurfaceBundleId' | 'targetActivation' + 'systemSurfaceOnly' | 'iosSystemSurfaceBundleId' | 'targetActivation' | 'unsettledGesture' >; /** @@ -58,20 +59,36 @@ export function withSystemSurfaceDisclosure( } /** - * Disclose a foreground repair the consumed capture carried (#2682). The fact applies to the whole - * tree, so it travels as response-level metadata and its sentence is APPENDED — an earlier warning - * (staleness, quality, an occluding surface) is never replaced. The typed fact lands even when the - * sentence was already carried: the field is this response's own claim, independent of who spoke. + * Disclose a fact about the whole answered tree (#2682 foreground repair, an unsettled gesture). Its + * sentence is APPENDED, never replacing an earlier warning, and the typed fact lands on either + * outcome (`data` or `error.details`) even when the sentence was already carried. */ +function withTreeFactDisclosure( + response: DaemonResponse, + key: K, + fact: CaptureProvenance[K], + sentence: (fact: NonNullable) => string, +): DaemonResponse { + if (!fact) return response; + const disclosed = appendDisclosure(response, sentence(fact), 'warnings'); + return disclosed.ok + ? { ...disclosed, data: { ...disclosed.data, [key]: fact } } + : { + ...disclosed, + error: { ...disclosed.error, details: { ...disclosed.error.details, [key]: fact } }, + }; +} + export function withTargetActivationDisclosure( response: DaemonResponse, snapshot: CaptureProvenance | undefined, ): DaemonResponse { - const fact = snapshot?.targetActivation; - if (!fact) return response; - const disclosed = appendDisclosure(response, iosTargetActivationDisclosure(fact), 'warnings'); - if (!disclosed.ok) return disclosed; - return { ...disclosed, data: { ...disclosed.data, targetActivation: fact } }; + return withTreeFactDisclosure( + response, + 'targetActivation', + snapshot?.targetActivation, + iosTargetActivationDisclosure, + ); } /** @@ -87,7 +104,12 @@ export function withCaptureDisclosures(params: { }): DaemonResponse { const { response, consumedTree, activationProof } = params; return withTargetActivationDisclosure( - withSystemSurfaceDisclosure(response, consumedTree), + withTreeFactDisclosure( + withSystemSurfaceDisclosure(response, consumedTree), + 'unsettledGesture', + consumedTree?.unsettledGesture, + formatGestureUnsettledWarning, + ), activationProof?.state, ); } diff --git a/src/daemon/deferred-interaction-outcome.ts b/src/daemon/deferred-interaction-outcome.ts index 79e6cc4d87..23208ad04a 100644 --- a/src/daemon/deferred-interaction-outcome.ts +++ b/src/daemon/deferred-interaction-outcome.ts @@ -30,7 +30,10 @@ import { snapshotSurfaceComparisonKey, type InteractionRetryTap, } from './interaction-outcome-policy.ts'; -import { runPostGestureStabilityLoop } from '@agent-device/capture-kit/post-gesture-stability'; +import { + runPostGestureStabilityLoop, + type PostGestureStabilityOutcome, +} from '@agent-device/capture-kit/post-gesture-stability'; import type { SessionState } from './session-state.ts'; /** @@ -262,10 +265,7 @@ async function captureInteractionOutcomeAwareSnapshot( }); } - return { - snapshot: latest.snapshot, - ...withGestureNoEffectWarning(latest.annotations, stabilized.gestureNoEffect), - }; + return resolvedPostGestureCapture(stabilized); } async function waitForDelayedInteractionSurfaceChange( @@ -294,11 +294,7 @@ async function capturePostGestureAwareSnapshot( capture: async () => await capturePostActionSnapshotAttempt(params), readSnapshot: (attempt) => attempt.snapshot, }); - const latest = stabilized.value; - return { - snapshot: latest.snapshot, - ...withGestureNoEffectWarning(latest.annotations, stabilized.gestureNoEffect), - }; + return resolvedPostGestureCapture(stabilized); } async function capturePostActionSnapshotAttempt( @@ -311,12 +307,6 @@ async function capturePostActionSnapshotAttempt( return await params.capture(); } -export type PostGestureStabilizedResult = { - value: T; - /** See `PostGestureStabilityOutcome` in post-gesture-stability.ts (#1600/#1601). */ - gestureNoEffect?: { action: string; positionals: string[] }; -}; - /** * Session-aware adapter over the pure stability loop * (`post-gesture-stability.ts`): reads the pending record, supplies the @@ -329,7 +319,7 @@ export async function capturePostGestureStabilizedResult(params: { capture: () => Promise; readSnapshot: (result: T) => SnapshotState; initial?: T; -}): Promise> { +}): Promise> { const { session, capture, readSnapshot } = params; const pending = session?.postGestureStabilization; if (!session || !supportsPostGestureStabilization(session.device) || !pending) { @@ -364,6 +354,15 @@ export async function capturePostGestureStabilizedResult(params: { return outcome; } +/** The stabilized attempt as a capture result: the tree carries an unsettled outcome as its own fact. */ +function resolvedPostGestureCapture( + stabilized: PostGestureStabilityOutcome, +): DeferredOutcomeCaptureResult { + const { snapshot, annotations } = stabilized.value; + if (stabilized.unsettledGesture) snapshot.unsettledGesture = stabilized.unsettledGesture; + return { snapshot, ...withGestureNoEffectWarning(annotations, stabilized.gestureNoEffect) }; +} + function isPostGestureStabilizingAction( action: string, positionals: string[], diff --git a/src/daemon/gesture-no-effect.ts b/src/daemon/gesture-no-effect.ts index ea7924ee3b..38ce4d9cb3 100644 --- a/src/daemon/gesture-no-effect.ts +++ b/src/daemon/gesture-no-effect.ts @@ -1,20 +1,5 @@ import type { SnapshotCaptureAnnotations } from '@agent-device/contracts/capture'; - -/** - * The agent-facing wording for a proven no-effect gesture. Names the exact - * gesture, admits the honest ambiguity (at-edge is a legitimate no-op the - * platform cannot distinguish), and hands over the one escape hatch that - * moved a stuck list when synthesized scrolls did not (#1600, element-18: - * raw `swipe` worked where scroll/fling/pan all silently no-opped). - */ -export function formatGestureNoEffectWarning(action: string, positionals: string[]): string { - const gesture = [action, ...positionals].join(' ').trim(); - return ( - `${gesture} produced no visible change: the tree still matches its pre-gesture state. ` + - 'Either the container is already at its edge, or it ignores synthesized scrolls — ' + - 'a raw drag moves such lists: swipe x1 y1 x2 y2 (start inside the list).' - ); -} +import { formatGestureNoEffectWarning } from '@agent-device/capture-kit/post-gesture-stability'; /** * #1600: a proven no-effect gesture must reach the agent inside the very diff --git a/src/daemon/interaction/internal/__tests__/find-target-activation-disclosure.test.ts b/src/daemon/interaction/internal/__tests__/find-target-activation-disclosure.test.ts index 0e37269b9a..fdce22ef13 100644 --- a/src/daemon/interaction/internal/__tests__/find-target-activation-disclosure.test.ts +++ b/src/daemon/interaction/internal/__tests__/find-target-activation-disclosure.test.ts @@ -7,6 +7,8 @@ import type { DaemonResponse } from '../../../daemon-request.ts'; import { legacyDispatchCapture } from '../../../__tests__/legacy-snapshot-capture-fixture.ts'; import { getRuntimeBindings } from '../../../__tests__/interaction-get-runtime-fixture.ts'; import { handleFindCommands } from '../../index.ts'; +import { markDeferredInteractionOutcome } from '../../../deferred-interaction-outcome.ts'; +import { formatGestureUnsettledWarning } from '@agent-device/capture-kit/post-gesture-stability'; vi.mock('../../../snapshot-interactor-capture.ts', async () => { const fixture = await import('../../../__tests__/legacy-snapshot-capture-fixture.ts'); @@ -71,9 +73,12 @@ beforeEach(() => { legacyDispatchCapture.mockReset(); }); -async function findClick(captures: Record[]) { +async function findClick(captures: Record[], afterScroll = false) { const sessionStore = makeSessionStore(); - sessionStore.set('default', makeIosSession('default', { appBundleId: 'com.example.app' })); + const session = makeIosSession('default', { appBundleId: 'com.example.app' }); + if (afterScroll) + markDeferredInteractionOutcome({ session, command: 'scroll', positionals: [], flags: {} }); + sessionStore.set('default', session); let call = 0; legacyDispatchCapture.mockImplementation( async () => captures[Math.min(call++, captures.length - 1)], @@ -123,3 +128,32 @@ test('a find that stayed sparse reports the repair on the failure it returns', a const hint = String(response.error.details?.hint ?? ''); expect(hint).toContain(iosTargetActivationDisclosure(FACT)); }); + +/** The tree a mutating find resolves against carries every provenance fact, not a hand-picked few. */ +test('a find that misses on a surface still moving after a scroll reports the unsettled fact', async () => { + const moving = Array.from({ length: 40 }, (_, call) => ({ + ...RECOVERED_TREE, + nodes: [ + RECOVERED_TREE.nodes[0], + { ...RECOVERED_TREE.nodes[1], label: 'Wi-Fi', rect: { ...SCREEN, y: 600 - call * 37 } }, + ], + })); + const realSetTimeout = globalThis.setTimeout; + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] }); + let done = false; + const pending = findClick(moving, true).finally(() => (done = true)); + // The route awaits real I/O between polls, so the faked clock advances while the test yields. + while (!done) { + await vi.advanceTimersByTimeAsync(50); + await new Promise((resolve) => realSetTimeout(resolve, 1)); + } + const { response } = await pending; + vi.useRealTimers(); + + expect(response?.ok === false && response.error.details).toMatchObject({ + unsettledGesture: { action: 'scroll', positionals: [] }, + hint: expect.stringContaining( + formatGestureUnsettledWarning({ action: 'scroll', positionals: [] }), + ), + }); +}); diff --git a/src/daemon/interaction/internal/find-target-capture.ts b/src/daemon/interaction/internal/find-target-capture.ts index ba42c27bee..f1698f340d 100644 --- a/src/daemon/interaction/internal/find-target-capture.ts +++ b/src/daemon/interaction/internal/find-target-capture.ts @@ -1,11 +1,7 @@ import type { FindLocator } from '@agent-device/selectors'; import type { BoundSelectorCapture } from '../../selector-capture-binding.ts'; -import type { - SnapshotKeyboardBandFact, - SnapshotQualityVerdict, - SnapshotState, -} from '@agent-device/kernel/snapshot'; -import type { RequestActivationProof } from '../../capture-disclosure.ts'; +import type { SnapshotQualityVerdict, SnapshotState } from '@agent-device/kernel/snapshot'; +import type { CaptureProvenance, RequestActivationProof } from '../../capture-disclosure.ts'; import { createSelectorCaptureRuntime } from '../../selector-capture-runtime.ts'; import { SessionStore } from '../../session-store.ts'; import type { DaemonRequest, DaemonResponse } from '../../daemon-request.ts'; @@ -13,14 +9,8 @@ import type { SessionState } from '../../session-state.ts'; import { errorResponse } from '@agent-device/kernel/contracts'; /** The tree a mutating find resolves its target against, plus what the capture disclosed. */ -export type FindTargetTree = { - nodes: SnapshotState['nodes']; - snapshotQuality?: SnapshotQualityVerdict; - systemSurfaceOnly?: boolean; - iosSystemSurfaceBundleId?: string; - /** The keyboard band this capture's producer measured, when it measured one (#2660). */ - keyboard?: SnapshotKeyboardBandFact; -}; +export type FindTargetTree = CaptureProvenance & + Pick; /** * Find's target capture. A mutating find (click/fill/focus/type) resolves its target from its @@ -75,13 +65,7 @@ export function createFindTargetCapture( }, }, }); - return { - nodes: snapshot.nodes, - snapshotQuality: snapshot.snapshotQuality, - systemSurfaceOnly: snapshot.systemSurfaceOnly, - iosSystemSurfaceBundleId: snapshot.iosSystemSurfaceBundleId, - ...(snapshot.keyboard ? { keyboard: snapshot.keyboard } : {}), - }; + return snapshot; }; } diff --git a/src/daemon/selector-capture-runtime.ts b/src/daemon/selector-capture-runtime.ts index 5f7c9c7e61..1b24f5510a 100644 --- a/src/daemon/selector-capture-runtime.ts +++ b/src/daemon/selector-capture-runtime.ts @@ -273,7 +273,7 @@ function canUseSessionSnapshotCache( if (request.cache?.useSessionSnapshot !== true) return false; if (getActiveAndroidSnapshotFreshness(session)) return false; if (shouldBypassForPostGestureStabilization(session, request)) return false; - return true; + return session.snapshot?.unsettledGesture === undefined; } function isFreshSelectorSnapshot(snapshot: SnapshotState, timestamp: number): boolean { diff --git a/test/integration/ios-simulator-e2e-visibility-scroll.test.ts b/test/integration/ios-simulator-e2e-visibility-scroll.test.ts index 4824634682..fd79c11b4d 100644 --- a/test/integration/ios-simulator-e2e-visibility-scroll.test.ts +++ b/test/integration/ios-simulator-e2e-visibility-scroll.test.ts @@ -52,3 +52,45 @@ test('a stalled capture retries without scrolling or consuming an attempt', asyn assert.deepEqual(probeAttempts, [1, 1]); assert.deepEqual(scrollAttempts, []); }); + +/** + * The CI failure shape: the element is on screen only at offset 1, and the first read after each + * scroll lands on a surface still moving, so it misses with `unsettledGesture`. + */ +function listWithUnsettledFirstReads(visibleAt?: number) { + let offset = 0; + let moving = false; + const probes: number[] = []; + const scrolls: number[] = []; + const probe = async (attempt: number) => { + probes.push(attempt); + const unsettled = moving; + moving = false; + if (!unsettled && offset === visibleAt) return result(0); + return result(1, unsettled ? { unsettledGesture: { action: 'scroll', positionals: [] } } : {}); + }; + const scroll = async (attempt: number) => { + scrolls.push(attempt); + offset += 1; + moving = true; + }; + return { probes, scrolls, probe, scroll }; +} + +test('an unsettled miss after the scroll that reached the element is re-read at the same offset', async () => { + const list = listWithUnsettledFirstReads(1); + + await searchForVisibleElement('id="target"', list.probe, list.scroll); + + assert.deepEqual([list.probes, list.scrolls], [[1, 2, 2], [1]]); +}); + +test('a real absence still fails after the forward scrolls, naming every step', async () => { + const list = listWithUnsettledFirstReads(); + + await assert.rejects( + searchForVisibleElement('id="target"', list.probe, list.scroll), + /scroll after attempt 3: [\s\S]*probe 4:/, + ); + assert.deepEqual(list.scrolls, [1, 2, 3]); +}); diff --git a/test/integration/ios-simulator-e2e/live-assertions.ts b/test/integration/ios-simulator-e2e/live-assertions.ts index f01d909243..af9cdbf757 100644 --- a/test/integration/ios-simulator-e2e/live-assertions.ts +++ b/test/integration/ios-simulator-e2e/live-assertions.ts @@ -41,9 +41,10 @@ export function snapshotNodes(result: { json?: any }): LiveSnapshotNode[] { } const SCROLL_SEARCH_ATTEMPTS = 4; -// A stalled capture says nothing about where the element is, so it must not consume the scroll -// budget outright; a couple of retries absorb a slow runner without masking a real absence. -const SCROLL_SEARCH_STALL_RETRIES = 2; +// A stalled capture, or one taken while the last scroll was still moving, says nothing about where +// the element is, so re-reading it must not consume a scroll. A couple of re-reads per scroll absorb +// a slow runner without masking a real absence. +const SCROLL_SEARCH_REREADS = 2; export async function assertElementTextAfterScrolling( context: LiveContext, @@ -64,7 +65,7 @@ export async function assertElementTextAfterScrolling( 'scroll', 'down', '0.75', - ]).then(() => undefined), + ]).then((result) => result.json?.data), ); await assertElementText(context, selector, expected); } @@ -78,31 +79,31 @@ export async function assertElementTextAfterScrolling( export async function searchForVisibleElement( selector: string, probeVisibility: (attempt: number) => Promise, - scrollAfterAttempt: (attempt: number) => Promise, + scrollAfterAttempt: (attempt: number) => Promise, ): Promise { - let stallRetriesLeft = SCROLL_SEARCH_STALL_RETRIES; - let lastFailure: CliJsonResult | undefined; + let rereadsLeft = SCROLL_SEARCH_REREADS; + const history: string[] = []; for (let attempt = 1; attempt <= SCROLL_SEARCH_ATTEMPTS;) { const probe = await probeVisibility(attempt); + history.push(`probe ${attempt}: ${JSON.stringify(probe.json ?? { status: probe.status })}`); if (probe.status === 0) return; - lastFailure = probe; - // The snapshot never came back, so the surface was never read. Scrolling here would move the - // surface for a reason unrelated to visibility and spend an attempt on no evidence. - if (probe.json?.error?.details?.captureStalled === true && stallRetriesLeft > 0) { - stallRetriesLeft -= 1; + const details = probe.json?.error?.details; + const readNothing = details?.captureStalled === true || details?.unsettledGesture !== undefined; + if (readNothing && rereadsLeft > 0) { + rereadsLeft -= 1; continue; } attempt += 1; if (attempt <= SCROLL_SEARCH_ATTEMPTS) { - await scrollAfterAttempt(attempt - 1); + const scrolled = await scrollAfterAttempt(attempt - 1); + history.push(`scroll after attempt ${attempt - 1}: ${JSON.stringify(scrolled ?? null)}`); + rereadsLeft = SCROLL_SEARCH_REREADS; } } - assert.fail( - `${selector} did not become visible after scrolling\nlast visibility probe: ${JSON.stringify(lastFailure?.json ?? null)}`, - ); + assert.fail(`${selector} did not become visible after scrolling\n${history.join('\n')}`); } function requireNode( diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 8e1c4643f4..ad3175a063 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -557,7 +557,8 @@ agent-device is text 'id="greeting"' "Welcome back" - Supported predicates are `visible`, `hidden`, `exists`, `absent`, `editable`, `selected`, `focused`, and `text`. - `is visible` checks whether the resolved element is present in the current visible snapshot viewport. A node without its own rect still passes when a visible ancestor within the viewport provides the on-screen geometry. - `is exists` only checks whether the selector matches in the current snapshot. -- `is absent` passes only when the selector has zero matches in one readable, complete, unscoped, full-depth accessibility capture. It does not mean hidden; `--scope` and `--depth` are rejected, and sparse, unreadable, or truncated captures fail closed. +- `is absent` passes only when the selector has zero matches in one readable, complete, settled, unscoped, full-depth accessibility capture. It does not mean hidden; `--scope` and `--depth` are rejected, and sparse, unreadable, truncated, or unsettled captures fail closed. +- A read that answers from the first capture after a `scroll`, `swipe`, or `gesture swipe` waits for two consecutive captures to agree. When the surface is still changing when that budget runs out, `is`, `get`, `find`, and `wait` carry `unsettledGesture` (`{ "action", "positionals" }`) in `error.details` or `data` and append a warning, and `snapshot` appends the warning. A miss on that capture is not proof of absence: read again. `is absent` refuses it with `observation: "unsettled"`, and `wait absent` keeps polling. - `wait text` is a text-presence wait, not a hittability assertion. - Strict `wait absent` is not exported to Maestro's lenient `notVisible` condition; Maestro export reports it as unsupported unless an exact zero-candidate primitive becomes available. - `is text ` compares the resolved element text against the expected value. @@ -817,7 +818,7 @@ state the session app was found in and why the runner activated it: tripped before dispatching. - The disclosure rides capture-consuming commands — `snapshot`, `find`, `get`, `is`, `wait`, and an interaction whose target tree was captured for it — at every response level, including - `--level digest`. It is disclosed only for the command that paid for the repair: a read answered + `--level digest`. On a failure the typed fact is in `error.details` and the sentence in its hint. It is disclosed only for the command that paid for the repair: a read answered from a cached or stored tree did no device work and reports no repair of its own. - In text mode the CLI prints every response warning as a `Warning:` line after the command's own output, for every command — not only `snapshot`. Four commands declare their stdout to be the