diff --git a/CHANGELOG.md b/CHANGELOG.md index f84ddd4613..417d4ee122 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,11 +20,15 @@ - Fixed (mobile): a read taken right after a `scroll`, `swipe`, or `gesture swipe` no longer reports a definite miss when the surface never settled. When post-gesture stabilization ran out of budget on a surface still moving, `is visible` answered a plain `selector_not_found` and `is absent` - passed. That capture now carries `unsettledGesture`: `is`, `get`, `find`, and `wait` report it (in - `error.details` or `data`) with an appended warning, `snapshot` appends the warning, `is absent` - refuses with `observation: "unsettled"`, `wait absent` keeps polling, and the next read captures - afresh. Click, press, and fill by selector do not disclose it yet. A failed read now also carries - `targetActivation` in `error.details`, the same place as `unsettledGesture`. + passed. The capture now carries `postGestureOutcome` (`{ kind, gesture: { action, positionals } }`) + with `kind: "unsettled"`, and so does a re-capture taken at once to recover or widen it. A proven + no-effect gesture rides the same field with `kind: "no-effect"`; before, its warning reached only + `snapshot`. `is`, `get`, `find`, `wait`, and every interaction that captured it (`click`, `press`, + `fill`, and the other touch and gesture commands) report the field in `data` or `error.details` + with an appended warning; `snapshot` appends the warning. `is absent` refuses an unsettled capture + with `observation: "unsettled"`, `wait absent` keeps polling, and the next read captures afresh. + A failed read also carries `targetActivation` in `error.details`, and a failed interaction now + keeps the disclosure sentences in its hint. - Fixed (ios): `open` on a local Simulator now waits for the launched app's discovery before it decides whether the app is observable. On a loaded host `simctl spawn launchctl list` outlasts one 1.5 s discovery wait slice, and the launch observation read that slice as an unobservable app, so diff --git a/packages/capture-kit/src/post-gesture-stability.ts b/packages/capture-kit/src/post-gesture-stability.ts index 438a674bbb..5ce237f1ce 100644 --- a/packages/capture-kit/src/post-gesture-stability.ts +++ b/packages/capture-kit/src/post-gesture-stability.ts @@ -1,6 +1,6 @@ import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; import { sleep } from '@agent-device/host-kit/retry'; -import type { PostGestureAction } from '@agent-device/kernel/snapshot'; +import type { PostGestureAction, PostGestureOutcome } from '@agent-device/kernel/snapshot'; /** * Pure post-gesture stability mechanics: the quiet-window polling loop and the @@ -59,41 +59,37 @@ export type PostGestureStabilityHooks = { export type PostGestureStabilityOutcome = { value: T; - /** Present when the deadline expired while the last two captures still disagreed. */ - unsettledGesture?: PostGestureAction; /** - * Present ONLY when the accept-stale verdict is corroborated by full-surface - * evidence (`surfacesIdentical`). The bare verdict is NOT enough — it is - * subset-tolerant by design, and a successful scroll that replaced every - * list cell under fixed chrome still reads accept-stale (#1601 review P1). - * Callers surface this to the agent: a diagnostics-only signal let one - * benchmark run burn 40 calls re-issuing scrolls that moved nothing (#1600). + * `unsettled` when the deadline expired while the last two captures still disagreed. + * `no-effect` ONLY when the accept-stale verdict is corroborated by full-surface evidence + * (`surfacesIdentical`): the bare verdict is subset-tolerant by design, and a successful scroll + * that replaced every list cell under fixed chrome still reads accept-stale (#1601 review P1). + * Callers surface it to the agent: a diagnostics-only signal let one benchmark run burn 40 calls + * re-issuing scrolls that moved nothing (#1600). */ - gestureNoEffect?: PostGestureAction; + postGestureOutcome?: PostGestureOutcome; }; -function describePostGestureAction(gesture: PostGestureAction): string { - return [gesture.action, ...gesture.positionals].join(' ').trim(); -} - /** - * The agent-facing wording for a proven no-effect gesture. Names the exact - * gesture, admits the honest ambiguity (at-edge is a legitimate no-op the - * platform cannot distinguish), and hands over the one escape hatch that - * moved a stuck list when synthesized scrolls did not (#1600, element-18: - * raw `swipe` worked where scroll/fling/pan all silently no-opped). + * The agent-facing sentence for a post-gesture outcome, true whether the read that carries it found + * its target or not. A no-effect gesture admits the honest ambiguity (at-edge is a legitimate no-op + * the platform cannot distinguish) and hands over the escape hatch that moved a stuck list when + * synthesized scrolls did not (#1600: raw `swipe` worked where scroll/fling/pan all no-opped). */ -export function formatGestureNoEffectWarning(action: string, positionals: string[]): string { - return ( - `${describePostGestureAction({ action, positionals })} produced no visible change: the tree still matches its pre-gesture state. ` + - 'Either the container is already at its edge, or it ignores synthesized scrolls — ' + - 'a raw drag moves such lists: swipe x1 y1 x2 y2 (start inside the list).' - ); +export function formatPostGestureOutcomeWarning({ kind, gesture }: PostGestureOutcome): string { + const named = [gesture.action, ...gesture.positionals].join(' ').trim(); + return kind === 'unsettled' + ? `The surface was still changing after ${named} when this tree was read, so it may not match where the surface comes to rest: an element missing from it is not proof of absence.` + : `${named} produced no visible change: the tree still matches its pre-gesture state. ` + + 'Either the container is already at its edge, or it ignores synthesized scrolls — ' + + 'a raw drag moves such lists: swipe x1 y1 x2 y2 (start inside the list).'; } -/** A miss on a tree read while a gesture's surface was still changing is not proof of absence. */ -export function formatGestureUnsettledWarning(gesture: PostGestureAction): string { - return `The surface was still changing after ${describePostGestureAction(gesture)} when this was read, so an element missing from it may still be on screen. Read again before treating it as absent.`; +function postGestureOutcome( + kind: PostGestureOutcome['kind'], + pending: PostGestureAction, +): PostGestureOutcome { + return { kind, gesture: { action: pending.action, positionals: pending.positionals } }; } /** @@ -199,10 +195,7 @@ export async function runPostGestureStabilityLoop = { @@ -266,13 +259,7 @@ function buildAcceptedOutcome( baselineSignature !== undefined && hooks.surfacesIdentical(baselineSignature, current.signature) ) { - return { - value: current.value, - gestureNoEffect: { - action: pending.action, - positionals: pending.positionals, - }, - }; + return { value: current.value, postGestureOutcome: postGestureOutcome('no-effect', pending) }; } emitDiagnostic({ level: 'info', diff --git a/packages/kernel/src/snapshot.ts b/packages/kernel/src/snapshot.ts index ff57222c86..7c0bdfdf89 100644 --- a/packages/kernel/src/snapshot.ts +++ b/packages/kernel/src/snapshot.ts @@ -502,13 +502,34 @@ export type SnapshotState = { * the foreground instead (#2682). Consumers that surface this tree disclose the repair. */ targetActivation?: IosTargetActivation; - /** The gesture whose surface was still changing when stabilization gave up on this capture. */ - unsettledGesture?: PostGestureAction; + /** What post-gesture stabilization proved about the gesture before this capture. */ + postGestureOutcome?: PostGestureOutcome; } & SnapshotStateProvenance; /** The gesture a post-gesture outcome fact names: the command and its positionals. */ export type PostGestureAction = { action: string; positionals: string[] }; +/** + * `unsettled`: the surface was still changing when the stabilization deadline expired. + * `no-effect`: the settled surface still matches the pre-gesture tree (#1600). + */ +export type PostGestureOutcome = { + kind: 'unsettled' | 'no-effect'; + gesture: PostGestureAction; +}; + +/** + * A capture taken at once to recover or widen `previous` reads the same moment after the same + * gesture, so it carries that capture's outcome. + */ +export function inheritPostGestureOutcome( + previous: SnapshotState, + recapture: T, +): T { + recapture.postGestureOutcome ??= previous.postGestureOutcome; + return recapture; +} + export type SnapshotUnchanged = { ageMs: number; nodeCount: number; diff --git a/packages/selectors/src/absence-observation.ts b/packages/selectors/src/absence-observation.ts index 02964761ef..71067fa7e0 100644 --- a/packages/selectors/src/absence-observation.ts +++ b/packages/selectors/src/absence-observation.ts @@ -72,7 +72,7 @@ export function absenceCaptureOptionMessage( export function classifyAbsenceObservation( snapshot: Pick< SnapshotState, - 'backend' | 'nodes' | 'snapshotQuality' | 'truncated' | 'unsettledGesture' + 'backend' | 'nodes' | 'snapshotQuality' | 'truncated' | 'postGestureOutcome' >, matches: readonly SnapshotNode[], ): AbsenceObservation { @@ -100,7 +100,10 @@ export function classifyAbsenceObservation( }; } if (matchCount === 0) { - return { kind: snapshot.unsettledGesture ? 'unsettled' : 'absent', matches: 0 }; + return { + kind: snapshot.postGestureOutcome?.kind === 'unsettled' ? 'unsettled' : 'absent', + matches: 0, + }; } return { kind: 'present', matches: matchCount, firstMatch: firstMatch! }; } diff --git a/src/commands/capture/runtime/snapshot.test.ts b/src/commands/capture/runtime/snapshot.test.ts index d24f27a490..b5b908b7d9 100644 --- a/src/commands/capture/runtime/snapshot.test.ts +++ b/src/commands/capture/runtime/snapshot.test.ts @@ -13,7 +13,8 @@ import { type CommandSessionStore, } from '../../../runtime.ts'; import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; -import { formatGestureUnsettledWarning } from '@agent-device/capture-kit/post-gesture-stability'; +import type { PostGestureOutcome } from '@agent-device/kernel/snapshot'; +import { formatPostGestureOutcomeWarning } from '@agent-device/capture-kit/post-gesture-stability'; test('runtime snapshot captures nodes and updates the session baseline', async () => { let stored: Parameters[0] | undefined; @@ -789,17 +790,20 @@ test('runtime snapshot leaves the keyboard band off when the backend measured no }); test('runtime snapshot warns when its tree was read on a surface still moving after a gesture', async () => { - const gesture = { action: 'scroll', positionals: ['down'] }; + const outcome: PostGestureOutcome = { + kind: 'unsettled', + gesture: { action: 'scroll', positionals: ['down'] }, + }; const device = createSnapshotOnlyDevice({ snapshot: { ...makeSnapshotState([{ index: 0, depth: 0, type: 'Window', label: 'Home' }], { backend: 'xctest', }), - unsettledGesture: gesture, + postGestureOutcome: outcome, }, }); const result = await device.capture.snapshot({ session: 'default' }); - assert.deepEqual(result.warnings, [formatGestureUnsettledWarning(gesture)]); + assert.deepEqual(result.warnings, [formatPostGestureOutcomeWarning(outcome)]); }); diff --git a/src/commands/capture/runtime/snapshot.ts b/src/commands/capture/runtime/snapshot.ts index 21f058fe53..c0d9cd9ca7 100644 --- a/src/commands/capture/runtime/snapshot.ts +++ b/src/commands/capture/runtime/snapshot.ts @@ -26,7 +26,7 @@ import { renderSnapshotQualityWarnings, truncatedCaptureWarning, } from '@agent-device/capture-kit/quality-warnings'; -import { formatGestureUnsettledWarning } from '@agent-device/capture-kit/post-gesture-stability'; +import { formatPostGestureOutcomeWarning } from '@agent-device/capture-kit/post-gesture-stability'; import { buildSnapshotVisibility } from '@agent-device/capture-kit/snapshot-visibility'; import { ANDROID_SYSTEM_SURFACE_DISCLOSURE } from '@agent-device/contracts/android-system-surface-disclosure'; import { formatReactNativeOverlayWarning } from '../../react-native/overlay.ts'; @@ -261,8 +261,8 @@ function buildSnapshotWarnings(params: { ); } warnings.push(...truncatedCaptureWarning(snapshotTruncationForResult(params.snapshot))); - if (params.snapshot.unsettledGesture) { - warnings.push(formatGestureUnsettledWarning(params.snapshot.unsettledGesture)); + if (params.snapshot.postGestureOutcome) { + warnings.push(formatPostGestureOutcomeWarning(params.snapshot.postGestureOutcome)); } warnings.push(...buildEmptyAndroidInteractiveWarnings(params)); if (!params.annotations.quality) { diff --git a/src/commands/interaction/runtime/resolution.ts b/src/commands/interaction/runtime/resolution.ts index 48696a74b4..c85b5a7420 100644 --- a/src/commands/interaction/runtime/resolution.ts +++ b/src/commands/interaction/runtime/resolution.ts @@ -5,7 +5,11 @@ import type { SnapshotNode, SnapshotState, } from '@agent-device/kernel/snapshot'; -import { findNodeByRef, normalizeRef } from '@agent-device/kernel/snapshot'; +import { + findNodeByRef, + inheritPostGestureOutcome, + normalizeRef, +} from '@agent-device/kernel/snapshot'; import { resolveRectCenter } from '@agent-device/kernel/rect-center'; import type { AgentDeviceRuntime, @@ -390,7 +394,9 @@ async function resolveSelectorInteractionTarget( params.pipeline, ); if ((!resolved || !resolved.node.rect) && params.requireInteractive) { + const interactive = capture.snapshot; capture = await captureInteractionSnapshot(runtime, options, false); + inheritPostGestureOutcome(interactive, capture.snapshot); resolved = resolveActionSelector( capture.snapshot.nodes, selectorExpression, diff --git a/src/commands/interaction/runtime/wait-absent.test.ts b/src/commands/interaction/runtime/wait-absent.test.ts index 2c5bf6f35f..88d69c0f27 100644 --- a/src/commands/interaction/runtime/wait-absent.test.ts +++ b/src/commands/interaction/runtime/wait-absent.test.ts @@ -145,7 +145,10 @@ test('wait absent does not take a miss on a surface still moving after a gesture const unsettled = { snapshot: { ...makeSnapshotState([]), - unsettledGesture: { action: 'scroll', positionals: ['down'] }, + postGestureOutcome: { + kind: 'unsettled' as const, + gesture: { action: 'scroll', positionals: ['down'] }, + }, }, }; const device = absentDevice([unsettled, snapshot('Removed')]); diff --git a/src/daemon/__tests__/capture-disclosure-target-activation.test.ts b/src/daemon/__tests__/capture-disclosure-target-activation.test.ts index b381114465..570d4d916f 100644 --- a/src/daemon/__tests__/capture-disclosure-target-activation.test.ts +++ b/src/daemon/__tests__/capture-disclosure-target-activation.test.ts @@ -54,7 +54,7 @@ test('a failure response appends the repair to its hint and keeps the original d assert.equal(response.error.message, 'selector missed'); assert.equal(response.error.code, 'COMMAND_FAILED'); assert.equal(response.error.details?.blockedBy, 'android_foreground_surface'); - assert.match(String(response.error.details?.hint), /prior state runningBackground/); + assert.match(String(response.error.hint), /prior state runningBackground/); }); test('a capture with no repair leaves the response byte-identical', () => { @@ -64,7 +64,7 @@ test('a capture with no repair leaves the response byte-identical', () => { }); /** - * A failure carries the sentence in `error.details.hint`. That carrier is exactly where a route could + * A failure carries the sentence in `error.hint`. That carrier is exactly where a route could * cheaply borrow the previous command's repair off the stored snapshot and blame it on this request, * so the repair travels only on the request's own proof (#2682). */ @@ -77,7 +77,7 @@ test('a failure does not borrow a repair the stored snapshot happens to carry', const response = withCaptureDisclosures({ response: failed, consumedTree: { targetActivation: FACT }, - activationProof: {}, + captureProof: {}, }); assert.equal(response, failed); @@ -95,7 +95,7 @@ test('surface and foreground disclosures ride one response together', () => { iosSystemSurfaceBundleId: 'com.apple.SafariViewService', targetActivation: FACT, }, - activationProof: { state: { targetActivation: FACT } }, + captureProof: { targetActivation: FACT }, }); const data = dataOf(response); assert.match(String(data.warning), /system web sign-in sheet/); @@ -117,22 +117,22 @@ test('a repair that passes through two wrappers is named once in the failure hin details: { hint: 'Use snapshot to see the current tree.' }, }, }; - const proof = { state: { targetActivation: FACT } }; + const proof = { targetActivation: FACT }; const once = withCaptureDisclosures({ response: missed, consumedTree: { targetActivation: FACT }, - activationProof: proof, + captureProof: proof, }); const twice = withCaptureDisclosures({ response: once, consumedTree: { targetActivation: FACT }, - activationProof: proof, + captureProof: proof, }); assert.equal(twice.ok, false); if (twice.ok) return; - const hint = String(twice.error.details?.hint); + const hint = String(twice.error.hint); assert.equal( hint.split(iosTargetActivationDisclosure(FACT)).length - 1, 1, diff --git a/src/daemon/__tests__/capture-disclosure.test.ts b/src/daemon/__tests__/capture-disclosure.test.ts index a7a52de12f..e848ded858 100644 --- a/src/daemon/__tests__/capture-disclosure.test.ts +++ b/src/daemon/__tests__/capture-disclosure.test.ts @@ -148,7 +148,7 @@ test('wait timeout for app text hidden behind a system surface discloses the occ expect(response.ok).toBe(false); if (response.ok) return; expect(response.error.message).toMatch(/wait timed out for text: Bakery list/); - expect(String(response.error.details?.hint)).toContain(ANDROID_SYSTEM_SURFACE_DISCLOSURE); + expect(String(response.error.hint)).toContain(ANDROID_SYSTEM_SURFACE_DISCLOSURE); }); test('sessionless read-only find still discloses the occluding system surface', async () => { @@ -204,7 +204,7 @@ test('disclosure appends after an existing failure hint instead of replacing it' ); expect(response.ok).toBe(false); if (response.ok) return; - const hint = String(response.error.details?.hint); + const hint = String(response.error.hint); expect(hint).toContain('prior hint text'); expect(hint).toContain(ANDROID_SYSTEM_SURFACE_DISCLOSURE); expect(hint.indexOf('prior hint text')).toBeLessThan( @@ -261,7 +261,7 @@ test('sessionless wait timeout still discloses the occluding system surface', as expect(response.ok).toBe(false); if (response.ok) return; expect(sessionStore.get('default')).toBeUndefined(); - expect(String(response.error.details?.hint)).toContain(ANDROID_SYSTEM_SURFACE_DISCLOSURE); + expect(String(response.error.hint)).toContain(ANDROID_SYSTEM_SURFACE_DISCLOSURE); }); // --- #2438: an in-place iOS system surface (web sign-in sheet) discloses on the same shared seam --- @@ -350,7 +350,7 @@ test('mutating find that misses on an in-place system surface still discloses it expect(response?.ok).toBe(false); if (response?.ok) return; - expect(String(response?.error.details?.hint)).toContain(WEB_SIGN_IN_DISCLOSURE); + expect(String(response?.error.hint)).toContain(WEB_SIGN_IN_DISCLOSURE); }); test('the shared disclosure helper reports an iOS system surface on both outcomes', () => { @@ -368,13 +368,13 @@ test('the shared disclosure helper reports an iOS system surface on both outcome ); expect(failed.ok).toBe(false); if (failed.ok) return; - expect(String(failed.error.details?.hint)).toContain(WEB_SIGN_IN_DISCLOSURE); + expect(String(failed.error.hint)).toContain(WEB_SIGN_IN_DISCLOSURE); }); /** * A timed-out `wait text` polled the device and the runner had to re-activate the session app to * answer those polls (#2682). The disclosure arrives on the failure the same way the occlusion one - * does: in `error.details.hint`. + * does: in `error.hint`. */ test('wait timeout whose polls required a foreground repair discloses the repair', async () => { const sessionStore = makeSessionStore(); @@ -412,7 +412,7 @@ test('wait timeout whose polls required a foreground repair discloses the repair expect(response.ok).toBe(false); if (response.ok) return; - expect(String(response.error.details?.hint)).toContain( + expect(String(response.error.hint)).toContain( iosTargetActivationDisclosure(TARGET_ACTIVATION_FACT), ); }); diff --git a/src/daemon/__tests__/deferred-interaction-outcome.test.ts b/src/daemon/__tests__/deferred-interaction-outcome.test.ts index 1dbc8374c7..e112fe4a62 100644 --- a/src/daemon/__tests__/deferred-interaction-outcome.test.ts +++ b/src/daemon/__tests__/deferred-interaction-outcome.test.ts @@ -280,7 +280,7 @@ test('a pending stabilization resolves through the quiet-window loop and clears assert.equal(result?.warnings, undefined); }); -test('a proven no-effect gesture surfaces its warning on the resolved capture (iOS accept-stale)', async () => { +test('a proven no-effect gesture is stamped on the resolved capture tree (iOS accept-stale)', async () => { vi.useFakeTimers(); const session = makeSession('ios'); session.snapshot = pickupSnapshot(); @@ -299,8 +299,10 @@ test('a proven no-effect gesture surfaces its warning on the resolved capture (i } const result = await pendingResult; - assert.equal(result?.warnings?.length, 1); - assert.match(result?.warnings?.[0] ?? '', /produced no visible change/); + assert.deepEqual(result?.snapshot.postGestureOutcome, { + kind: 'no-effect', + gesture: { action: 'scroll', positionals: ['down'] }, + }); assert.equal(isPostGestureStabilizationPending(session), false); }); diff --git a/src/daemon/__tests__/is-runtime.test.ts b/src/daemon/__tests__/is-runtime.test.ts index 722948d9ab..32c84ba5b3 100644 --- a/src/daemon/__tests__/is-runtime.test.ts +++ b/src/daemon/__tests__/is-runtime.test.ts @@ -10,10 +10,11 @@ import { import { makeSessionStore } from '../../__tests__/test-utils/store-factory.ts'; import { withTestDeviceInventory } from '../../__tests__/test-utils/device-inventory-gateways.ts'; import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; +import type { PostGestureOutcome } from '@agent-device/kernel/snapshot'; import type { DaemonRequest } from '../daemon-request.ts'; import { selectorCaptureFixture } from './selector-capture-fixture.ts'; import { markDeferredInteractionOutcome } from '../deferred-interaction-outcome.ts'; -import { formatGestureUnsettledWarning } from '@agent-device/capture-kit/post-gesture-stability'; +import { formatPostGestureOutcomeWarning } from '@agent-device/capture-kit/post-gesture-stability'; const { mockRunAppleRunnerCommand } = vi.hoisted(() => ({ mockRunAppleRunnerCommand: vi.fn() })); @@ -455,16 +456,59 @@ test('a miss on a surface that never settled carries the unsettled fact, and the const pending = isVisible(); // Just past the 1.5s stabilization deadline, so the re-read lands inside the cache window. await vi.advanceTimersByTimeAsync(1_700); - const gesture = { action: 'scroll', positionals: [] }; + const outcome: PostGestureOutcome = { + kind: 'unsettled', + gesture: { action: 'scroll', positionals: [] }, + }; const response = await pending; - expect(response?.ok === false && response.error.details).toMatchObject({ - reason: 'selector_not_found', - unsettledGesture: gesture, - hint: expect.stringContaining(formatGestureUnsettledWarning(gesture)), + expect(response?.ok === false && response.error).toMatchObject({ + hint: expect.stringContaining(formatPostGestureOutcomeWarning(outcome)), + details: { reason: 'selector_not_found', postGestureOutcome: outcome }, }); const captures = fixture.captures.length; const reread = await isVisible(); expect(fixture.captures.length).toBe(captures + 1); - expect(reread?.ok === false && reread.error.details?.unsettledGesture).toBeUndefined(); + expect(reread?.ok === false && reread.error.details?.postGestureOutcome).toBeUndefined(); +}); + +test('a read after a scroll that moved nothing carries the no-effect outcome, and the re-read reuses its settled tree', async () => { + vi.useFakeTimers(); + const row = { + index: 0, + type: 'Cell', + identifier: 'row', + rect: { x: 0, y: 200, width: 390, height: 60 }, + }; + const fixture = selectorCaptureFixture({ + snapshot: () => ({ nodes: [row], backend: 'xctest', producer: 'apple-runner' }), + }); + const sessionStore = makeSessionStore(); + const session = makeIosAppSession('is-no-effect', { snapshot: makeSnapshotState([row]) }); + markDeferredInteractionOutcome({ session, command: 'scroll', positionals: ['down'], flags: {} }); + sessionStore.set('is-no-effect', session); + const isVisible = () => + dispatchIsViaRuntime({ + req: isRequest('is-no-effect', ['visible', 'id=row']), + sessionName: 'is-no-effect', + sessionStore, + inspectFacts: fixture.inspectFacts, + bindDevice: fixture.bindDevice, + }); + const pending = isVisible(); + // A tree that still matches its pre-gesture baseline is distrusted up to the 3.5s cap. + await vi.advanceTimersByTimeAsync(3_700); + const outcome: PostGestureOutcome = { + kind: 'no-effect', + gesture: { action: 'scroll', positionals: ['down'] }, + }; + + const response = await pending; + expect(response?.ok && response.data).toMatchObject({ + postGestureOutcome: outcome, + warnings: [formatPostGestureOutcomeWarning(outcome)], + }); + const captures = fixture.captures.length; + await isVisible(); + expect(fixture.captures.length).toBe(captures); }); diff --git a/src/daemon/__tests__/post-gesture-no-effect-claim.test.ts b/src/daemon/__tests__/post-gesture-no-effect-claim.test.ts index bfe0ddca12..d07512cd9e 100644 --- a/src/daemon/__tests__/post-gesture-no-effect-claim.test.ts +++ b/src/daemon/__tests__/post-gesture-no-effect-claim.test.ts @@ -17,7 +17,8 @@ import { capturePostGestureStabilizedResult, markDeferredInteractionOutcome, } from '../deferred-interaction-outcome.ts'; -import { formatGestureNoEffectWarning } from '@agent-device/capture-kit/post-gesture-stability'; +import { formatPostGestureOutcomeWarning } from '@agent-device/capture-kit/post-gesture-stability'; +import type { PostGestureAction } from '@agent-device/kernel/snapshot'; import type { SessionState } from '../session-state.ts'; import { chromeWithListSnapshot, @@ -25,7 +26,10 @@ import { pickupSnapshot, } from './post-gesture-stabilization-fixtures.ts'; -// When the agent-facing gestureNoEffect claim may and may not surface — split +const noEffectWarning = (gesture: PostGestureAction) => + formatPostGestureOutcomeWarning({ kind: 'no-effect', gesture }); + +// When the agent-facing no-effect claim may and may not surface — split // by subject from post-gesture-stabilization.test.ts (the capture loop), per // #1563. Loop mechanics (rebase, distrust budget, timeouts) stay in the // loop file; everything here is about the claim and its veto instrumentation. @@ -129,7 +133,7 @@ test('a backend flip mid-poll withholds the no-effect claim, and records the rea assert.equal(rebased, 1); assert.equal(staleAccepts, 1, 'the loop still accepts the stale read after the distrust budget'); - assert.equal(result.gestureNoEffect, undefined); + assert.equal(result.postGestureOutcome, undefined); assert.equal(vetoed, 1, 'the withheld claim must be observable'); // The reason is the point: a rebase means the corroboration pair is // cross-backend, which is a categorically different answer from "the @@ -165,7 +169,7 @@ test('a successful scroll that flips the capture backend must not claim no-effec assert.equal(rebased, 1); assert.equal( - result.gestureNoEffect, + result.postGestureOutcome, undefined, 'the scroll swapped every list cell — a no-effect claim here is a false positive', ); @@ -204,7 +208,7 @@ test('no pre-gesture snapshot means no baseline, no rebase, and no no-effect cla assert.equal(rebased, 0); assert.equal(vetoed, 0); assert.equal( - result.gestureNoEffect, + result.postGestureOutcome, undefined, 'a no-effect claim needs a real pre-gesture baseline, never one the loop invented for itself', ); @@ -234,7 +238,7 @@ test('scope drift accepts stale but is vetoed from claiming no-effect, observabl ); assert.equal(staleAccepts, 1); - assert.equal(result.gestureNoEffect, undefined); + assert.equal(result.postGestureOutcome, undefined); assert.equal(vetoed, 1); // Scope drift reads as one-sided membership, never as movement: both cells // are missing from the narrowed capture, the shared chrome button has not @@ -292,7 +296,7 @@ test('same-backend membership drift vetoes the claim and records the divergence assert.equal(rebased, 0, 'same backend throughout: this is drift, not a flip'); assert.equal(staleAccepts, 1); - assert.equal(result.gestureNoEffect, undefined); + assert.equal(result.postGestureOutcome, undefined); assert.equal(vetoed, 1); // The counts are what distinguish this from the scope-drift case above: // one extra key on the CURRENT side, everything shared unmoved. Same veto, @@ -359,22 +363,31 @@ test('summarizeDiscriminatingSurfaceDivergence counts one-sided keys and moved r }); }); -test('formatGestureNoEffectWarning names the gesture and the raw-drag escape hatch', () => { +test('the no-effect warning names the gesture and the raw-drag escape hatch', () => { // Positionals echo verbatim: the warning names the gesture the agent issued, // and `scroll down 1` is what they issued. - const scrollWarning = formatGestureNoEffectWarning('scroll', ['down', '1']); + const scrollWarning = noEffectWarning({ + action: 'scroll', + positionals: ['down', '1'], + }); assert.match(scrollWarning, /scroll down 1 produced no visible change/); assert.match(scrollWarning, /swipe x1 y1 x2 y2/); assert.match(scrollWarning, /already at its edge/); - const gestureWarning = formatGestureNoEffectWarning('gesture', ['swipe', 'left']); + const gestureWarning = noEffectWarning({ + action: 'gesture', + positionals: ['swipe', 'left'], + }); assert.match(gestureWarning, /gesture swipe left produced no visible change/); - const bareWarning = formatGestureNoEffectWarning('swipe', []); + const bareWarning = noEffectWarning({ action: 'swipe', positionals: [] }); assert.match(bareWarning, /swipe produced no visible change/); // The regression the deleted heuristic caused: every positional of a swipe is // a coordinate, so "drop anything numeric-looking" left a contentless "swipe". - const swipeWarning = formatGestureNoEffectWarning('swipe', ['10', '20', '30', '40']); + const swipeWarning = noEffectWarning({ + action: 'swipe', + positionals: ['10', '20', '30', '40'], + }); assert.match(swipeWarning, /^swipe 10 20 30 40 produced no visible change/); }); diff --git a/src/daemon/__tests__/post-gesture-stabilization.test.ts b/src/daemon/__tests__/post-gesture-stabilization.test.ts index d8c99751f8..48ee133df3 100644 --- a/src/daemon/__tests__/post-gesture-stabilization.test.ts +++ b/src/daemon/__tests__/post-gesture-stabilization.test.ts @@ -176,7 +176,8 @@ test('capturePostGestureStabilizedResult keeps polling past the normal deadline assert.equal(session.postGestureStabilization, undefined); // #1600: a stale-accept is the daemon PROVING the gesture moved nothing — // that verdict must reach the caller, not only the diagnostics stream. - assert.equal(result.gestureNoEffect?.action, 'scroll'); + assert.equal(result.postGestureOutcome?.kind, 'no-effect'); + assert.equal(result.postGestureOutcome?.gesture.action, 'scroll'); // Proves it kept polling well past the OLD 1.5s accept point (2 attempts, // ~200ms) instead of trusting the first quiet match. assert.ok(captureCount > 8, `expected sustained polling, saw ${captureCount} captures`); @@ -217,7 +218,7 @@ test('a replaced list under fixed chrome now settles outright, and still claims const { result, staleAccepts } = await resultPromise; assert.equal(staleAccepts, 0); - assert.equal(result.gestureNoEffect, undefined); + assert.equal(result.postGestureOutcome, undefined); }); test('capturePostGestureStabilizedResult trusts a quiet signature once content genuinely differs from the baseline (iOS)', async () => { @@ -247,7 +248,7 @@ test('capturePostGestureStabilizedResult trusts a quiet signature once content g assert.equal(settled, 1); assert.equal(staleAccepts, 0); // A genuine settle carries no no-effect claim. - assert.equal(result.gestureNoEffect, undefined); + assert.equal(result.postGestureOutcome, undefined); // Accepted at the first quiet match (initial capture + one poll = 2 // attempts): no distrust cost for a genuine settle. assert.equal(capture.mock.calls.length, 2); @@ -364,7 +365,7 @@ test('a deadline that expires right after a rebased quiet pair is not reported a const { result, rebased, timeouts } = await resultPromise; assert.deepEqual([rebased, timeouts], [1, 1]); - assert.equal(result.unsettledGesture, undefined); + assert.equal(result.postGestureOutcome, undefined); }); test('capturePostGestureStabilizedResult catches a frozen target even when the baseline came from a broader-scope capture than the post-gesture reads (iOS, live regression)', async () => { diff --git a/src/daemon/__tests__/selector-capture-runtime.test.ts b/src/daemon/__tests__/selector-capture-runtime.test.ts index 64cf3655e8..9384dc7553 100644 --- a/src/daemon/__tests__/selector-capture-runtime.test.ts +++ b/src/daemon/__tests__/selector-capture-runtime.test.ts @@ -9,7 +9,7 @@ import { type SnapshotState, } from '@agent-device/kernel/snapshot'; import type { DaemonResponse } from '../daemon-request.ts'; -import { type RequestActivationProof, withCaptureDisclosures } from '../capture-disclosure.ts'; +import { type RequestCaptureProof, withCaptureDisclosures } from '../capture-disclosure.ts'; import { makeIosSession } from '../../__tests__/test-utils/session-factories.ts'; import { makeSessionStore } from '../../__tests__/test-utils/store-factory.ts'; import { createSelectorCaptureRuntime } from '../selector-capture-runtime.ts'; @@ -223,14 +223,14 @@ function proofRuntime(params: { : {}), } as never); const consumedSnapshot: { state?: SnapshotState } = {}; - const activationProof: RequestActivationProof = {}; + const captureProof: RequestCaptureProof = {}; const runtime = createSelectorCaptureRuntime({ device: session.device, session, sessionStore, sessionName: params.sessionName, consumedSnapshot, - activationProof, + captureProof, capture: boundCapture, req: { token: 't', @@ -240,7 +240,7 @@ function proofRuntime(params: { flags: {}, }, }); - return { runtime, consumedSnapshot, activationProof }; + return { runtime, consumedSnapshot, captureProof }; } /** @@ -264,12 +264,12 @@ test('a session-snapshot cache hit consumes a repaired tree without earning the expect(boundCapture).not.toHaveBeenCalled(); expect(holders.consumedSnapshot.state?.targetActivation).toEqual(REPAIR); - expect(holders.activationProof.state).toBeUndefined(); + expect(holders.captureProof.targetActivation).toBeUndefined(); const response = withCaptureDisclosures({ response: { ok: true, data: { nodes: [] } } as DaemonResponse, consumedTree: holders.consumedSnapshot.state, - activationProof: holders.activationProof, + captureProof: holders.captureProof, }); expect(response.ok).toBe(true); if (response.ok) { @@ -287,7 +287,7 @@ test('a capture the request took itself earns the repair proof', async () => { await holders.runtime.capture({ flags: {}, cache: { useSessionSnapshot: true } }); expect(boundCapture).toHaveBeenCalledTimes(1); - expect(holders.activationProof.state?.targetActivation).toEqual(REPAIR); + expect(holders.captureProof.targetActivation).toEqual(REPAIR); }); /** @@ -310,7 +310,7 @@ test('a later fact-less capture does not erase an earlier repair proof', async ( await holders.runtime.capture({ flags: {}, cache: { forceFresh: true } }); expect(boundCapture).toHaveBeenCalledTimes(2); - expect(holders.activationProof.state?.targetActivation).toEqual(REPAIR); + expect(holders.captureProof.targetActivation).toEqual(REPAIR); }); function makeCaptureRuntime(sessionName: string) { diff --git a/src/daemon/__tests__/snapshot-runtime-target-activation.test.ts b/src/daemon/__tests__/snapshot-runtime-disclosure.test.ts similarity index 71% rename from src/daemon/__tests__/snapshot-runtime-target-activation.test.ts rename to src/daemon/__tests__/snapshot-runtime-disclosure.test.ts index ee199d77ed..75e59093d1 100644 --- a/src/daemon/__tests__/snapshot-runtime-target-activation.test.ts +++ b/src/daemon/__tests__/snapshot-runtime-disclosure.test.ts @@ -1,7 +1,10 @@ import path from 'node:path'; -import { beforeEach, expect, test, vi } from 'vitest'; +import { afterEach, beforeEach, expect, test, vi } from 'vitest'; import { iosTargetActivationDisclosure } from '@agent-device/contracts/ios-target-activation'; -import type { IosTargetActivation } from '@agent-device/kernel/snapshot'; +import type { IosTargetActivation, PostGestureOutcome } from '@agent-device/kernel/snapshot'; +import { formatPostGestureOutcomeWarning } from '@agent-device/capture-kit/post-gesture-stability'; +import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; +import { markDeferredInteractionOutcome } from '../deferred-interaction-outcome.ts'; import { makeIosSession } from '../../__tests__/test-utils/session-factories.ts'; import { mkdtempForTestSync } from '../../__tests__/test-utils/tmp-dir.ts'; import { SessionStore } from '../session-store.ts'; @@ -24,6 +27,10 @@ beforeEach(() => { legacyDispatchCapture.mockReset(); }); +afterEach(() => { + vi.useRealTimers(); +}); + test('a snapshot that captured a repaired tree discloses the repair it paid for', async () => { const input = scenario({}); legacyDispatchCapture.mockResolvedValue({ @@ -95,13 +102,50 @@ test('a snapshot refused before it captured reports no repair and no surface it expect(response.ok).toBe(false); if (response.ok) return; - const hint = String(response.error.details?.hint ?? ''); + const hint = String(response.error.hint ?? ''); expect(hint.includes(iosTargetActivationDisclosure(REPAIR))).toBe(false); expect(hint.includes('was not foreground')).toBe(false); expect(hint.includes('system web sign-in sheet')).toBe(false); expect(JSON.stringify(response.error.details ?? {})).not.toContain('targetActivation'); }); +/** A scroll that moved nothing reaches the agent on the snapshot that proved it (#1600). */ +test('a snapshot after a scroll that moved nothing warns and stamps the no-effect outcome', async () => { + const input = scenario({}); + const button = { + index: 0, + depth: 0, + type: 'Button', + label: 'Continue', + rect: { x: 0, y: 0, width: 100, height: 44 }, + hittable: true, + }; + const session = input.sessionStore.get(input.sessionName)!; + session.snapshot = makeSnapshotState([button], { backend: 'xctest' }); + markDeferredInteractionOutcome({ session, command: 'scroll', positionals: ['up'], flags: {} }); + legacyDispatchCapture.mockResolvedValue({ backend: 'xctest', truncated: false, nodes: [button] }); + const outcome: PostGestureOutcome = { + kind: 'no-effect', + gesture: { action: 'scroll', positionals: ['up'] }, + }; + + const realSetTimeout = globalThis.setTimeout; + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] }); + let done = false; + const pending = dispatchSnapshot(input).finally(() => (done = true)); + // The route awaits real I/O between polls, so the faked clock advances while the test yields. + while (!done) { + await vi.advanceTimersByTimeAsync(100); + await new Promise((resolve) => realSetTimeout(resolve, 1)); + } + const response = await pending; + + expect(response.ok && response.data?.warnings).toContain( + formatPostGestureOutcomeWarning(outcome), + ); + expect(input.sessionStore.get(input.sessionName)?.snapshot?.postGestureOutcome).toEqual(outcome); +}); + function scenario(params: { storedRepair?: boolean }) { const root = mkdtempForTestSync('agent-device-snapshot-target-activation'); const sessionName = 'default'; diff --git a/src/daemon/capture-disclosure.ts b/src/daemon/capture-disclosure.ts index 1e594f1613..5a6ac46f6d 100644 --- a/src/daemon/capture-disclosure.ts +++ b/src/daemon/capture-disclosure.ts @@ -2,18 +2,19 @@ import type { SnapshotState } from '@agent-device/kernel/snapshot'; import { systemSurfaceDisclosure } from '@agent-device/contracts/android-system-surface-disclosure'; import { iosSystemSurfaceDisclosure } from '@agent-device/contracts/ios-system-surface'; import { iosTargetActivationDisclosure } from '@agent-device/contracts/ios-target-activation'; -import { formatGestureUnsettledWarning } from '@agent-device/capture-kit/post-gesture-stability'; +import { formatPostGestureOutcomeWarning } from '@agent-device/capture-kit/post-gesture-stability'; import type { DaemonResponse } from './daemon-request.ts'; -/** The capture provenance a response must be disclosed against (#2438, #2682, unsettled gestures). */ +/** The capture provenance a response must be disclosed against (#2438, #2682, gesture outcomes). */ export type CaptureProvenance = Pick< SnapshotState, - 'systemSurfaceOnly' | 'iosSystemSurfaceBundleId' | 'targetActivation' | 'unsettledGesture' + 'systemSurfaceOnly' | 'iosSystemSurfaceBundleId' | 'targetActivation' | 'postGestureOutcome' >; /** - * The foreground repair THIS request paid for, filled by the capture path only when the request - * actually captured a tree (#2682). + * The whole-tree facts THIS request's own captures observed, filled by the capture path only when the + * request actually captured a tree: the foreground repair it paid for (#2682) and the outcome of the + * gesture it read after. * * Separate from the consumed tree on purpose. A selector read may answer from a cached or stored * tree — that tree still describes the surface the response is about, which is what #2438 discloses @@ -21,23 +22,23 @@ export type CaptureProvenance = Pick< * consumed tree would tell a command "you found the session app out of foreground" when it never * looked, which is a fabricated observation rather than a disclosure. */ -export type RequestActivationProof = { - state?: CaptureProvenance; -}; +export type RequestCaptureProof = Pick< + CaptureProvenance, + 'targetActivation' | 'postGestureOutcome' +>; /** - * Note the repair a capture paid for, and hand that capture back. First fact wins: a later capture in - * the same request that reports no repair — a sparse recovery's fresh tree, a poll's fact-less read — - * cannot erase the capture that did (#2682). One rule, because three capture paths owe it and a - * hand-copied condition drifts from the other two the moment one of them learns something. + * Note the facts a capture observed, and hand that capture back. First fact wins: a later capture in + * the same request that reports none — a sparse recovery's fresh tree, a poll's fact-less read, a + * post-action observation — cannot erase the capture that did (#2682). */ -export function recordActivationProof( - proof: RequestActivationProof | undefined, +export function recordCaptureProof( + proof: RequestCaptureProof | undefined, snapshot: T, ): T { - if (proof !== undefined && proof.state === undefined && snapshot.targetActivation !== undefined) { - proof.state = snapshot; - } + if (proof === undefined) return snapshot; + proof.targetActivation ??= snapshot.targetActivation; + proof.postGestureOutcome ??= snapshot.postGestureOutcome; return snapshot; } @@ -59,15 +60,15 @@ export function withSystemSurfaceDisclosure( } /** - * Disclose a fact about the whole answered tree (#2682 foreground repair, an unsettled gesture). Its - * sentence is APPENDED, never replacing an earlier warning, and the typed fact lands on either - * outcome (`data` or `error.details`) even when the sentence was already carried. + * Disclose a fact about the whole answered tree. Its sentence is APPENDED, never replacing an earlier + * warning, and the typed fact lands on either outcome (`data` or `error.details`) even when the + * sentence was already carried. */ -function withTreeFactDisclosure( +function withTreeFactDisclosure( response: DaemonResponse, key: K, - fact: CaptureProvenance[K], - sentence: (fact: NonNullable) => string, + fact: RequestCaptureProof[K], + sentence: (fact: NonNullable) => string, ): DaemonResponse { if (!fact) return response; const disclosed = appendDisclosure(response, sentence(fact), 'warnings'); @@ -93,31 +94,32 @@ export function withTargetActivationDisclosure( /** * Every capture-provenance disclosure a response owes, from the two different things a capture can - * prove: what the answered tree describes (#2438 — cache tiers included, because the surface is - * still on screen) and what this request's own capture found (#2682 — cache hits excluded, because - * a request that captured nothing repaired nothing). + * prove: what the answered tree describes (#2438 and the post-gesture outcome — cache tiers + * included, because the tree still describes the surface) and what this request's own capture found + * (#2682 — cache hits excluded, because a request that captured nothing repaired nothing). A route + * that aims at the trees its own captures read passes its proof as the consumed tree. */ export function withCaptureDisclosures(params: { response: DaemonResponse; consumedTree: CaptureProvenance | undefined; - activationProof?: RequestActivationProof; + captureProof?: RequestCaptureProof; }): DaemonResponse { - const { response, consumedTree, activationProof } = params; + const { response, consumedTree, captureProof } = params; return withTargetActivationDisclosure( withTreeFactDisclosure( withSystemSurfaceDisclosure(response, consumedTree), - 'unsettledGesture', - consumedTree?.unsettledGesture, - formatGestureUnsettledWarning, + 'postGestureOutcome', + consumedTree?.postGestureOutcome, + formatPostGestureOutcomeWarning, ), - activationProof?.state, + captureProof, ); } /** * Which success-side field a disclosure enters. #2438's surface sentence shipped on the singular * `warning`; the repair sentence ships on the `warnings` array beside the typed fact that travels - * with it. Failure has one carrier for both: `error.details.hint`. + * with it. Failure has one carrier for both: `error.hint`, the hint request finalization keeps. */ type DisclosureCarrier = 'warning' | 'warnings'; @@ -135,14 +137,9 @@ function appendDisclosure( ): DaemonResponse { if (carriesDisclosure(response, disclosure)) return response; if (!response.ok) { - const details = response.error.details ?? {}; - return { - ...response, - error: { - ...response.error, - details: { ...details, hint: appended(details.hint, disclosure) }, - }, - }; + // A route that built its failure from details carries its hint there until finalization. + const hint = response.error.hint ?? response.error.details?.hint; + return { ...response, error: { ...response.error, hint: appended(hint, disclosure) } }; } if (carrier === 'warnings') { return { @@ -166,7 +163,7 @@ function appendDisclosure( function carriesDisclosure(response: DaemonResponse, disclosure: string): boolean { const carriers: unknown[] = response.ok ? [response.data?.warning, ...responseWarnings(response.data?.warnings)] - : [response.error.details?.hint]; + : [response.error.hint]; return carriers.some((carrier) => typeof carrier === 'string' && carrier.includes(disclosure)); } diff --git a/src/daemon/deferred-interaction-outcome.ts b/src/daemon/deferred-interaction-outcome.ts index 23208ad04a..c772b48db5 100644 --- a/src/daemon/deferred-interaction-outcome.ts +++ b/src/daemon/deferred-interaction-outcome.ts @@ -13,7 +13,6 @@ import { isNavigationSensitiveAction, type SnapshotFreshnessMode, } from '@agent-device/capture-kit/snapshot-freshness'; -import { withGestureNoEffectWarning } from './gesture-no-effect.ts'; import { areInteractionSurfaceSignaturesStable, buildInteractionSurfaceSignature, @@ -354,13 +353,13 @@ export async function capturePostGestureStabilizedResult(params: { return outcome; } -/** The stabilized attempt as a capture result: the tree carries an unsettled outcome as its own fact. */ +/** The stabilized attempt as a capture result: the tree carries the gesture's outcome as its own fact. */ function resolvedPostGestureCapture( stabilized: PostGestureStabilityOutcome, ): DeferredOutcomeCaptureResult { const { snapshot, annotations } = stabilized.value; - if (stabilized.unsettledGesture) snapshot.unsettledGesture = stabilized.unsettledGesture; - return { snapshot, ...withGestureNoEffectWarning(annotations, stabilized.gestureNoEffect) }; + if (stabilized.postGestureOutcome) snapshot.postGestureOutcome = stabilized.postGestureOutcome; + return { snapshot, ...annotations }; } function isPostGestureStabilizingAction( diff --git a/src/daemon/gesture-no-effect.ts b/src/daemon/gesture-no-effect.ts deleted file mode 100644 index 38ce4d9cb3..0000000000 --- a/src/daemon/gesture-no-effect.ts +++ /dev/null @@ -1,22 +0,0 @@ -import type { SnapshotCaptureAnnotations } from '@agent-device/contracts/capture'; -import { formatGestureNoEffectWarning } from '@agent-device/capture-kit/post-gesture-stability'; - -/** - * #1600: a proven no-effect gesture must reach the agent inside the very - * response it reads next, not only the diagnostics stream. Warnings ride the - * existing annotations channel so every renderer that already prints capture - * warnings picks this up with no new plumbing. - */ -export function withGestureNoEffectWarning( - annotations: SnapshotCaptureAnnotations, - gestureNoEffect: { action: string; positionals: string[] } | undefined, -): SnapshotCaptureAnnotations { - if (!gestureNoEffect) return annotations; - return { - ...annotations, - warnings: [ - ...(annotations.warnings ?? []), - formatGestureNoEffectWarning(gestureNoEffect.action, gestureNoEffect.positionals), - ], - }; -} diff --git a/src/daemon/interaction/internal/__tests__/find-target-activation-disclosure.test.ts b/src/daemon/interaction/internal/__tests__/find-target-activation-disclosure.test.ts index fdce22ef13..c795e37258 100644 --- a/src/daemon/interaction/internal/__tests__/find-target-activation-disclosure.test.ts +++ b/src/daemon/interaction/internal/__tests__/find-target-activation-disclosure.test.ts @@ -1,5 +1,5 @@ import { beforeEach, expect, test, vi } from 'vitest'; -import type { IosTargetActivation } from '@agent-device/kernel/snapshot'; +import type { IosTargetActivation, PostGestureOutcome } from '@agent-device/kernel/snapshot'; import { iosTargetActivationDisclosure } from '@agent-device/contracts/ios-target-activation'; import { makeSessionStore } from '../../../../__tests__/test-utils/store-factory.ts'; import { makeIosSession } from '../../../../__tests__/test-utils/session-factories.ts'; @@ -8,7 +8,7 @@ import { legacyDispatchCapture } from '../../../__tests__/legacy-snapshot-captur import { getRuntimeBindings } from '../../../__tests__/interaction-get-runtime-fixture.ts'; import { handleFindCommands } from '../../index.ts'; import { markDeferredInteractionOutcome } from '../../../deferred-interaction-outcome.ts'; -import { formatGestureUnsettledWarning } from '@agent-device/capture-kit/post-gesture-stability'; +import { formatPostGestureOutcomeWarning } from '@agent-device/capture-kit/post-gesture-stability'; vi.mock('../../../snapshot-interactor-capture.ts', async () => { const fixture = await import('../../../__tests__/legacy-snapshot-capture-fixture.ts'); @@ -73,7 +73,9 @@ beforeEach(() => { legacyDispatchCapture.mockReset(); }); -async function findClick(captures: Record[], afterScroll = false) { +type CaptureScript = (call: number, context?: Record) => Record; + +async function findClick(captures: Record[] | CaptureScript, afterScroll = false) { const sessionStore = makeSessionStore(); const session = makeIosSession('default', { appBundleId: 'com.example.app' }); if (afterScroll) @@ -81,7 +83,10 @@ async function findClick(captures: Record[], afterScroll = fals sessionStore.set('default', session); let call = 0; legacyDispatchCapture.mockImplementation( - async () => captures[Math.min(call++, captures.length - 1)], + async (_device, _command, _positionals, _out, context) => + typeof captures === 'function' + ? captures(call++, context) + : captures[Math.min(call++, captures.length - 1)], ); const response = await handleFindCommands({ @@ -125,23 +130,33 @@ test('a find that stayed sparse reports the repair on the failure it returns', a expect(response?.ok).toBe(false); if (!response || response.ok) return; - const hint = String(response.error.details?.hint ?? ''); - expect(hint).toContain(iosTargetActivationDisclosure(FACT)); + expect(response.error.hint).toContain(iosTargetActivationDisclosure(FACT)); }); -/** The tree a mutating find resolves against carries every provenance fact, not a hand-picked few. */ -test('a find that misses on a surface still moving after a scroll reports the unsettled fact', async () => { - const moving = Array.from({ length: 40 }, (_, call) => ({ - ...RECOVERED_TREE, +/** + * A sparse capture on a surface still moving after a scroll is replaced by find's query-scoped + * recovery right away. The recovery reads the same moment, so its miss is not proof of absence either. + */ +test('a find that misses after recovering a sparse capture of a moving surface reports the unsettled outcome', async () => { + const movingSparse = (call: number) => ({ + ...SPARSE_VERDICT, + targetActivation: undefined, nodes: [ - RECOVERED_TREE.nodes[0], + SPARSE_VERDICT.nodes[0], { ...RECOVERED_TREE.nodes[1], label: 'Wi-Fi', rect: { ...SCREEN, y: 600 - call * 37 } }, ], - })); + }); + const recoveredWithoutTarget = { + ...RECOVERED_TREE, + nodes: [RECOVERED_TREE.nodes[0], { ...RECOVERED_TREE.nodes[1], label: 'Wi-Fi' }], + }; const realSetTimeout = globalThis.setTimeout; vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] }); let done = false; - const pending = findClick(moving, true).finally(() => (done = true)); + const pending = findClick( + (call, context) => (context?.snapshotScope ? recoveredWithoutTarget : movingSparse(call)), + true, + ).finally(() => (done = true)); // The route awaits real I/O between polls, so the faked clock advances while the test yields. while (!done) { await vi.advanceTimersByTimeAsync(50); @@ -150,10 +165,12 @@ test('a find that misses on a surface still moving after a scroll reports the un const { response } = await pending; vi.useRealTimers(); - expect(response?.ok === false && response.error.details).toMatchObject({ - unsettledGesture: { action: 'scroll', positionals: [] }, - hint: expect.stringContaining( - formatGestureUnsettledWarning({ action: 'scroll', positionals: [] }), - ), + const outcome: PostGestureOutcome = { + kind: 'unsettled', + gesture: { action: 'scroll', positionals: [] }, + }; + expect(response?.ok === false && response.error).toMatchObject({ + hint: expect.stringContaining(formatPostGestureOutcomeWarning(outcome)), + details: { postGestureOutcome: outcome }, }); }); diff --git a/src/daemon/interaction/internal/__tests__/interaction-target-activation-disclosure.test.ts b/src/daemon/interaction/internal/__tests__/interaction-capture-disclosure.test.ts similarity index 53% rename from src/daemon/interaction/internal/__tests__/interaction-target-activation-disclosure.test.ts rename to src/daemon/interaction/internal/__tests__/interaction-capture-disclosure.test.ts index d915ae7be9..03c0e94f7d 100644 --- a/src/daemon/interaction/internal/__tests__/interaction-target-activation-disclosure.test.ts +++ b/src/daemon/interaction/internal/__tests__/interaction-capture-disclosure.test.ts @@ -1,7 +1,8 @@ -import { test, expect } from 'vitest'; +import { test, expect, vi } from 'vitest'; import { attachRefs, type IosTargetActivation, + type PostGestureOutcome, type SnapshotState, } from '@agent-device/kernel/snapshot'; import { iosTargetActivationDisclosure } from '@agent-device/contracts/ios-target-activation'; @@ -9,6 +10,14 @@ import { makeSessionStore } from '../../../../__tests__/test-utils/store-factory import { handleInteractionCommands } from '../../index.ts'; import { getRuntimeBindings } from '../../../__tests__/interaction-get-runtime-fixture.ts'; import { contextFromFlags, makeSession } from './interaction-touch-fixtures.ts'; +import { legacyDispatchCapture } from '../../../__tests__/legacy-snapshot-capture-fixture.ts'; +import { markDeferredInteractionOutcome } from '../../../deferred-interaction-outcome.ts'; +import { formatPostGestureOutcomeWarning } from '@agent-device/capture-kit/post-gesture-stability'; + +vi.mock('../../../snapshot-interactor-capture.ts', async () => { + const fixture = await import('../../../__tests__/legacy-snapshot-capture-fixture.ts'); + return { captureSnapshotWithInteractor: fixture.captureSnapshotThroughLegacyDispatchFixture }; +}); const FACT: IosTargetActivation = { reason: 'stale_target', @@ -124,3 +133,76 @@ test('a press that consumes no capture is not disclosed against an older tree', expect(response.data?.targetActivation).toBeUndefined(); } }); + +const UNSETTLED: PostGestureOutcome = { + kind: 'unsettled', + gesture: { action: 'scroll', positionals: ['down'] }, +}; + +/** A press by selector right after a scroll whose list never stops moving. */ +async function pressAfterUnsettledScroll(selector: string) { + const sessionStore = makeSessionStore(); + const session = makeSession('default'); + markDeferredInteractionOutcome({ session, command: 'scroll', positionals: ['down'], flags: {} }); + sessionStore.set('default', session); + let call = 0; + legacyDispatchCapture.mockImplementation(async () => { + call += 1; + return { + backend: 'xctest', + nodes: [ + { index: 0, depth: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }, + { + index: 1, + parentIndex: 0, + depth: 1, + type: 'Cell', + label: 'Wi-Fi', + rect: { x: 16, y: 600 - call * 37, width: 370, height: 52 }, + hittable: true, + }, + ], + }; + }); + const realSetTimeout = globalThis.setTimeout; + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] }); + let done = false; + const pending = handleInteractionCommands({ + req: { token: 't', session: 'default', command: 'press', positionals: [selector], flags: {} }, + sessionName: 'default', + sessionStore, + contextFromFlags, + ...getRuntimeBindings(), + }).finally(() => (done = true)); + // The route awaits real I/O between polls, so the faked clock advances while the test yields. + while (!done) { + await vi.advanceTimersByTimeAsync(50); + await new Promise((resolve) => realSetTimeout(resolve, 1)); + } + const response = await pending; + vi.useRealTimers(); + return { response, stored: sessionStore.get('default')?.snapshot }; +} + +/** + * The miss on the interactive capture is retried on a full tree at once; that retry reads the same + * moving surface, so neither the response nor the stored tree may present the miss as settled. + */ +test('a press that misses on a surface still moving after a scroll reports the unsettled outcome', async () => { + const { response, stored } = await pressAfterUnsettledScroll('label="General"'); + + expect(response?.ok === false && response.error).toMatchObject({ + hint: expect.stringContaining(formatPostGestureOutcomeWarning(UNSETTLED)), + details: { reason: 'selector_not_found', postGestureOutcome: UNSETTLED }, + }); + expect(stored?.postGestureOutcome).toEqual(UNSETTLED); +}); + +test('a press that lands on a surface still moving after a scroll reports the outcome once', async () => { + const { response } = await pressAfterUnsettledScroll('label="Wi-Fi"'); + + expect(response?.ok && response.data).toMatchObject({ + postGestureOutcome: UNSETTLED, + warnings: [formatPostGestureOutcomeWarning(UNSETTLED)], + }); +}); diff --git a/src/daemon/interaction/internal/find-target-capture.ts b/src/daemon/interaction/internal/find-target-capture.ts index f1698f340d..e0b906f539 100644 --- a/src/daemon/interaction/internal/find-target-capture.ts +++ b/src/daemon/interaction/internal/find-target-capture.ts @@ -1,7 +1,7 @@ import type { FindLocator } from '@agent-device/selectors'; import type { BoundSelectorCapture } from '../../selector-capture-binding.ts'; import type { SnapshotQualityVerdict, SnapshotState } from '@agent-device/kernel/snapshot'; -import type { CaptureProvenance, RequestActivationProof } from '../../capture-disclosure.ts'; +import type { CaptureProvenance, RequestCaptureProof } from '../../capture-disclosure.ts'; import { createSelectorCaptureRuntime } from '../../selector-capture-runtime.ts'; import { SessionStore } from '../../session-store.ts'; import type { DaemonRequest, DaemonResponse } from '../../daemon-request.ts'; @@ -32,7 +32,7 @@ export function createFindTargetCapture( * Filled by whichever capture this find actually took, including a re-capture that replaced a * sparse first tree — find's response is owed the repair its own first capture paid for. */ - activationProof: RequestActivationProof; + captureProof: RequestCaptureProof; }>, ): () => Promise { const { device, session, req, logPath, locator, query, sessionStore, sessionName } = params; @@ -44,7 +44,7 @@ export function createFindTargetCapture( req, logPath, capture: params.capture, - activationProof: params.activationProof, + captureProof: params.captureProof, }); return async () => { // Interaction targets need the full interactive tree so duplicate labels can diff --git a/src/daemon/interaction/internal/find.ts b/src/daemon/interaction/internal/find.ts index 552e4b68fd..8a24349bc5 100644 --- a/src/daemon/interaction/internal/find.ts +++ b/src/daemon/interaction/internal/find.ts @@ -18,7 +18,7 @@ import type { SessionState } from '../../session-state.ts'; import { SessionStore } from '../../session-store.ts'; import { contextFromFlags } from '../../context.ts'; import { readCommandMessage, successText } from '@agent-device/kernel/success-text'; -import type { RequestActivationProof } from '../../capture-disclosure.ts'; +import type { RequestCaptureProof } from '../../capture-disclosure.ts'; import { withCaptureDisclosures } from '../../capture-disclosure.ts'; import { recordSessionAction } from '../../session-action-recorder.ts'; import { stripInternalInteractionFlags } from '../../interaction-outcome-policy.ts'; @@ -129,7 +129,7 @@ export async function handleFindCommands(params: FindRouteInput): Promise { - const activationProof: RequestActivationProof = {}; - const routed = { ...params, refSnapshotFlagGuardResponse, activationProof }; + const captureProof: RequestCaptureProof = {}; + const routed = { ...params, refSnapshotFlagGuardResponse, captureProof }; const response = await dispatchInteractionCommand(routed); - // The interaction's own capture is what the gesture was aimed at, so a foreground repair inside - // it belongs on this response even though the interaction routes never read the stored snapshot. - return response ? withTargetActivationDisclosure(response, activationProof.state) : response; + return response + ? withCaptureDisclosures({ response, consumedTree: captureProof, captureProof }) + : response; } type RoutedInteractionInput = InteractionRouteInput & { diff --git a/src/daemon/interaction/internal/types.ts b/src/daemon/interaction/internal/types.ts index 8a63ae5524..9236285c0d 100644 --- a/src/daemon/interaction/internal/types.ts +++ b/src/daemon/interaction/internal/types.ts @@ -1,7 +1,7 @@ import type { CommandFlags } from '@agent-device/contracts/command'; import type { AndroidObservationAdapter } from '@agent-device/contracts/android-observation'; import type { Rect, SnapshotPreferredBackend, SnapshotState } from '@agent-device/kernel/snapshot'; -import type { RequestActivationProof } from '../../capture-disclosure.ts'; +import type { RequestCaptureProof } from '../../capture-disclosure.ts'; import type { DeviceInfo } from '@agent-device/kernel/device'; import type { CommandSessionStore } from '../../../runtime-contract.ts'; import type { DeferredInteractionOutcomeMark } from '../../deferred-interaction-outcome.ts'; @@ -35,7 +35,7 @@ export type InteractionRouteInput = { * The foreground repair this request's own capture reported, when it captured at all. A coordinate * press consumes no capture and must not be disclosed against an earlier request's tree (#2682). */ - activationProof?: RequestActivationProof; + captureProof?: RequestCaptureProof; }; export type FindRouteInput = { diff --git a/src/daemon/selector-capture-runtime.ts b/src/daemon/selector-capture-runtime.ts index bfbe7849c5..18a2664970 100644 --- a/src/daemon/selector-capture-runtime.ts +++ b/src/daemon/selector-capture-runtime.ts @@ -2,6 +2,7 @@ import type { CommandFlags } from '@agent-device/contracts/command'; import type { BackendSnapshotResult } from '../backend.ts'; import { buildSnapshotPresentationKey, + inheritPostGestureOutcome, snapshotPresentationOptionsFromFlags, type SnapshotState, } from '@agent-device/kernel/snapshot'; @@ -9,8 +10,8 @@ import { isSparseSnapshotQualityVerdict } from '@agent-device/capture-kit/snapsh import type { DaemonRequest } from './daemon-request.ts'; import type { SessionState } from './session-state.ts'; import { SessionStore } from './session-store.ts'; -import { recordActivationProof } from './capture-disclosure.ts'; -import type { RequestActivationProof } from './capture-disclosure.ts'; +import { recordCaptureProof } from './capture-disclosure.ts'; +import type { RequestCaptureProof } from './capture-disclosure.ts'; import { captureSnapshot } from './snapshot-capture.ts'; import { setSessionSnapshot } from './session-snapshot.ts'; import { getActiveAndroidSnapshotFreshness } from './session-snapshot-freshness.ts'; @@ -36,7 +37,7 @@ export type SelectorCaptureRuntimeParams = { * Filled ONLY by a capture this request actually took, never by a cache tier — the foreground * repair a route may disclose has to be one the route paid for (#2682). */ - activationProof?: RequestActivationProof; + captureProof?: RequestCaptureProof; /** * The request-bound capture from `resolveBoundSelectorCapture`: every cache tier, recovery * re-capture, and poll below reaches the platform through it. Required since find (R35) — @@ -134,22 +135,28 @@ async function captureSelectorSnapshot(params: { const { params: runtimeParams, request } = params; const snapshot = await runCapture(runtimeParams, request, request.snapshotScope); if (request.recovery?.legacyIosSparse && isLegacySparseIosInteractiveSnapshot(snapshot)) { - return await recoverLegacySparseIosSnapshot({ - runtimeParams, - request, - policy: request.recovery.legacyIosSparse, - }); + return inheritPostGestureOutcome( + snapshot, + await recoverLegacySparseIosSnapshot({ + runtimeParams, + request, + policy: request.recovery.legacyIosSparse, + }), + ); } if ( request.recovery?.sparseVerdictQueryScope?.shouldScope && isSparseSnapshotQualityVerdict(snapshot.snapshotQuality) ) { - return await recoverSparseVerdictWithQueryScope({ - runtimeParams, - request, - policy: request.recovery.sparseVerdictQueryScope, + return inheritPostGestureOutcome( snapshot, - }); + await recoverSparseVerdictWithQueryScope({ + runtimeParams, + request, + policy: request.recovery.sparseVerdictQueryScope, + snapshot, + }), + ); } return snapshot; } @@ -222,7 +229,7 @@ async function runCapture( // Recorded here rather than at the caller that consumes the result: a sparse recovery re-capture // DISCARDS this tree and returns a fresh one, and the repair this capture paid for belongs to the // request, not to whichever tree survives (#2682). - return recordActivationProof(params.activationProof, capture.snapshot); + return recordCaptureProof(params.captureProof, capture.snapshot); } function readReusableLastSnapshot(params: { @@ -274,7 +281,7 @@ function canUseSessionSnapshotCache( if (request.cache?.useSessionSnapshot !== true) return false; if (getActiveAndroidSnapshotFreshness(session)) return false; if (shouldBypassForPostGestureStabilization(session, request)) return false; - return session.snapshot?.unsettledGesture === undefined; + return session.snapshot?.postGestureOutcome?.kind !== 'unsettled'; } function isFreshSelectorSnapshot(snapshot: SnapshotState, timestamp: number): boolean { diff --git a/src/daemon/selector-runtime-backend.ts b/src/daemon/selector-runtime-backend.ts index 2348d31c3f..4af5d63305 100644 --- a/src/daemon/selector-runtime-backend.ts +++ b/src/daemon/selector-runtime-backend.ts @@ -28,7 +28,7 @@ import type { AndroidObservationAdapter } from '@agent-device/contracts/android- import type { PlatformResourceCleanup } from './platform-resource-cleanup.ts'; import { getRequestSignal } from '@agent-device/host-kit/request'; import { snapshotOptionsToFlags } from '../backend-snapshot-options.ts'; -import type { RequestActivationProof } from './capture-disclosure.ts'; +import type { RequestCaptureProof } from './capture-disclosure.ts'; import { checkIsArgs } from '@agent-device/selectors'; import { noActiveSessionError } from '@agent-device/kernel/contracts'; @@ -42,7 +42,7 @@ export type SelectorRuntimeParams = { // sessionless routes disclose from here because no session record stores the capture. consumedSnapshot?: { state?: SnapshotState }; /** The repair this request's own capture reported, when it captured at all (#2682). */ - activationProof?: RequestActivationProof; + captureProof?: RequestCaptureProof; signal?: AbortSignal; inspectFacts?: InspectDeviceRuntimeFacts; bindDevice?: BindDeviceRuntime; @@ -94,7 +94,7 @@ async function resolveSelectorRuntimeDevice( requireSession: boolean, ): Promise { params.consumedSnapshot ??= {}; - params.activationProof ??= {}; + params.captureProof ??= {}; const session = params.sessionStore.get(params.sessionName); if (!session && requireSession) return { ok: false, response: noActiveSessionError() }; const device = session?.device ?? (await resolveTargetDevice(params.req.flags ?? {})); @@ -172,7 +172,7 @@ function createSelectorBackend(params: SelectorRuntimeDeviceParams): AgentDevice sessionName, req, consumedSnapshot: params.consumedSnapshot, - activationProof: params.activationProof, + captureProof: params.captureProof, logPath, capture: boundOperations.capture, }); diff --git a/src/daemon/selector-runtime.ts b/src/daemon/selector-runtime.ts index c49279293a..13ea73daa6 100644 --- a/src/daemon/selector-runtime.ts +++ b/src/daemon/selector-runtime.ts @@ -104,7 +104,7 @@ export async function dispatchFindReadOnlyViaRuntime( return withCaptureDisclosures({ response, consumedTree: consumedSessionSnapshot(params), - activationProof: params.activationProof, + captureProof: params.captureProof, }); } @@ -182,7 +182,7 @@ export async function dispatchGetViaRuntime( return withCaptureDisclosures({ response, consumedTree: consumedSessionSnapshot(params), - activationProof: params.activationProof, + captureProof: params.captureProof, }); } @@ -242,7 +242,7 @@ export async function dispatchIsViaRuntime( return withCaptureDisclosures({ response: await maybeAndroidForegroundBlockerResponse(params, response, `is ${predicate}`), consumedTree: consumedSessionSnapshot(params), - activationProof: params.activationProof, + captureProof: params.captureProof, }); } diff --git a/src/daemon/snapshot-runtime.ts b/src/daemon/snapshot-runtime.ts index 96c938473f..b7cf97cf30 100644 --- a/src/daemon/snapshot-runtime.ts +++ b/src/daemon/snapshot-runtime.ts @@ -4,7 +4,7 @@ import { SNAPSHOT_COMMAND_OPTION_KEYS, snapshotOptionsFromFlags, } from '@agent-device/kernel/snapshot'; -import type { RequestActivationProof } from './capture-disclosure.ts'; +import type { RequestCaptureProof } from './capture-disclosure.ts'; import { withTargetActivationDisclosure } from './capture-disclosure.ts'; import { dispatchSnapshotRuntimeCommand } from './snapshot-command-runtime.ts'; import { captureSparseFallbackScreenshot } from './sparse-fallback-screenshot.ts'; @@ -15,7 +15,7 @@ import type { SessionState } from './session-state.ts'; export async function dispatchSnapshotViaRuntime( params: SnapshotRuntimeRouteParams, ): Promise { - const activationProof: RequestActivationProof = {}; + const captureProof: RequestCaptureProof = {}; const response = await dispatchSnapshotRuntimeCommand({ ...params, command: 'snapshot', @@ -33,9 +33,7 @@ export async function dispatchSnapshotViaRuntime( }); // This request's own capture, read here rather than off the stored snapshot: a snapshot that // failed before capturing must not inherit the previous command's repair (#2682). - if (result.targetActivation && !activationProof.state) { - activationProof.state = { targetActivation: result.targetActivation }; - } + if (result.targetActivation) captureProof.targetActivation ??= result.targetActivation; const refsGeneration = publishedSnapshotGeneration( request, params.sessionStore.get(resolvedSessionName), @@ -83,7 +81,7 @@ export async function dispatchSnapshotViaRuntime( // own annotations, so re-deriving it from the stored snapshot would copy a sentence the response // already carries — and on a snapshot that failed before capturing, would credit it with a surface // it never observed (#2682). - return withTargetActivationDisclosure(response, activationProof.state); + return withTargetActivationDisclosure(response, captureProof); } function publishedSnapshotGeneration( diff --git a/src/daemon/wait-current-surface.test.ts b/src/daemon/wait-current-surface.test.ts index f11a6aa40b..3b1865a865 100644 --- a/src/daemon/wait-current-surface.test.ts +++ b/src/daemon/wait-current-surface.test.ts @@ -1,7 +1,7 @@ import { expect, test, vi } from 'vitest'; import { ANDROID_EMULATOR } from '../__tests__/test-utils/device-fixtures.ts'; -import type { RequestActivationProof } from './capture-disclosure.ts'; +import type { RequestCaptureProof } from './capture-disclosure.ts'; import { maybeWaitTimeoutSurfaceResponse } from './wait-current-surface.ts'; import type { BoundSelectorCapture } from './selector-capture-binding.ts'; @@ -113,10 +113,10 @@ test('the decoration capture records the foreground repair it paid for', async ( targetActivation: repair, nodes: [{ index: 0, depth: 0, type: 'Button', label: 'Receipt' }], })) as unknown as BoundSelectorCapture; - const activationProof: RequestActivationProof = {}; + const captureProof: RequestCaptureProof = {}; await maybeWaitTimeoutSurfaceResponse( - { req, session: undefined, device: ANDROID_EMULATOR, capture, activationProof }, + { req, session: undefined, device: ANDROID_EMULATOR, capture, captureProof }, { ok: false as const, error: { @@ -127,5 +127,5 @@ test('the decoration capture records the foreground repair it paid for', async ( }, ); - expect(activationProof.state).toMatchObject({ targetActivation: repair }); + expect(captureProof.targetActivation).toEqual(repair); }); diff --git a/src/daemon/wait-current-surface.ts b/src/daemon/wait-current-surface.ts index 5238b2fb37..db7de91bcd 100644 --- a/src/daemon/wait-current-surface.ts +++ b/src/daemon/wait-current-surface.ts @@ -1,7 +1,7 @@ import { WAIT_REASONS } from '@agent-device/contracts/wait'; import type { SnapshotNode } from '@agent-device/kernel/snapshot'; -import { recordActivationProof } from './capture-disclosure.ts'; -import type { RequestActivationProof } from './capture-disclosure.ts'; +import { recordCaptureProof } from './capture-disclosure.ts'; +import type { RequestCaptureProof } from './capture-disclosure.ts'; import type { DaemonRequest, DaemonResponse } from './daemon-request.ts'; import type { SessionState } from './session-state.ts'; import { captureSnapshot } from './snapshot-capture.ts'; @@ -25,7 +25,7 @@ type WaitCurrentSurfaceParams = { * A timed-out wait still consumed that capture to describe its surface, so the repair it paid for * belongs to the response this module decorates. */ - activationProof?: RequestActivationProof; + captureProof?: RequestCaptureProof; }; type CurrentSurfaceDetails = { @@ -89,7 +89,7 @@ async function inspectCurrentSurface( }), ), }); - recordActivationProof(params.activationProof, capture.snapshot); + recordCaptureProof(params.captureProof, capture.snapshot); const orderedNodes = [...capture.snapshot.nodes].sort(compareSurfacePriority); const labels = topSurfaceTexts(orderedNodes, 6, { includeIdentifiers: true }); if (labels.length === 0) return null; diff --git a/src/daemon/wait-runtime.ts b/src/daemon/wait-runtime.ts index 025061dabd..766f7df922 100644 --- a/src/daemon/wait-runtime.ts +++ b/src/daemon/wait-runtime.ts @@ -54,7 +54,7 @@ export async function dispatchWaitViaRuntime(params: DispatchWaitParams): Promis // Wait builds its runtime directly (no createBoundSelectorRuntime), so the consumed-snapshot slot // must be initialized here too or sessionless waits have nowhere to report the capture from. params.consumedSnapshot ??= {}; - params.activationProof ??= {}; + params.captureProof ??= {}; // A pure sleep consumes no capture, so it never earns the system-surface disclosure below. if (parsed.kind === 'sleep') { return await executeWaitRequest( @@ -86,7 +86,7 @@ export async function dispatchWaitViaRuntime(params: DispatchWaitParams): Promis params.platformResourceCleanup, ), consumedTree: consumedSessionSnapshot(params), - activationProof: params.activationProof, + captureProof: params.captureProof, }); } @@ -187,7 +187,7 @@ async function executeWaitRequest( session, device, capture: waitOperations.capture, - activationProof: params.activationProof, + captureProof: params.captureProof, }, response, ) diff --git a/test/integration/ios-simulator-e2e-visibility-scroll.test.ts b/test/integration/ios-simulator-e2e-visibility-scroll.test.ts index ae2458bb30..e2de4a401b 100644 --- a/test/integration/ios-simulator-e2e-visibility-scroll.test.ts +++ b/test/integration/ios-simulator-e2e-visibility-scroll.test.ts @@ -16,13 +16,15 @@ function result(status: number, details?: Record): CliJsonResul }; } -const UNSETTLED = { unsettledGesture: { action: 'scroll', positionals: [] } }; +const UNSETTLED = { + postGestureOutcome: { kind: 'unsettled', gesture: { action: 'scroll', positionals: [] } }, +}; /** * A vertical list the search drives. Offsets are in viewports; the target is visible while the * offset lies inside `visible`. Each scroll moves by the next planned travel, clamped to the list * bounds. `movesUntilSettled` keeps the surface moving after every scroll that moved until the - * search pauses to settle it, so each read before that misses with `unsettledGesture`: the CI + * search pauses to settle it, so each read before that misses with an unsettled outcome: the CI * failure shape. */ function list(options: { @@ -82,6 +84,24 @@ test('a stalled capture retries without scrolling or consuming an attempt', asyn assert.deepEqual([probes.length, scrolls], [0, []]); }); +test('a miss after a gesture that moved nothing is a real read, not a moving surface', async () => { + const noEffect = { + postGestureOutcome: { kind: 'no-effect', gesture: { action: 'scroll', positionals: [] } }, + }; + const probes = [result(1, noEffect), result(0)]; + const scrolls: string[] = []; + + await searchForVisibleElement('id="target"', { + probeVisibility: async () => probes.shift() ?? result(1), + settle: async () => assert.fail('a no-effect read is settled'), + scroll: async (step) => { + scrolls.push(step.direction); + }, + }); + + assert.deepEqual([probes.length, scrolls], [0, ['down']]); +}); + test('an unsettled miss waits for the surface to settle and re-reads at the same offset', async () => { const { device, log } = list({ visible: [0.5, 1.2], movesUntilSettled: true }); diff --git a/test/integration/ios-simulator-e2e/live-assertions.ts b/test/integration/ios-simulator-e2e/live-assertions.ts index b0e33d0092..7b37a9842c 100644 --- a/test/integration/ios-simulator-e2e/live-assertions.ts +++ b/test/integration/ios-simulator-e2e/live-assertions.ts @@ -129,7 +129,7 @@ export async function searchForVisibleElement( /** Why a missed probe says nothing about where the element is, if it says nothing. */ function unreadSurface(result: CliJsonResult): 'moving' | 'stalled' | undefined { const details = result.json?.error?.details; - if (details?.unsettledGesture !== undefined) return 'moving'; + if (details?.postGestureOutcome?.kind === 'unsettled') return 'moving'; return details?.captureStalled === true ? 'stalled' : undefined; } diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 55319492a5..e470e55ce3 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -558,7 +558,9 @@ agent-device is text 'id="greeting"' "Welcome back" - `is visible` checks whether the resolved element is present in the current visible snapshot viewport. A node without its own rect still passes when a visible ancestor within the viewport provides the on-screen geometry. - `is exists` only checks whether the selector matches in the current snapshot. - `is absent` passes only when the selector has zero matches in one readable, complete, settled, unscoped, full-depth accessibility capture. It does not mean hidden; `--scope` and `--depth` are rejected, and sparse, unreadable, truncated, or unsettled captures fail closed. -- A read that answers from the first capture after a `scroll`, `swipe`, or `gesture swipe` waits for two consecutive captures to agree. When the surface is still changing when that budget runs out, `is`, `get`, `find`, and `wait` carry `unsettledGesture` (`{ "action", "positionals" }`) in `error.details` or `data` and append a warning, and `snapshot` appends the warning. A miss on that capture is not proof of absence: read again. `is absent` refuses it with `observation: "unsettled"`, and `wait absent` keeps polling. +- A read that answers from the first capture after a `scroll`, `swipe`, or `gesture swipe` waits for two consecutive captures to agree. That capture, and a re-capture taken at once to recover or widen it, carry `postGestureOutcome` (`{ "kind", "gesture": { "action", "positionals" } }`) when stabilization proved something about the gesture. `is`, `get`, `find`, `wait`, and an interaction that captured it (such as `click`, `press`, or `fill`) report it in `error.details` or `data` and append a warning, and `snapshot` appends the warning. + - `kind: "unsettled"`: the surface was still changing when the budget ran out. A miss on that capture is not proof of absence: read again. `is absent` refuses it with `observation: "unsettled"`, and `wait absent` keeps polling. + - `kind: "no-effect"`: the settled tree still matches the tree from before the gesture. The container may be at its edge or may ignore synthesized scrolls; a raw `swipe x1 y1 x2 y2` inside the list moves such lists. - `wait text` is a text-presence wait, not a hittability assertion. - Strict `wait absent` is not exported to Maestro's lenient `notVisible` condition; Maestro export reports it as unsupported unless an exact zero-candidate primitive becomes available. - `is text ` compares the resolved element text against the expected value.