diff --git a/packages/contracts/src/facades/session.ts b/packages/contracts/src/facades/session.ts index 6750c38f76..264075cdc0 100644 --- a/packages/contracts/src/facades/session.ts +++ b/packages/contracts/src/facades/session.ts @@ -1,4 +1,13 @@ export type { SessionAction } from '../session-action.ts'; export type { SessionScope } from '../session-scope.ts'; -export { SESSION_SURFACES, parseSessionSurface } from '../session-surface.ts'; -export type { SessionSurface } from '../session-surface.ts'; +export { + SESSION_SURFACES, + macOsHelperSurface, + macOsSurfaceBackend, + parseSessionSurface, +} from '../session-surface.ts'; +export type { + MacOsHelperSurface, + MacOsSurfaceBackend, + SessionSurface, +} from '../session-surface.ts'; diff --git a/packages/contracts/src/session-surface.test.ts b/packages/contracts/src/session-surface.test.ts new file mode 100644 index 0000000000..eb42f1cf9e --- /dev/null +++ b/packages/contracts/src/session-surface.test.ts @@ -0,0 +1,23 @@ +import { expect, test } from 'vitest'; +import { + SESSION_SURFACES, + macOsHelperSurface, + macOsSurfaceBackend, + type MacOsSurfaceBackend, + type SessionSurface, +} from './session-surface.ts'; + +const EXPECTED_BACKENDS: Record = { + app: 'xctest', + 'frontmost-app': 'macos-helper', + desktop: 'macos-helper', + menubar: 'macos-helper', +}; + +test.each([ + ...SESSION_SURFACES.map((surface) => [surface, EXPECTED_BACKENDS[surface]] as const), + [undefined, 'xctest'] as const, +])('the macOS %s surface is served by %s', (surface, backend) => { + expect(macOsSurfaceBackend(surface)).toBe(backend); + expect(macOsHelperSurface(surface)).toBe(backend === 'macos-helper' ? surface : undefined); +}); diff --git a/packages/contracts/src/session-surface.ts b/packages/contracts/src/session-surface.ts index d5728edb62..ab3ddcb9b5 100644 --- a/packages/contracts/src/session-surface.ts +++ b/packages/contracts/src/session-surface.ts @@ -1,3 +1,4 @@ +import type { SnapshotBackend } from '@agent-device/kernel/snapshot'; import { defineStringEnum } from './string-enum.ts'; export const SESSION_SURFACES = ['app', 'frontmost-app', 'desktop', 'menubar'] as const; @@ -10,3 +11,34 @@ const SESSION_SURFACE_ENUM = defineStringEnum(SESSION_SURFACES, { export function parseSessionSurface(value: string | undefined): SessionSurface { return SESSION_SURFACE_ENUM.parse(value); } + +/** The backend that serves every operation on a macOS surface. */ +export type MacOsSurfaceBackend = Extract; + +const MACOS_SURFACE_BACKENDS = { + app: 'xctest', + 'frontmost-app': 'macos-helper', + desktop: 'macos-helper', + menubar: 'macos-helper', +} as const satisfies Record; + +/** An absent surface is an app session, the reading every route already gives it. */ +export function macOsSurfaceBackend(surface: SessionSurface | undefined): MacOsSurfaceBackend { + return MACOS_SURFACE_BACKENDS[surface ?? 'app']; +} + +type HelperRoutedSurface = { + [S in SessionSurface]: (typeof MACOS_SURFACE_BACKENDS)[S] extends 'macos-helper' ? S : never; +}[SessionSurface]; + +declare const helperSurface: unique symbol; +/** A surface the owner routed to the macOS helper; only `macOsHelperSurface` produces one. */ +export type MacOsHelperSurface = HelperRoutedSurface & { readonly [helperSurface]: true }; + +export function macOsHelperSurface( + surface: SessionSurface | undefined, +): MacOsHelperSurface | undefined { + return surface !== undefined && macOsSurfaceBackend(surface) === 'macos-helper' + ? (surface as MacOsHelperSurface) + : undefined; +} diff --git a/packages/platform-apple/src/__tests__/interactor-macos-surface.test.ts b/packages/platform-apple/src/__tests__/interactor-macos-surface.test.ts new file mode 100644 index 0000000000..b0f5dc97b8 --- /dev/null +++ b/packages/platform-apple/src/__tests__/interactor-macos-surface.test.ts @@ -0,0 +1,181 @@ +import { beforeEach, expect, test, vi } from 'vitest'; +import type { DeviceInfo } from '@agent-device/kernel/device'; +import { SCREENSHOT_FULLSCREEN_REASONS } from '@agent-device/contracts/capture'; +import type { PressPointOptions } from '@agent-device/contracts/interactor-types'; +import { + SESSION_SURFACES, + type MacOsSurfaceBackend, + type SessionSurface, +} from '@agent-device/contracts/session'; + +vi.mock('../os/macos/helper.ts', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + runMacOsScreenshotAction: vi.fn(async (outPath: string) => ({ path: outPath })), + runMacOsSnapshotAction: vi.fn(async (surface: SessionSurface) => ({ + surface, + nodes: [], + truncated: false, + backend: 'macos-helper' as const, + })), + runMacOsReadTextAction: vi.fn(async () => ({ text: 'helper' })), + runMacOsPressAction: vi.fn(async (x: number, y: number) => ({ x, y })), + }; +}); + +vi.mock('../core/screenshot.ts', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + captureScreenshotViaRunner: vi.fn(), + screenshotIos: vi.fn(), + }; +}); + +vi.mock('../core/runner-client.ts', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + runAppleRunnerCommand: vi.fn(async () => ({})), + }; +}); + +import { createAppleInteractor } from '../interactor.ts'; +import { + runMacOsPressAction, + runMacOsReadTextAction, + runMacOsScreenshotAction, + runMacOsSnapshotAction, +} from '../os/macos/helper.ts'; +import { captureScreenshotViaRunner, screenshotIos } from '../core/screenshot.ts'; +import { runAppleRunnerCommand } from '../core/runner-client.ts'; + +const macOsDevice: DeviceInfo = { + platform: 'apple', + appleOs: 'macos', + id: 'host-mac', + name: 'Host Mac', + kind: 'device', + target: 'desktop', + booted: true, +}; + +const MACOS_SURFACE_BACKENDS: Record = { + app: 'xctest', + 'frontmost-app': 'macos-helper', + desktop: 'macos-helper', + menubar: 'macos-helper', +}; + +const SURFACE_ROWS = [ + ...SESSION_SURFACES.map((surface) => [surface, MACOS_SURFACE_BACKENDS[surface]] as const), + [undefined, 'xctest'] as const, +]; + +const HELPER_ROWS = SESSION_SURFACES.filter( + (surface) => MACOS_SURFACE_BACKENDS[surface] === 'macos-helper', +); + +const PRIMARY_PRESS: Omit = { + button: 'primary', + count: 1, + intervalMs: 0, + holdMs: 0, + jitterPx: 0, + doubleTap: false, +}; + +const helperEntryPoints = [ + runMacOsScreenshotAction, + runMacOsSnapshotAction, + runMacOsReadTextAction, + runMacOsPressAction, +]; +const runnerEntryPoints = [runAppleRunnerCommand, screenshotIos, captureScreenshotViaRunner]; + +function clearEntryPoints(): void { + for (const entry of [...helperEntryPoints, ...runnerEntryPoints]) vi.mocked(entry).mockClear(); +} + +beforeEach(clearEntryPoints); + +function reachedBackend(): MacOsSurfaceBackend { + const helper = helperEntryPoints.some((entry) => vi.mocked(entry).mock.calls.length > 0); + const runner = runnerEntryPoints.some((entry) => vi.mocked(entry).mock.calls.length > 0); + expect(helper !== runner).toBe(true); + return helper ? 'macos-helper' : 'xctest'; +} + +test.each(SURFACE_ROWS)( + 'every macOS operation on the %s surface reaches the %s backend', + async (surface, backend) => { + const interactor = createAppleInteractor(macOsDevice, {}); + const operations: Record Promise> = { + screenshot: async () => await interactor.screenshot('/tmp/out.png', { surface }), + snapshot: async () => await interactor.snapshot({ surface }), + readTextAtPoint: async () => await interactor.readTextAtPoint?.({ x: 1, y: 2 }, { surface }), + pressPoint: async () => + await interactor.pressPoint?.({ x: 1, y: 2 }, { ...PRIMARY_PRESS, surface }), + }; + const reached: Record = {}; + for (const [name, run] of Object.entries(operations)) { + clearEntryPoints(); + await run(); + reached[name] = reachedBackend(); + } + expect(reached).toEqual({ + screenshot: backend, + snapshot: backend, + readTextAtPoint: backend, + pressPoint: backend, + }); + }, +); + +test.each(HELPER_ROWS)( + 'refuses an explicit --fullscreen on the macOS %s surface before any capture', + async (surface) => { + const interactor = createAppleInteractor(macOsDevice, {}); + + await expect( + interactor.screenshot('/tmp/out.png', { surface, fullscreen: true }), + ).rejects.toMatchObject({ + code: 'INVALID_ARGS', + details: expect.objectContaining({ + reason: SCREENSHOT_FULLSCREEN_REASONS.macOsHelperSurfaceFixedFrame, + surface, + }), + }); + + expect(runMacOsScreenshotAction).not.toHaveBeenCalled(); + }, +); + +test.each(HELPER_ROWS)( + 'captures the %s surface through the helper when --fullscreen is not requested', + async (surface) => { + const interactor = createAppleInteractor(macOsDevice, {}); + + await interactor.screenshot('/tmp/out.png', { surface }); + + expect(runMacOsScreenshotAction).toHaveBeenCalledOnce(); + const [, options] = vi.mocked(runMacOsScreenshotAction).mock.calls[0]!; + expect(options).toEqual({ surface }); + expect(Object.hasOwn(options ?? {}, 'fullscreen')).toBe(false); + }, +); + +test('keeps a macOS app session on the runner path with --fullscreen unchanged', async () => { + const interactor = createAppleInteractor(macOsDevice, {}); + + await interactor.screenshot('/tmp/out.png', { surface: 'app', fullscreen: true }); + + expect(runMacOsScreenshotAction).not.toHaveBeenCalled(); + expect(screenshotIos).toHaveBeenCalledOnce(); + expect(screenshotIos).toHaveBeenCalledWith( + macOsDevice, + '/tmp/out.png', + expect.objectContaining({ fullscreen: true }), + ); +}); diff --git a/packages/platform-apple/src/__tests__/screenshot-macos-surface.test.ts b/packages/platform-apple/src/__tests__/screenshot-macos-surface.test.ts deleted file mode 100644 index 3d41e5d0d1..0000000000 --- a/packages/platform-apple/src/__tests__/screenshot-macos-surface.test.ts +++ /dev/null @@ -1,92 +0,0 @@ -import { beforeEach, expect, test, vi } from 'vitest'; -import type { DeviceInfo } from '@agent-device/kernel/device'; -import { SCREENSHOT_FULLSCREEN_REASONS } from '@agent-device/contracts/capture'; -import { SESSION_SURFACES } from '@agent-device/contracts/session'; - -vi.mock('../os/macos/helper.ts', async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - runMacOsScreenshotAction: vi.fn(async (outPath: string) => ({ path: outPath })), - }; -}); - -vi.mock('../core/screenshot.ts', async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - captureScreenshotViaRunner: vi.fn(), - screenshotIos: vi.fn(), - }; -}); - -import { createAppleInteractor } from '../interactor.ts'; -import { runMacOsScreenshotAction } from '../os/macos/helper.ts'; -import { screenshotIos } from '../core/screenshot.ts'; - -const macOsDevice: DeviceInfo = { - platform: 'apple', - appleOs: 'macos', - id: 'host-mac', - name: 'Host Mac', - kind: 'device', - target: 'desktop', - booted: true, -}; - -beforeEach(() => { - vi.mocked(runMacOsScreenshotAction).mockClear(); - vi.mocked(screenshotIos).mockClear(); -}); - -// The helper-routed domain, derived from the same condition `usesMacOsSurfaceScreenshot` applies -// (every session surface except `app`) rather than a hand-picked list — so adding a surface to -// `SESSION_SURFACES` extends this coverage automatically instead of silently falling outside it. -const helperRoutedSurfaces = SESSION_SURFACES.filter((surface) => surface !== 'app'); - -test.each(helperRoutedSurfaces)( - 'refuses an explicit --fullscreen on the macOS %s surface before any capture', - async (surface) => { - const interactor = createAppleInteractor(macOsDevice, {}); - - await expect( - interactor.screenshot('/tmp/out.png', { surface, fullscreen: true }), - ).rejects.toMatchObject({ - code: 'INVALID_ARGS', - details: expect.objectContaining({ - reason: SCREENSHOT_FULLSCREEN_REASONS.macOsHelperSurfaceFixedFrame, - surface, - }), - }); - - expect(runMacOsScreenshotAction).not.toHaveBeenCalled(); - }, -); - -test.each(helperRoutedSurfaces)( - 'captures the %s surface through the helper when --fullscreen is not requested', - async (surface) => { - const interactor = createAppleInteractor(macOsDevice, {}); - - await interactor.screenshot('/tmp/out.png', { surface }); - - expect(runMacOsScreenshotAction).toHaveBeenCalledOnce(); - const [, options] = vi.mocked(runMacOsScreenshotAction).mock.calls[0]!; - expect(options).toEqual({ surface }); - expect(Object.hasOwn(options ?? {}, 'fullscreen')).toBe(false); - }, -); - -test('keeps a macOS app session on the runner path with --fullscreen unchanged', async () => { - const interactor = createAppleInteractor(macOsDevice, {}); - - await interactor.screenshot('/tmp/out.png', { surface: 'app', fullscreen: true }); - - expect(runMacOsScreenshotAction).not.toHaveBeenCalled(); - expect(screenshotIos).toHaveBeenCalledOnce(); - expect(screenshotIos).toHaveBeenCalledWith( - macOsDevice, - '/tmp/out.png', - expect.objectContaining({ fullscreen: true }), - ); -}); diff --git a/packages/platform-apple/src/interactions.ts b/packages/platform-apple/src/interactions.ts index e6048040a0..a770e19f3d 100644 --- a/packages/platform-apple/src/interactions.ts +++ b/packages/platform-apple/src/interactions.ts @@ -10,6 +10,7 @@ import { type TextEntryRoute, type TypeTextBackendResult, } from '@agent-device/contracts/interactor-types'; +import { macOsHelperSurface, type MacOsHelperSurface } from '@agent-device/contracts/session'; import { SCROLL_DURATION_MAX_MS, normalizeScrollDurationMs, @@ -194,8 +195,9 @@ async function runApplePressPoint( point: { x: number; y: number }, options: PressPointOptions, ): Promise> { - if (isMacOs(device) && options.surface && options.surface !== 'app') { - return await runMacOsSurfacePress(context, point, options); + const helper = isMacOs(device) ? macOsHelperSurface(options.surface) : undefined; + if (helper) { + return await runMacOsSurfacePress(context, point, options, helper); } if (options.button !== 'primary') { return await runAppleAlternateClick(device, context, runnerOpts, point, options.button); @@ -216,11 +218,12 @@ async function runMacOsSurfacePress( context: RunnerContext, point: { x: number; y: number }, options: PressPointOptions, + surface: MacOsHelperSurface, ): Promise> { if (options.button !== 'primary') { throw new AppError( 'UNSUPPORTED_OPERATION', - `${options.button} click is not supported on macOS ${options.surface} sessions.`, + `${options.button} click is not supported on macOS ${surface} sessions.`, ); } const { runMacOsPressAction } = await import('./os/macos/helper.ts'); @@ -228,7 +231,7 @@ async function runMacOsSurfacePress( // the same reading every other platform gives the two flags. const posted = await runMacOsPressAction(point.x, point.y, { bundleId: context.appBundleId, - surface: options.surface, + surface, holdMs: options.holdMs, clicks: options.count, doubleClick: options.doubleTap, diff --git a/packages/platform-apple/src/interactor.ts b/packages/platform-apple/src/interactor.ts index 945a32309f..0d960fea39 100644 --- a/packages/platform-apple/src/interactor.ts +++ b/packages/platform-apple/src/interactor.ts @@ -14,7 +14,7 @@ import { } from './runner/index.ts'; import { toAppleTvRemoteButton } from '@agent-device/contracts/tv-remote'; import { SCREENSHOT_FULLSCREEN_REASONS } from '@agent-device/contracts/capture'; -import type { SessionSurface } from '@agent-device/contracts/session'; +import { macOsHelperSurface, type MacOsHelperSurface } from '@agent-device/contracts/session'; import { DEVICE_ROTATIONS, type DeviceRotation } from '@agent-device/contracts/device'; import { normalizeSnapshotScope } from '@agent-device/contracts/snapshot'; import { withDiagnosticTimer } from '@agent-device/host-kit/diagnostics'; @@ -68,12 +68,14 @@ export function createAppleInteractor( close: (app) => closeIosApp(device, app, runnerOpts), screenshot: (outPath, options) => runAppleScreenshot(device, outPath, options, runnerOpts), snapshot: async (options) => await captureAppleSnapshot(device, options, runnerOpts), - // The live text at a point: helper for macOS desktop/menubar surfaces, XCTest runner for + // The live text at a point: helper for a helper-routed macOS surface, XCTest runner for // every other Apple leaf including a macOS app session. - readTextAtPoint: async (point, options) => - usesMacOsHelperSurface(device, options?.surface) - ? await readMacOsSurfaceTextAtPoint(point, options) - : await readRunnerTextAtPoint(device, point, options, runnerOpts), + readTextAtPoint: async (point, options) => { + const helper = isMacOs(device) ? macOsHelperSurface(options?.surface) : undefined; + return helper + ? await readMacOsSurfaceTextAtPoint(point, helper, options?.appBundleId) + : await readRunnerTextAtPoint(device, point, options, runnerOpts); + }, // The XCTest runner's own text reading: it observes the live accessibility hierarchy // directly, so it answers without the cost — and without the pruning — of a tree capture. // Only a positive answer is authoritative; see `FindTextResult`. @@ -209,8 +211,9 @@ async function captureAppleSnapshot( options: SnapshotOptions | undefined, runnerOpts: RunnerCallOptions, ) { - if (isMacOs(device) && options?.surface && options.surface !== 'app') { - return await captureMacOsSurfaceSnapshot(options, options.signal); + const helper = isMacOs(device) ? macOsHelperSurface(options?.surface) : undefined; + if (helper) { + return await captureMacOsSurfaceSnapshot({ ...options, surface: helper }, options?.signal); } return await captureAppleRunnerSnapshot(device, options, runnerOpts); } @@ -386,20 +389,19 @@ async function runAppleScreenshot( options: ScreenshotOptions = {}, runnerOpts: RunnerCallOptions, ): Promise { - if (usesMacOsSurfaceScreenshot(device, options.surface)) { + const helper = isMacOs(device) ? macOsHelperSurface(options.surface) : undefined; + if (helper) { if (options.fullscreen) { throw new AppError( 'INVALID_ARGS', - `screenshot --fullscreen is not accepted on the macOS ${options.surface} surface: it always captures the main display`, + `screenshot --fullscreen is not accepted on the macOS ${helper} surface: it always captures the main display`, { reason: SCREENSHOT_FULLSCREEN_REASONS.macOsHelperSurfaceFixedFrame, - surface: options.surface, + surface: helper, }, ); } - await runMacOsScreenshotAction(outPath, { - surface: options.surface, - }); + await runMacOsScreenshotAction(outPath, { surface: helper }); return; } if (options.captureBackend === 'runner') { @@ -424,31 +426,15 @@ async function runAppleScreenshot( }); } -/** - * Every surface this admits captures through the macOS helper's fixed main-display frame, so - * `runAppleScreenshot` also keys its `--fullscreen` refusal directly off this predicate: whichever - * surface routes here cannot vary its captured frame, helper-routed today or added later. - */ -function usesMacOsSurfaceScreenshot( - device: DeviceInfo, - surface: ScreenshotOptions['surface'], -): surface is Exclude { - return isMacOs(device) && surface !== undefined && surface !== 'app'; -} - -/** Only non-app macOS surfaces are helper-read; an app session is runner-read like any leaf. */ -function usesMacOsHelperSurface(device: DeviceInfo, surface: SessionSurface | undefined): boolean { - return isMacOs(device) && surface !== undefined && surface !== 'app'; -} - async function readMacOsSurfaceTextAtPoint( point: Point, - options?: { appBundleId?: string; surface?: SessionSurface }, + surface: MacOsHelperSurface, + appBundleId: string | undefined, ): Promise { const { runMacOsReadTextAction } = await import('./os/macos/helper.ts'); const result = await runMacOsReadTextAction(point.x, point.y, { - bundleId: options?.appBundleId, - surface: options?.surface, + bundleId: appBundleId, + surface, }); return result.text; } diff --git a/packages/platform-apple/src/os/macos/helper.test.ts b/packages/platform-apple/src/os/macos/helper.test.ts index 483483d494..a6d129fbe6 100644 --- a/packages/platform-apple/src/os/macos/helper.test.ts +++ b/packages/platform-apple/src/os/macos/helper.test.ts @@ -1,13 +1,19 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; +import { macOsHelperSurface } from '@agent-device/contracts/session'; import { createLocalAppleToolProvider, withAppleToolProvider } from '../../core/tool-provider.ts'; import { macOsClickScheduleMs, runMacOsPressAction, + runMacOsReadTextAction, runMacOsScreenshotAction, runMacOsSnapshotAction, } from './helper.ts'; +const desktop = macOsHelperSurface('desktop')!; +const menubar = macOsHelperSurface('menubar')!; +const frontmostApp = macOsHelperSurface('frontmost-app')!; + test('macOS helper snapshot passes cancellation to the helper process', async () => { const controller = new AbortController(); let receivedSignal: AbortSignal | undefined; @@ -34,7 +40,7 @@ test('macOS helper snapshot passes cancellation to the helper process', async () await withAppleToolProvider( provider, - async () => await runMacOsSnapshotAction('desktop', { signal: controller.signal }), + async () => await runMacOsSnapshotAction(desktop, { signal: controller.signal }), ); assert.equal(receivedSignal, controller.signal); @@ -63,7 +69,7 @@ test('macOS helper press carries the hold, repeat count, and interval to the cli provider, async () => await runMacOsPressAction(12, 34, { - surface: 'menubar', + surface: menubar, bundleId: 'com.example.Menu', holdMs: 800, clicks: 2, @@ -104,7 +110,7 @@ test('macOS helper press carries an explicit zero interval instead of dropping i await withAppleToolProvider( provider, - async () => await runMacOsPressAction(7, 8, { surface: 'desktop', clicks: 2, intervalMs: 0 }), + async () => await runMacOsPressAction(7, 8, { surface: desktop, clicks: 2, intervalMs: 0 }), ); assert.ok(receivedArgs.includes('--clicks'), receivedArgs.join(' ')); @@ -131,7 +137,7 @@ test('macOS helper press keeps repeats independent and names a double-click expl await withAppleToolProvider( provider, async () => - await runMacOsPressAction(7, 8, { surface: 'frontmost-app', clicks: 3, doubleClick: true }), + await runMacOsPressAction(7, 8, { surface: frontmostApp, clicks: 3, doubleClick: true }), ); // `--count 3 --double-tap` is three double-clicks: the count stays the press count and the @@ -163,7 +169,7 @@ test('macOS helper press outlives its own click schedule and forwards cancellati provider, async () => await runMacOsPressAction(1, 2, { - surface: 'desktop', + surface: desktop, holdMs: 10_000, clicks: 4, intervalMs: 120, @@ -205,7 +211,7 @@ test('macOS helper press stays a single held click when nothing is repeated', as await withAppleToolProvider( provider, - async () => await runMacOsPressAction(5, 6, { surface: 'frontmost-app' }), + async () => await runMacOsPressAction(5, 6, { surface: frontmostApp }), ); assert.equal(receivedArgs.includes('--clicks'), false); @@ -226,8 +232,26 @@ test('macOS helper screenshot argv carries only --out and --surface', async () = await withAppleToolProvider( provider, - async () => await runMacOsScreenshotAction('/tmp/out.png', { surface: 'desktop' }), + async () => await runMacOsScreenshotAction('/tmp/out.png', { surface: desktop }), ); assert.deepEqual(receivedArgs, ['screenshot', '--out', '/tmp/out.png', '--surface', 'desktop']); }); + +test('helper entry points accept only an owner-routed surface', () => { + // Never invoked: each directive fails typecheck once its entry point widens back to an + // unbranded or optional surface. + const widenedCalls = async (out: string) => { + // @ts-expect-error a bare literal skips the routing owner + await runMacOsSnapshotAction('desktop'); + // @ts-expect-error a bare literal skips the routing owner + await runMacOsScreenshotAction(out, { surface: 'menubar' }); + // @ts-expect-error the surface is required + await runMacOsReadTextAction(1, 2, { bundleId: 'com.example' }); + // @ts-expect-error the surface is required + await runMacOsPressAction(1, 2, {}); + // @ts-expect-error the surface is required + await runMacOsScreenshotAction(out); + }; + assert.equal(typeof widenedCalls, 'function'); +}); diff --git a/packages/platform-apple/src/os/macos/helper.ts b/packages/platform-apple/src/os/macos/helper.ts index b807ab1589..599a51e82a 100644 --- a/packages/platform-apple/src/os/macos/helper.ts +++ b/packages/platform-apple/src/os/macos/helper.ts @@ -25,7 +25,7 @@ import { runCmdBackground, type ExecBackgroundResult, } from '@agent-device/host-kit/command'; -import type { SessionSurface } from '@agent-device/contracts/session'; +import type { MacOsHelperSurface, SessionSurface } from '@agent-device/contracts/session'; import { hasScopedAppleToolProvider, resolveAppleToolProvider, @@ -372,10 +372,10 @@ export async function runMacOsAlertAction( } export async function runMacOsSnapshotAction( - surface: Exclude, + surface: MacOsHelperSurface, options: { bundleId?: string; signal?: AbortSignal } = {}, ): Promise<{ - surface: Exclude; + surface: SessionSurface; nodes: MacOsSnapshotNode[]; truncated: boolean; backend: 'macos-helper'; @@ -388,7 +388,7 @@ export async function runMacOsSnapshotAction( export async function runMacOsReadTextAction( x: number, y: number, - options: { bundleId?: string; surface?: SessionSurface } = {}, + options: { surface: MacOsHelperSurface; bundleId?: string }, ): Promise<{ text: string; }> { @@ -430,8 +430,8 @@ export async function runMacOsPressAction( x: number, y: number, options: { + surface: MacOsHelperSurface; bundleId?: string; - surface?: SessionSurface; holdMs?: number; /** Independent presses, each a single click; `--count` on every platform. */ clicks?: number; @@ -439,7 +439,7 @@ export async function runMacOsPressAction( doubleClick?: boolean; intervalMs?: number; signal?: AbortSignal; - } = {}, + }, ): Promise<{ x: number; y: number; @@ -473,7 +473,7 @@ export async function runMacOsPressAction( export async function runMacOsScreenshotAction( outPath: string, - options: { surface?: SessionSurface } = {}, + options: { surface: MacOsHelperSurface }, ): Promise<{ path: string; surface?: SessionSurface; diff --git a/packages/platform-apple/src/os/macos/surface-snapshot.ts b/packages/platform-apple/src/os/macos/surface-snapshot.ts index 47d0899eb7..28df3a9693 100644 --- a/packages/platform-apple/src/os/macos/surface-snapshot.ts +++ b/packages/platform-apple/src/os/macos/surface-snapshot.ts @@ -1,16 +1,16 @@ import type { CaptureSnapshotInput } from '@agent-device/contracts/snapshot-runtime'; +import type { MacOsHelperSurface } from '@agent-device/contracts/session'; import { shapeDesktopSurfaceSnapshot } from '@agent-device/capture-kit/snapshot-desktop-projection'; -type SnapshotSurfaceOptions = NonNullable; +type SnapshotSurfaceOptions = Omit, 'surface'> & { + surface: MacOsHelperSurface; +}; export async function captureMacOsSurfaceSnapshot( options: SnapshotSurfaceOptions, signal?: AbortSignal, ) { const surface = options.surface; - if (!surface || surface === 'app') { - throw new TypeError('Apple surface capture requires a non-app macOS surface'); - } const { runMacOsSnapshotAction } = await import('./helper.ts'); const result = await runMacOsSnapshotAction(surface, { bundleId: surface === 'menubar' ? options.appBundleId : undefined, diff --git a/packages/platform-apple/src/runtime-snapshot.ts b/packages/platform-apple/src/runtime-snapshot.ts index e394c48ab0..c3608e6b49 100644 --- a/packages/platform-apple/src/runtime-snapshot.ts +++ b/packages/platform-apple/src/runtime-snapshot.ts @@ -11,6 +11,7 @@ import type { PlatformRuntimeHost, PlatformRuntimeOperations, } from '@agent-device/contracts/platform-runtime-operations'; +import { macOsSurfaceBackend, type SessionSurface } from '@agent-device/contracts/session'; import { isMacOs, type DeviceInfo } from '@agent-device/kernel/device'; import { hasSimulatorBridge } from './snapshot-observability.ts'; import type { AppleSnapshotRoute } from './snapshot-route.ts'; @@ -27,11 +28,7 @@ export function bindAppleSnapshotRuntime( resolveInteractor: host.localInteractors.resolve, }); const captureSnapshot = async (input: CaptureSnapshotInput) => { - if ( - isMacOs(request.device) && - input.options?.surface !== undefined && - input.options.surface !== 'app' - ) { + if (isMacOs(request.device) && macOsSurfaceBackend(input.options?.surface) === 'macos-helper') { return await host.snapshot.captureSurface( request.device, input.options, @@ -66,7 +63,7 @@ type SnapshotRuntimeOperation = Pick< * * - No tracked app bundle id: the runner query is scoped to an application, so there is nothing * to ask about. - * - macOS on an explicit non-app surface: the runner reads the *application*, so a positive + * - macOS on a helper-routed surface: the runner reads the *application*, so a positive * answer would describe the wrong surface. Reporting `false` sends the poll to the desktop * surface capture, which is the reading that matches the request. * @@ -106,15 +103,16 @@ async function admitAppleNativeFind( host: Pick, request: Readonly<{ device: DeviceInfo; signal: AbortSignal }>, input: Readonly<{ - options?: Readonly<{ appBundleId?: string; surface?: string }>; + options?: Readonly<{ appBundleId?: string; surface?: SessionSurface }>; execution?: Readonly<{ requestId?: string }>; signal?: AbortSignal; }>, ): Promise { const appBundleId = input.options?.appBundleId; if (appBundleId === undefined) return undefined; - const surface = input.options?.surface; - if (isMacOs(request.device) && surface !== undefined && surface !== 'app') return undefined; + if (isMacOs(request.device) && macOsSurfaceBackend(input.options?.surface) === 'macos-helper') { + return undefined; + } const signal = input.signal ? AbortSignal.any([request.signal, input.signal]) : request.signal; signal.throwIfAborted(); if (!(await runnerCanAnswerNow(host, request.device, input.execution))) return undefined; diff --git a/packages/platform-apple/src/runtime.test.ts b/packages/platform-apple/src/runtime.test.ts index fcd1b6ed25..233190dda6 100644 --- a/packages/platform-apple/src/runtime.test.ts +++ b/packages/platform-apple/src/runtime.test.ts @@ -9,6 +9,11 @@ import { listIosApps } from './core/app-resolution.ts'; import type { DeviceBinding, RuntimeFacts } from '@agent-device/contracts/platform-runtime'; import type { PlatformRuntimeOperations } from '@agent-device/contracts/platform-runtime-operations'; import type { SnapshotRuntimeHost } from '@agent-device/contracts/snapshot-runtime'; +import { + SESSION_SURFACES, + type MacOsSurfaceBackend, + type SessionSurface, +} from '@agent-device/contracts/session'; import { HOVER_UNAVAILABLE_HINT } from '@agent-device/contracts/touch-runtime'; import type { AppleOS, DeviceInfo } from '@agent-device/kernel/device'; import { createApplePlatformRuntime } from './runtime.ts'; @@ -396,9 +401,19 @@ function expectTvRemoteFact( } } -test.each(['frontmost-app', 'desktop', 'menubar'] as const)( - 'routes the macOS %s surface through the exact Apple surface host', - async (surface) => { +const MACOS_SURFACE_BACKENDS: Record = { + app: 'xctest', + 'frontmost-app': 'macos-helper', + desktop: 'macos-helper', + menubar: 'macos-helper', +}; + +test.each([ + ...SESSION_SURFACES.map((surface) => [surface, MACOS_SURFACE_BACKENDS[surface]] as const), + [undefined, 'xctest'] as const, +])( + 'the macOS %s surface captures and finds text through the %s backend', + async (surface, backend) => { const host = platformRuntimeHostFixture(); const captureSurface = vi.fn(async () => ({ backend: 'macos-helper' as const, @@ -406,7 +421,14 @@ test.each(['frontmost-app', 'desktop', 'menubar'] as const)( nodes: [], truncated: false, })); - const resolve = vi.fn(async () => ({}) as never); + const snapshot = vi.fn(async () => ({ + backend: 'xctest' as const, + producer: 'apple-runner' as const, + nodes: [], + truncated: false, + })); + const findText = vi.fn(async () => ({ found: true })); + const resolve = vi.fn(async () => ({ snapshot, findText }) as never); const binding = await createApplePlatformRuntime({ ...host, localInteractors: { resolve }, @@ -420,18 +442,18 @@ test.each(['frontmost-app', 'desktop', 'menubar'] as const)( progress: { report: () => {} }, }, }); + const options = { surface, appBundleId: 'com.example.app', depth: 3 }; - await expect( - binding.operations.captureSnapshot?.({ - options: { surface, appBundleId: 'com.example.app', depth: 3 }, - }), - ).resolves.toMatchObject({ backend: 'macos-helper' }); - expect(captureSurface).toHaveBeenCalledWith( - leaves.macos, - { surface, appBundleId: 'com.example.app', depth: 3 }, - expect.any(AbortSignal), + await binding.operations.captureSnapshot?.({ options }); + const found = await binding.operations.findText?.({ text: 'Settings', options }); + + const helperRouted = backend === 'macos-helper'; + expect(captureSurface.mock.calls).toEqual( + helperRouted ? [[leaves.macos, options, expect.any(AbortSignal)]] : [], ); - expect(resolve).not.toHaveBeenCalled(); + expect(snapshot).toHaveBeenCalledTimes(helperRouted ? 0 : 1); + expect(findText).toHaveBeenCalledTimes(helperRouted ? 0 : 1); + expect(found).toEqual({ found: !helperRouted }); }, ); diff --git a/src/daemon/__tests__/screenshot-crop-target.test.ts b/src/daemon/__tests__/screenshot-crop-target.test.ts index 7eb0fcc91b..0b9ef7f36f 100644 --- a/src/daemon/__tests__/screenshot-crop-target.test.ts +++ b/src/daemon/__tests__/screenshot-crop-target.test.ts @@ -9,7 +9,11 @@ import { WEB_DESKTOP_DEVICE, } from '../../__tests__/test-utils/device-fixtures.ts'; import { SCREENSHOT_CROP_REASONS } from '@agent-device/contracts/capture'; -import type { SessionSurface } from '@agent-device/contracts/session'; +import { + SESSION_SURFACES, + type MacOsSurfaceBackend, + type SessionSurface, +} from '@agent-device/contracts/session'; import type { DeviceInfo } from '@agent-device/kernel/device'; import { expect, test } from 'vitest'; import { @@ -31,7 +35,7 @@ const CROP_TARGET_DEVICES: readonly CropTargetDevice[] = [ { target: 'android-device', device: ANDROID_DEVICE, surface: undefined }, { target: 'macos-app-window', device: MACOS_DEVICE, surface: 'app' }, { target: 'ios-physical', device: IOS_DEVICE, surface: undefined }, - { target: 'macos-helper', device: MACOS_DEVICE, surface: undefined }, + { target: 'macos-helper', device: MACOS_DEVICE, surface: 'desktop' }, { target: 'web', device: WEB_DESKTOP_DEVICE, surface: undefined }, { target: 'linux', device: LINUX_DEVICE, surface: undefined }, { target: 'tvos', device: TVOS_SIMULATOR, surface: undefined }, @@ -67,6 +71,24 @@ test('the classifier and the acceptance matrix agree one-to-one, and the accepte } }); +const MACOS_SURFACE_BACKENDS: Record = { + app: 'xctest', + 'frontmost-app': 'macos-helper', + desktop: 'macos-helper', + menubar: 'macos-helper', +}; +const MACOS_CROP_TARGETS: Record = { + xctest: 'macos-app-window', + 'macos-helper': 'macos-helper', +}; + +test.each([ + ...SESSION_SURFACES.map((surface) => [surface, MACOS_SURFACE_BACKENDS[surface]] as const), + [undefined, 'xctest'] as const, +])('a macOS %s session crops in the frame of the backend that captures it', (surface, backend) => { + expect(classifyScreenshotCropTarget(MACOS_DEVICE, surface)).toBe(MACOS_CROP_TARGETS[backend]); +}); + test('an apple device with an unpopulated reserved OS is a typed refusal, not a guess', () => { const device: DeviceInfo = { platform: 'apple', diff --git a/src/daemon/screenshot-crop-target.ts b/src/daemon/screenshot-crop-target.ts index 066f68ca2d..8d71e342cf 100644 --- a/src/daemon/screenshot-crop-target.ts +++ b/src/daemon/screenshot-crop-target.ts @@ -2,7 +2,7 @@ import { SCREENSHOT_CROP_REASONS, type ScreenshotCropReason, } from '@agent-device/contracts/capture'; -import type { SessionSurface } from '@agent-device/contracts/session'; +import { macOsSurfaceBackend, type SessionSurface } from '@agent-device/contracts/session'; import { resolveDeviceAppleOs, type DeviceInfo } from '@agent-device/kernel/device'; import { AppError } from '@agent-device/kernel/errors'; import { validateSelectorExpression } from '@agent-device/selectors'; @@ -99,7 +99,7 @@ function classifyAppleCropTarget( } function classifyMacOsCropTarget(surface: SessionSurface | undefined): CropTarget { - return surface === 'app' || surface === 'frontmost-app' ? 'macos-app-window' : 'macos-helper'; + return macOsSurfaceBackend(surface) === 'macos-helper' ? 'macos-helper' : 'macos-app-window'; } function cropRefusal(target: string, rejectionReason?: ScreenshotCropReason): AppError { diff --git a/src/platform-runtime-operation-host.test.ts b/src/platform-runtime-operation-host.test.ts index 07d0ed9deb..d8611ab95f 100644 --- a/src/platform-runtime-operation-host.test.ts +++ b/src/platform-runtime-operation-host.test.ts @@ -19,8 +19,21 @@ vi.mock('./platform-runtime-apple-tool-host.ts', () => ({ vi.mock('./platform-runtime-toolchain-host.ts', () => ({ createHostToolchainPreparer: () => capabilities.toolchains, })); +vi.mock('@agent-device/platform-apple/macos', async (importOriginal) => ({ + ...(await importOriginal()), + captureMacOsSurfaceSnapshot: vi.fn(async () => ({ + backend: 'macos-helper' as const, + producer: 'macos-helper' as const, + nodes: [], + truncated: false, + })), +})); -import { createPlatformRuntimeHost } from './platform-runtime-operation-host.ts'; +import { captureMacOsSurfaceSnapshot } from '@agent-device/platform-apple/macos'; +import { + createPlatformRuntimeHost, + loadMacOsSurfaceSnapshot, +} from './platform-runtime-operation-host.ts'; const snapshot = { captureSurface: async () => ({ @@ -71,3 +84,26 @@ test('composes focused deployment executors instead of a cross-family deployment expect(source).not.toContain('appDeployment:'); expect(existsSync(join(directory, 'platform-runtime-app-deployment-host.ts'))).toBe(false); }); + +test.each([undefined, 'app'] as const)( + 'the macOS surface loader refuses a %s surface the owner routes to the runner', + async (surface) => { + vi.mocked(captureMacOsSurfaceSnapshot).mockClear(); + const refusal = loadMacOsSurfaceSnapshot({ surface }); + await expect(refusal).rejects.toBeInstanceOf(TypeError); + await expect(refusal).rejects.toThrow( + 'Apple surface capture requires a helper-routed macOS surface', + ); + expect(captureMacOsSurfaceSnapshot).not.toHaveBeenCalled(); + }, +); + +test('the macOS surface loader forwards a helper-routed surface unchanged', async () => { + vi.mocked(captureMacOsSurfaceSnapshot).mockClear(); + const signal = new AbortController().signal; + await loadMacOsSurfaceSnapshot({ surface: 'frontmost-app', depth: 2 }, signal); + expect(captureMacOsSurfaceSnapshot).toHaveBeenCalledWith( + { surface: 'frontmost-app', depth: 2 }, + signal, + ); +}); diff --git a/src/platform-runtime-operation-host.ts b/src/platform-runtime-operation-host.ts index cf005779a1..309b7fafec 100644 --- a/src/platform-runtime-operation-host.ts +++ b/src/platform-runtime-operation-host.ts @@ -5,6 +5,7 @@ import type { OwnedProcessRecordWriter, } from '@agent-device/contracts/platform-runtime-host'; import type { PlatformRuntimeHost } from '@agent-device/contracts/platform-runtime-operations'; +import { macOsHelperSurface } from '@agent-device/contracts/session'; import type { CaptureSnapshotInput, SnapshotResult, @@ -41,8 +42,12 @@ export async function loadMacOsSurfaceSnapshot( options: CaptureSnapshotInput['options'], signal?: AbortSignal, ): Promise { + const surface = macOsHelperSurface(options?.surface); + if (!surface) { + throw new TypeError('Apple surface capture requires a helper-routed macOS surface'); + } const { captureMacOsSurfaceSnapshot } = await import('@agent-device/platform-apple/macos'); - return await captureMacOsSurfaceSnapshot(options ?? {}, signal); + return await captureMacOsSurfaceSnapshot({ ...options, surface }, signal); } export function createPlatformRuntimeHost(options: {