Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions packages/kernel/src/device.ts
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,19 @@ export function isTvOsDevice(device: Pick<DeviceInfo, 'platform' | 'target'>): b
return isApplePlatform(device.platform) && device.target === 'tv';
}

/**
* The Apple leaves whose runner synthesizes tap input (`RunnerTests+SynthesizedInteraction.swift`
* gates two-finger HID synthesis behind `#if os(iOS)`, which covers iOS and iPadOS only): every
* `isIosFamily` leaf except tvOS (no touchscreen) and visionOS (the runner's `#else` branch, no
* synthesis path). Every producer of `synthesized: true` gates on this predicate so none of them
* pays for a synthesis attempt the runner cannot perform.
*/
export function runnerSynthesizesTap(
device: Pick<DeviceInfo, 'platform' | 'appleOs' | 'target'>,
): boolean {
return isIosFamily(device) && !isTvOsDevice(device) && device.appleOs !== 'visionos';
}

/** Resolve the stored Apple OS, preserving legacy target/leaf inference for old device records. */
export function resolveDeviceAppleOs(
device: Pick<DeviceInfo, 'platform' | 'target' | 'appleOs'>,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,3 +34,12 @@ export const TVOS_TEST_SIMULATOR: DeviceInfo = {
target: 'tv',
booted: true,
};

export const VISIONOS_TEST_SIMULATOR: DeviceInfo = {
platform: 'apple',
appleOs: 'visionos',
id: 'visionos-sim-1',
name: 'Apple Vision Pro',
kind: 'simulator',
booted: true,
};
72 changes: 72 additions & 0 deletions packages/platform-apple/src/core/__tests__/interactions.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import {
IOS_TEST_SIMULATOR,
MACOS_TEST_DEVICE,
TVOS_TEST_SIMULATOR,
VISIONOS_TEST_SIMULATOR,
} from './apple-core-stub-helpers.ts';

vi.mock('../runner-client.ts', async (importOriginal) => {
Expand Down Expand Up @@ -141,6 +142,77 @@ test('iosRunnerOverrides uses synthesized iOS coordinate taps for selectors', as
});
});

test('iosRunnerOverrides does not request synthesis for visionOS coordinate taps', async () => {
mockRunAppleRunnerCommand.mockResolvedValue({});

const { overrides } = iosRunnerOverrides(VISIONOS_TEST_SIMULATOR, {
appBundleId: 'com.example.App',
});

await overrides.tap(100, 200);

assert.deepEqual(mockRunAppleRunnerCommand.mock.calls[0]?.[1], {
command: 'tap',
x: 100,
y: 200,
appBundleId: 'com.example.App',
});
});

test('iosRunnerOverrides does not request synthesis for visionOS selector taps', async () => {
mockRunAppleRunnerCommand.mockResolvedValue({});

const { overrides } = iosRunnerOverrides(VISIONOS_TEST_SIMULATOR, {
appBundleId: 'com.example.App',
});

await overrides.tapElementSelector!({
key: 'label',
value: 'General',
expectedPoint: { x: 200, y: 300 },
});

assert.deepEqual(mockRunAppleRunnerCommand.mock.calls[0]?.[1], {
command: 'tap',
selectorKey: 'label',
selectorValue: 'General',
allowNonHittableCoordinateFallback: undefined,
x: 200,
y: 300,
appBundleId: 'com.example.App',
});
});

test('iosRunnerOverrides does not request synthesis for visionOS fused presses', async () => {
mockRunAppleRunnerCommand.mockResolvedValue({
completedSteps: 3,
sequenceResults: Array.from({ length: 3 }, () => ({ ok: true, kind: 'tap' })),
});
const { overrides } = iosRunnerOverrides(VISIONOS_TEST_SIMULATOR, {
appBundleId: 'com.example.App',
});

await overrides.pressPoint!(
{ x: 100, y: 200 },
{
button: 'primary',
count: 3,
intervalMs: 40,
holdMs: 0,
jitterPx: 2,
doubleTap: false,
},
);

const command = mockRunAppleRunnerCommand.mock.calls[0]?.[1] as RunnerCommand;
assert.equal(command.command, 'sequence');
assert.deepEqual(command.steps, [
{ kind: 'tap', x: 100, y: 200, pauseMs: 40 },
{ kind: 'tap', x: 102, y: 200, pauseMs: 40 },
{ kind: 'tap', x: 100, y: 202 },
]);
});

test('iosRunnerOverrides owns fused repeated presses with deterministic jitter', async () => {
mockRunAppleRunnerCommand.mockResolvedValue({
completedSteps: 3,
Expand Down
16 changes: 8 additions & 8 deletions packages/platform-apple/src/interactions.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,12 @@ import {
assertScrollGestureInput,
} from '@agent-device/contracts/scroll-gesture';
import { assertAppleMultiTouchSupported } from './multitouch-support.ts';
import { isIosFamily, isMacOs, isTvOsDevice, type DeviceInfo } from '@agent-device/kernel/device';
import {
isMacOs,
isTvOsDevice,
runnerSynthesizesTap,
type DeviceInfo,
} from '@agent-device/kernel/device';
import { AppError } from '@agent-device/kernel/errors';
import { runAppleRunnerCommand, runApplePressSeries } from './core/runner-client.ts';
import {
Expand Down Expand Up @@ -99,7 +104,7 @@ export function iosRunnerOverrides(
...(selector.expectedPoint
? { x: selector.expectedPoint.x, y: selector.expectedPoint.y }
: {}),
...(shouldUseSynthesizedIosGesture(device) ? { synthesized: true } : {}),
...(runnerSynthesizesTap(device) ? { synthesized: true } : {}),
appBundleId: ctx.appBundleId,
},
runnerOpts,
Expand Down Expand Up @@ -386,16 +391,11 @@ function iosTapCommand(
command: 'tap',
x,
y,
...(shouldUseSynthesizedIosGesture(device) ? { synthesized: true } : {}),
...(runnerSynthesizesTap(device) ? { synthesized: true } : {}),
appBundleId: ctx.appBundleId,
};
}

function shouldUseSynthesizedIosGesture(device: DeviceInfo): boolean {
// Two-finger HID synthesis is for touch-input iOS only; the tvOS leaf has no touch.
return isIosFamily(device) && !isTvOsDevice(device);
}

async function runAppleScroll(
runRunnerCommand: RunAppleRunnerCommand,
device: DeviceInfo,
Expand Down
4 changes: 2 additions & 2 deletions packages/platform-apple/src/runner/runner-sequence.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import type { PressPointOptions } from '@agent-device/contracts/interactor-types';
import { pressJitter } from '@agent-device/contracts/touch-runtime';
import { isIosFamily, isTvOsDevice, type DeviceInfo } from '@agent-device/kernel/device';
import { runnerSynthesizesTap, type DeviceInfo } from '@agent-device/kernel/device';
import { AppError, toAppErrorCode } from '@agent-device/kernel/errors';
import type { RunnerCommand, RunnerSequenceStep } from './runner-contract.ts';

Expand Down Expand Up @@ -216,7 +216,7 @@ function buildPressSteps(
options: PressPointOptions,
): RunnerSequenceStep[] {
const kind = options.doubleTap ? 'doubleTap' : options.holdMs > 0 ? 'longPress' : 'tap';
const synthesized = kind === 'tap' && isIosFamily(device) && !isTvOsDevice(device);
const synthesized = kind === 'tap' && runnerSynthesizesTap(device);
return Array.from({ length: options.count }, (_, index) => {
const [dx, dy] = pressJitter(index, options.jitterPx);
return {
Expand Down
14 changes: 14 additions & 0 deletions src/core/__tests__/device.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,16 @@ import {
resolveApplePlatformName,
resolveAppleSimulatorSetPathForSelector,
resolveDevice,
runnerSynthesizesTap,
} from '@agent-device/kernel/device';
import type { DeviceInfo } from '@agent-device/kernel/device';
import {
ANDROID_TV_DEVICE,
IOS_SIMULATOR,
IPADOS_SIMULATOR,
MACOS_DEVICE,
TVOS_SIMULATOR,
VISIONOS_SIMULATOR,
} from '../../__tests__/test-utils/device-fixtures.ts';
import { AppError } from '@agent-device/kernel/errors';

Expand All @@ -28,6 +31,17 @@ test('isTvOsDevice selects only the Apple tvOS leaf, not any TV target', () => {
assert.equal(isTvOsDevice(ANDROID_TV_DEVICE), false);
});

test('runnerSynthesizesTap selects only the leaves whose runner has a synthesis path', () => {
// iOS and iPadOS: the runner's `#if os(iOS)` branch synthesizes.
assert.equal(runnerSynthesizesTap(IOS_SIMULATOR), true);
assert.equal(runnerSynthesizesTap(IPADOS_SIMULATOR), true);
// tvOS has no touchscreen; visionOS's runner has no synthesis path (falls to the
// Swift `#else` branch) — neither may carry `synthesized: true`.
assert.equal(runnerSynthesizesTap(TVOS_SIMULATOR), false);
assert.equal(runnerSynthesizesTap(VISIONOS_SIMULATOR), false);
assert.equal(runnerSynthesizesTap(MACOS_DEVICE), false);
});

test('matchesPlatformSelector resolves apple selector across Apple platforms', () => {
assert.equal(matchesPlatformSelector({ platform: 'apple', appleOs: 'ios' }, 'apple'), true);
assert.equal(matchesPlatformSelector({ platform: 'apple', appleOs: 'macos' }, 'apple'), true);
Expand Down
Loading