From 05b150eac388dbd29b0498aa201e48773aa43bf0 Mon Sep 17 00:00:00 2001 From: Rohit <40559587+Rohit3523@users.noreply.github.com> Date: Tue, 22 Sep 2026 03:17:20 +0530 Subject: [PATCH 001/132] feat(maestro): support killApp via close mode Maestro killApp triggers system-initiated process death (adb shell am kill on Android) instead of stopApp's force-stop. On other platforms it aliases stopApp through the shared close dispatcher. - contracts: optional Interactor.kill, CloseApplicationInput.mode, with fallback to close in invokeApplicationClose - android: killAndroidApp (am kill) wired onto the interactor - maestro: killApp IR kind, parser, runtime port, daemon projection to an app-only close carrying killApp dispatch, conformance canonical - daemon: replay dispatch folds killApp; session close passes mode kill into closeApplication - upstream/116_kill_app now classifies identical; divergence removed --- .../application-lifecycle-interaction.test.ts | 47 +++++++++++++++++ .../src/application-lifecycle-interaction.ts | 8 ++- .../src/application-lifecycle-runtime.ts | 6 +++ packages/contracts/src/interactor-types.ts | 6 +++ packages/contracts/src/replay.ts | 5 ++ .../__tests__/daemon-runtime-port.test.ts | 41 +++++++++++++++ .../daemon-runtime-public-operation.test.ts | 16 ++++++ .../src/daemon-port/daemon-runtime-port.ts | 4 ++ .../daemon-runtime-public-operation.ts | 22 +++++++- .../__tests__/program-ir-parser.test.ts | 21 ++++++++ .../__tests__/runtime-port-fixtures.ts | 1 + .../internal/__tests__/runtime-port.test.ts | 19 +++++++ .../src/internal/conformance-normalize.ts | 7 ++- .../src/internal/program-ir-command-parser.ts | 12 +++++ packages/maestro/src/internal/program-ir.ts | 7 +++ .../src/internal/runtime-port-commands.ts | 11 +++- .../src/internal/runtime-port-types.ts | 1 + .../maestro/src/internal/support-matrix.ts | 2 +- .../test/conformance/expected-divergence.ts | 5 -- .../src/__tests__/app-lifecycle-open.test.ts | 43 +++++++++++++++- .../platform-android/src/app-lifecycle.ts | 20 ++++++++ packages/platform-android/src/lifecycle.ts | 1 + packages/platform-android/src/mechanics.ts | 1 + .../fuzz/validation-arbitraries-maestro.ts | 2 + src/core/interactors/android.ts | 2 + .../session-replay-maestro-request.test.ts | 13 +++++ .../session-replay-maestro-request.ts | 1 + .../session-close-lifecycle-runtime.test.ts | 50 +++++++++++++++++++ .../internal/session-close.ts | 1 + website/docs/docs/replay-e2e.md | 2 +- 30 files changed, 364 insertions(+), 13 deletions(-) diff --git a/packages/contracts/src/application-lifecycle-interaction.test.ts b/packages/contracts/src/application-lifecycle-interaction.test.ts index 2e42310f7c..662aca9474 100644 --- a/packages/contracts/src/application-lifecycle-interaction.test.ts +++ b/packages/contracts/src/application-lifecycle-interaction.test.ts @@ -5,6 +5,7 @@ import type { OpenApplicationInput } from './application-lifecycle-runtime.ts'; import { bindDirectApplicationLifecycle, bindLocalApplicationLifecycleInteractor, + invokeApplicationClose, invokeApplicationOpen, } from './application-lifecycle-interaction.ts'; @@ -91,6 +92,52 @@ test('direct lifecycle owners preserve the daemon runtime launch URL follow-up', expect(calls[1]?.options).toHaveProperty('launchArgs', undefined); }); +test('kill mode dispatches Interactor.kill when the owner implements it', async () => { + const calls: string[] = []; + const interactor: Interactor = { + ...interactorWithOpen(), + close: async (app) => { + calls.push(`close:${app}`); + }, + kill: async (app) => { + calls.push(`kill:${app}`); + }, + }; + + await invokeApplicationClose({ + device: LINUX_DEVICE, + interactor, + positionals: ['com.example.app'], + mode: 'kill', + }); + + expect(calls).toEqual(['kill:com.example.app']); +}); + +test('kill mode falls back to close when the owner has no kill', async () => { + const calls: string[] = []; + const interactor: Interactor = { + ...interactorWithOpen(), + close: async (app) => { + calls.push(`close:${app}`); + }, + }; + + await invokeApplicationClose({ + device: LINUX_DEVICE, + interactor, + positionals: ['com.example.app'], + mode: 'kill', + }); + await invokeApplicationClose({ + device: LINUX_DEVICE, + interactor, + positionals: ['com.example.app'], + }); + + expect(calls).toEqual(['close:com.example.app', 'close:com.example.app']); +}); + test.each([ { name: 'more than two positionals', diff --git a/packages/contracts/src/application-lifecycle-interaction.ts b/packages/contracts/src/application-lifecycle-interaction.ts index 0db4ce162f..c82d7ce814 100644 --- a/packages/contracts/src/application-lifecycle-interaction.ts +++ b/packages/contracts/src/application-lifecycle-interaction.ts @@ -370,13 +370,19 @@ export async function invokeApplicationClose( device: DeviceInfo; interactor: Interactor; positionals: readonly string[]; + /** `kill` dispatches `Interactor.kill` with fallback to `close`; absent means `stop`. */ + mode?: 'stop' | 'kill'; }>, ): Promise { - const { device, interactor, positionals } = params; + const { device, interactor, positionals, mode } = params; const app = positionals[0]; if (!app) { if (device.platform === 'web') await interactor.close(''); return; } + if (mode === 'kill' && interactor.kill) { + await interactor.kill(app); + return; + } await interactor.close(app); } diff --git a/packages/contracts/src/application-lifecycle-runtime.ts b/packages/contracts/src/application-lifecycle-runtime.ts index 58304a5235..73d1b5c78e 100644 --- a/packages/contracts/src/application-lifecycle-runtime.ts +++ b/packages/contracts/src/application-lifecycle-runtime.ts @@ -148,6 +148,12 @@ export type CloseApplicationInput = Readonly<{ surface: SessionSurface; /** A selector-only close establishes readiness inside its admitted lifecycle binding. */ ensureReady?: boolean; + /** + * `kill` is Maestro `killApp` (system-initiated process death: `am kill` on + * Android); absent means `stop`. Owners without an `Interactor.kill` alias + * it to `close` in the shared dispatcher. + */ + mode?: 'stop' | 'kill'; execution: ApplicationLifecycleExecution; }>; diff --git a/packages/contracts/src/interactor-types.ts b/packages/contracts/src/interactor-types.ts index 77f3982c58..d63cee6025 100644 --- a/packages/contracts/src/interactor-types.ts +++ b/packages/contracts/src/interactor-types.ts @@ -298,6 +298,12 @@ export type Interactor = { ): Promise; openDevice(): Promise; close(app: string): Promise; + /** + * System-initiated process death (Maestro `killApp`): on Android `am kill` + * rather than `am force-stop`. Owners without a lighter-weight kill leave + * it undefined and the shared close dispatcher falls back to `close(app)`. + */ + kill?(app: string): Promise; tap(x: number, y: number): Promise | void>; /** Complete point-press semantics for owners with fused series, alternate buttons, or surfaces. */ pressPoint?(point: Point, options: PressPointOptions): Promise | void>; diff --git a/packages/contracts/src/replay.ts b/packages/contracts/src/replay.ts index fab9e8d8c8..8ddd81e9c2 100644 --- a/packages/contracts/src/replay.ts +++ b/packages/contracts/src/replay.ts @@ -277,6 +277,11 @@ export type ReplayDispatchOptions = Readonly<{ openLifecycle?: ReplayOpenLifecycle; /** Terminate the targeted app without ending the owning daemon session. */ closeAppOnly?: boolean; + /** + * With `closeAppOnly`: Maestro `killApp` system-initiated process death + * (`am kill` on Android, `stopApp` alias elsewhere) instead of `stop`. + */ + killApp?: boolean; /** * Daemon-composed hierarchy capture used as operational evidence only. It must not issue or * replace client ref authority. diff --git a/packages/maestro/src/daemon-port/__tests__/daemon-runtime-port.test.ts b/packages/maestro/src/daemon-port/__tests__/daemon-runtime-port.test.ts index dccc465176..c505138a71 100644 --- a/packages/maestro/src/daemon-port/__tests__/daemon-runtime-port.test.ts +++ b/packages/maestro/src/daemon-port/__tests__/daemon-runtime-port.test.ts @@ -173,6 +173,47 @@ test('projects standalone clearState to settings without opening the app', async ]); }); +test('projects standalone killApp to an app-only close carrying the kill mode', async () => { + const requests: MaestroDaemonOperationRequest[] = []; + const invoke: MaestroDaemonOperationInvoke = async (request) => { + requests.push(request); + return { ok: true, data: {} }; + }; + const port = createDaemonMaestroRuntimePort({ + ...makeRuntimeEnvelope({ flags: { platform: 'android', replayBackend: 'maestro' } }), + invoke, + dependencies: makeDependencies(), + platform: 'android', + }); + + await port.execute({ + command: { kind: 'killApp', source: { line: 2 }, appId: 'com.example.app' }, + generation: 0, + env: {}, + invalidateObservation() {}, + }); + await port.execute({ + command: { kind: 'killApp', source: { line: 3 } }, + generation: 1, + env: {}, + appId: 'com.example.session', + invalidateObservation() {}, + }); + + expect(requests).toEqual([ + expect.objectContaining({ + command: 'close', + positionals: ['com.example.app'], + dispatch: { closeAppOnly: true, killApp: true }, + }), + expect.objectContaining({ + command: 'close', + positionals: ['com.example.session'], + dispatch: { closeAppOnly: true, killApp: true }, + }), + ]); +}); + test('uses the direct viewport without snapshot and pairs it with the nested gesture request', async () => { const requests: MaestroDaemonOperationRequest[] = []; const viewport = { x: 10, y: 20, width: 400, height: 800 }; diff --git a/packages/maestro/src/daemon-port/__tests__/daemon-runtime-public-operation.test.ts b/packages/maestro/src/daemon-port/__tests__/daemon-runtime-public-operation.test.ts index ad9409bf02..1f45660a6a 100644 --- a/packages/maestro/src/daemon-port/__tests__/daemon-runtime-public-operation.test.ts +++ b/packages/maestro/src/daemon-port/__tests__/daemon-runtime-public-operation.test.ts @@ -49,6 +49,22 @@ describe('Maestro public operation projection', () => { operation: { kind: 'stopApp' }, expected: { command: 'close', positionals: [], dispatch: { closeAppOnly: true } }, }, + { + operation: { kind: 'killApp', appId: 'com.example' }, + expected: { + command: 'close', + positionals: ['com.example'], + dispatch: { closeAppOnly: true, killApp: true }, + }, + }, + { + operation: { kind: 'killApp' }, + expected: { + command: 'close', + positionals: [], + dispatch: { closeAppOnly: true, killApp: true }, + }, + }, { operation: { kind: 'clearState', appId: 'com.example' }, expected: { command: 'settings', positionals: ['clear-app-state', 'com.example'] }, diff --git a/packages/maestro/src/daemon-port/daemon-runtime-port.ts b/packages/maestro/src/daemon-port/daemon-runtime-port.ts index d2f8917c6a..65788203ff 100644 --- a/packages/maestro/src/daemon-port/daemon-runtime-port.ts +++ b/packages/maestro/src/daemon-port/daemon-runtime-port.ts @@ -189,6 +189,10 @@ function createDaemonMaestroRuntimeParts(options: CreateDaemonMaestroRuntimeOper const appId = input.appId ?? context.appId; await invokeMutation({ kind: 'stopApp', ...(appId ? { appId } : {}) }, context); }, + killApp: async (input, context) => { + const appId = input.appId ?? context.appId; + await invokeMutation({ kind: 'killApp', ...(appId ? { appId } : {}) }, context); + }, setPermissions: async (input, context) => { await applyPermissionMutations( input.appId ?? context.appId, diff --git a/packages/maestro/src/daemon-port/daemon-runtime-public-operation.ts b/packages/maestro/src/daemon-port/daemon-runtime-public-operation.ts index 9016723661..d30915bf52 100644 --- a/packages/maestro/src/daemon-port/daemon-runtime-public-operation.ts +++ b/packages/maestro/src/daemon-port/daemon-runtime-public-operation.ts @@ -21,6 +21,7 @@ export type MaestroPublicOperation = launchArgs: string[]; } | { kind: 'stopApp'; appId?: string } + | { kind: 'killApp'; appId?: string } | { kind: 'clearState'; appId?: string } | { kind: 'settingsPermission'; @@ -53,6 +54,8 @@ export type MaestroPublicOperation = export type MaestroDaemonDispatchOptions = Readonly<{ /** Terminate the targeted app without ending the owning daemon session. */ closeAppOnly?: true; + /** With `closeAppOnly`: Maestro `killApp` process death (`am kill` on Android) instead of `stop`. */ + killApp?: true; /** A hierarchy capture used as operational evidence only; it issues no client ref authority. */ observationOnly?: true; /** Provider-owned viewport already resolved for a nested gesture command. */ @@ -88,12 +91,15 @@ export function projectMaestroPublicOperation( type MaestroAppOperation = Extract< MaestroPublicOperation, - { kind: 'launchApp' | 'stopApp' | 'openLink' } + { kind: 'launchApp' | 'stopApp' | 'killApp' | 'openLink' } >; function isAppOperation(operation: MaestroPublicOperation): operation is MaestroAppOperation { return ( - operation.kind === 'launchApp' || operation.kind === 'stopApp' || operation.kind === 'openLink' + operation.kind === 'launchApp' || + operation.kind === 'stopApp' || + operation.kind === 'killApp' || + operation.kind === 'openLink' ); } @@ -103,6 +109,8 @@ function projectAppOperation(operation: MaestroAppOperation): MaestroDaemonOpera return projectLaunchApp(operation); case 'stopApp': return projectStopApp(operation); + case 'killApp': + return projectKillApp(operation); case 'openLink': return projectOpenLink(operation); } @@ -132,6 +140,16 @@ function projectStopApp( }; } +function projectKillApp( + operation: Extract, +): MaestroDaemonOperationRequest { + return { + command: 'close', + positionals: operation.appId ? [operation.appId] : [], + dispatch: { closeAppOnly: true, killApp: true }, + }; +} + function projectClearState( operation: Extract, ): MaestroDaemonOperationRequest { diff --git a/packages/maestro/src/internal/__tests__/program-ir-parser.test.ts b/packages/maestro/src/internal/__tests__/program-ir-parser.test.ts index 52dd72b9bb..090c3815c2 100644 --- a/packages/maestro/src/internal/__tests__/program-ir-parser.test.ts +++ b/packages/maestro/src/internal/__tests__/program-ir-parser.test.ts @@ -613,6 +613,27 @@ describe('parseMaestroProgram', () => { }); }); + test('parses standalone killApp with an explicit or config app id', () => { + const program = parseMaestroProgram( + `appId: example.app +--- +- killApp: example.app +- killApp +`, + { sourcePath: '/flows/kill.yaml' }, + ); + + assert.deepEqual(program.commands[0], { + kind: 'killApp', + source: { path: '/flows/kill.yaml', line: 3 }, + appId: 'example.app', + }); + assert.deepEqual(program.commands[1], { + kind: 'killApp', + source: { path: '/flows/kill.yaml', line: 4 }, + }); + }); + test('preserves source paths for unsupported and malformed flows', () => { const sourcePath = '/flows/includes/child.yaml'; assert.throws( diff --git a/packages/maestro/src/internal/__tests__/runtime-port-fixtures.ts b/packages/maestro/src/internal/__tests__/runtime-port-fixtures.ts index 765892c9b2..b69da541af 100644 --- a/packages/maestro/src/internal/__tests__/runtime-port-fixtures.ts +++ b/packages/maestro/src/internal/__tests__/runtime-port-fixtures.ts @@ -59,6 +59,7 @@ export function makeOperations( resolveGestureViewport: async () => ({ x: 0, y: 0, width: 402, height: 874 }), launchApp: noOp, stopApp: noOp, + killApp: noOp, setPermissions: noOp, clearState: noOp, openLink: noOp, diff --git a/packages/maestro/src/internal/__tests__/runtime-port.test.ts b/packages/maestro/src/internal/__tests__/runtime-port.test.ts index 9da89973c2..24a684acdd 100644 --- a/packages/maestro/src/internal/__tests__/runtime-port.test.ts +++ b/packages/maestro/src/internal/__tests__/runtime-port.test.ts @@ -178,6 +178,25 @@ describe('MaestroRuntimePort', () => { expect(calls[1]).toMatchObject({ input: { appId: 'com.example.checkout' } }); }); + test('dispatches standalone killApp with an explicit or config app id', async () => { + const calls: RecordedCall[] = []; + const operations = makeOperations({ + killApp: vi.fn(async (input, context) => record(calls, 'killApp', input, context)), + }); + const program = parseMaestroProgram( + ['appId: com.example.checkout', '---', '- killApp: com.example.checkout', '- killApp'].join( + '\n', + ), + ); + + const result = await executeMaestroProgram(program, createMaestroRuntimePort(operations)); + + expect(result).toMatchObject({ executed: 2, skipped: 0 }); + expect(calls.map(({ kind }) => kind)).toEqual(['killApp', 'killApp']); + expect(calls[0]).toMatchObject({ input: { appId: 'com.example.checkout' } }); + expect(calls[1]).toMatchObject({ input: { appId: 'com.example.checkout' } }); + }); + test('preserves observation validity after visual waits and scripts', async () => { const waitInvalidation = vi.fn(); const scriptInvalidation = vi.fn(); diff --git a/packages/maestro/src/internal/conformance-normalize.ts b/packages/maestro/src/internal/conformance-normalize.ts index 6d61c13be4..1d4993eacf 100644 --- a/packages/maestro/src/internal/conformance-normalize.ts +++ b/packages/maestro/src/internal/conformance-normalize.ts @@ -83,6 +83,7 @@ export type CanonicalCommand = | { kind: 'takeScreenshot' } | { kind: 'waitForAnimationToEnd'; timeout?: number | string } | { kind: 'stopApp' } + | { kind: 'killApp' } | { kind: 'setPermissions'; appId?: string; permissions?: Record } | { kind: 'clearState'; appId?: string } | { kind: 'repeat'; times: string | number } @@ -116,6 +117,8 @@ function canonicalizeUpstreamLifecycleCommand( }); case 'StopAppCommand': return { kind: 'stopApp' }; + case 'KillAppCommand': + return { kind: 'killApp' }; case 'ClearStateCommand': return dropUndefined({ kind: 'clearState' as const, appId: str(f.appId) }); default: @@ -367,7 +370,7 @@ type AgentLifecycleCommand = Extract< { kind: (typeof AGENT_LIFECYCLE_KINDS)[number] } >; -const AGENT_LIFECYCLE_KINDS = ['launchApp', 'stopApp', 'clearState'] as const; +const AGENT_LIFECYCLE_KINDS = ['launchApp', 'stopApp', 'killApp', 'clearState'] as const; function isAgentLifecycleCommand(command: MaestroCommand): command is AgentLifecycleCommand { return (AGENT_LIFECYCLE_KINDS as readonly string[]).includes(command.kind); @@ -388,6 +391,8 @@ function canonicalizeAgentLifecycleCommand( }); case 'stopApp': return { kind: 'stopApp' }; + case 'killApp': + return { kind: 'killApp' }; case 'clearState': return dropUndefined({ kind: 'clearState', appId: command.appId ?? config.appId }); } diff --git a/packages/maestro/src/internal/program-ir-command-parser.ts b/packages/maestro/src/internal/program-ir-command-parser.ts index f1d797c950..a2b6a3dc4d 100644 --- a/packages/maestro/src/internal/program-ir-command-parser.ts +++ b/packages/maestro/src/internal/program-ir-command-parser.ts @@ -10,6 +10,7 @@ import type { MaestroExtendedWaitUntilCommand, MaestroHideKeyboardCommand, MaestroInputTextCommand, + MaestroKillAppCommand, MaestroLaunchAppCommand, MaestroLaunchArguments, MaestroOpenLinkCommand, @@ -127,6 +128,7 @@ const COMMAND_VALUE_PARSERS: Readonly> = { back: parseBack, waitForAnimationToEnd: parseWaitForAnimationToEnd, stopApp: parseStopApp, + killApp: parseKillApp, setPermissions: parseSetPermissions, clearState: parseClearState, runScript: parseMaestroRunScriptCommand, @@ -459,6 +461,16 @@ function parseStopApp( return { kind: 'stopApp', source, appId: readRequiredString(value, 'stopApp', context) }; } +function parseKillApp( + value: Node | null, + commandNode: Node, + context: MaestroProgramParseContext, +): MaestroKillAppCommand { + const source = sourceAt(commandNode, context); + if (isNullNode(value)) return { kind: 'killApp', source }; + return { kind: 'killApp', source, appId: readRequiredString(value, 'killApp', context) }; +} + function parseSetPermissions( value: Node | null, commandNode: Node, diff --git a/packages/maestro/src/internal/program-ir.ts b/packages/maestro/src/internal/program-ir.ts index d8020097ee..0798618c5d 100644 --- a/packages/maestro/src/internal/program-ir.ts +++ b/packages/maestro/src/internal/program-ir.ts @@ -208,6 +208,12 @@ export type MaestroStopAppCommand = { appId?: string; }; +export type MaestroKillAppCommand = { + kind: 'killApp'; + source: MaestroSourceLocation; + appId?: string; +}; + export type MaestroSetPermissionsCommand = MaestroOptionalCommand & { kind: 'setPermissions'; source: MaestroSourceLocation; @@ -286,6 +292,7 @@ export type MaestroCommand = | MaestroBackCommand | MaestroWaitForAnimationToEndCommand | MaestroStopAppCommand + | MaestroKillAppCommand | MaestroSetPermissionsCommand | MaestroClearStateCommand | MaestroRunScriptCommand diff --git a/packages/maestro/src/internal/runtime-port-commands.ts b/packages/maestro/src/internal/runtime-port-commands.ts index cb4fafb422..e398600ee1 100644 --- a/packages/maestro/src/internal/runtime-port-commands.ts +++ b/packages/maestro/src/internal/runtime-port-commands.ts @@ -30,7 +30,7 @@ type MaestroCommandOf = Extract< >; type MaestroLifecycleCommand = MaestroCommandOf< - 'launchApp' | 'stopApp' | 'setPermissions' | 'clearState' | 'openLink' + 'launchApp' | 'stopApp' | 'killApp' | 'setPermissions' | 'clearState' | 'openLink' >; type MaestroTargetCommand = MaestroCommandOf<'tapOn' | 'doubleTapOn' | 'longPressOn'>; type MaestroTextCommand = MaestroCommandOf<'inputText' | 'eraseText'>; @@ -55,6 +55,7 @@ type MaestroRuntimeCommandHandlers = { const MAESTRO_RUNTIME_COMMAND_HANDLERS = { launchApp: executeLifecycleCommand, stopApp: executeLifecycleCommand, + killApp: executeLifecycleCommand, setPermissions: executeLifecycleCommand, clearState: executeLifecycleCommand, openLink: executeLifecycleCommand, @@ -82,6 +83,7 @@ const MAESTRO_RUNTIME_COMMAND_HANDLERS = { const MAESTRO_COMMAND_REQUIRES_SETTLED_PREDECESSOR = { launchApp: true, stopApp: true, + killApp: true, setPermissions: true, clearState: true, openLink: true, @@ -150,6 +152,13 @@ async function executeLifecycleCommand( context, 'invalidate', ); + case 'killApp': + return await invokeOperation( + operations.killApp, + { appId: command.appId ?? request.appId }, + context, + 'invalidate', + ); case 'setPermissions': return await invokeOperation( operations.setPermissions, diff --git a/packages/maestro/src/internal/runtime-port-types.ts b/packages/maestro/src/internal/runtime-port-types.ts index 4c82492049..924468a460 100644 --- a/packages/maestro/src/internal/runtime-port-types.ts +++ b/packages/maestro/src/internal/runtime-port-types.ts @@ -124,6 +124,7 @@ export type MaestroRuntimeOperations = { readonly launchArguments?: MaestroLaunchArguments; }>; readonly stopApp: MaestroRuntimeOperation<{ readonly appId?: string }>; + readonly killApp: MaestroRuntimeOperation<{ readonly appId?: string }>; readonly setPermissions: MaestroRuntimeOperation<{ readonly appId?: string; readonly permissions: Readonly>; diff --git a/packages/maestro/src/internal/support-matrix.ts b/packages/maestro/src/internal/support-matrix.ts index a58f2bcff7..dd265a7eaf 100644 --- a/packages/maestro/src/internal/support-matrix.ts +++ b/packages/maestro/src/internal/support-matrix.ts @@ -1,7 +1,7 @@ export const MAESTRO_COMPAT_SUPPORTED_CAPABILITIES = [ 'Flows: launchApp (with clearState, permissions, and Apple-only launch arguments; permissions apply after state clearing but before launch, and a launchApp without permissions touches nothing — there is no silent all: allow default); setPermissions (mid-flow permission grants, denials, and resets; all resolves in the backend — one simctl call on iOS, the declared permissions on Android — with specific entries overriding after it); runFlow file/inline with platform, visibility, and limited boolean conditions; onFlowStart/onFlowComplete; repeat.times and retry.', 'Interactions: tapOn, doubleTapOn, longPressOn, inputText on the focused element, eraseText, openLink, hideKeyboard, basic pressKey, and back; selector targets poll until available and support recursive index, childOf, above, below, leftOf, rightOf, containsChild, containsDescendants, points, and optional; outer command labels are metadata, not target selectors.', - 'Assertions and navigation: assertVisible, assertNotVisible, assertTrue (literal values and ${VAR} lookups only; "", "false", "0", "null", and "undefined" are falsy, everything else is truthy), extendedWaitUntil, scroll, scrollUntilVisible, absolute/percentage/target swipe, takeScreenshot, waitForAnimationToEnd, clearState, and stopApp.', + 'Assertions and navigation: assertVisible, assertNotVisible, assertTrue (literal values and ${VAR} lookups only; "", "false", "0", "null", and "undefined" are falsy, everything else is truthy), extendedWaitUntil, scroll, scrollUntilVisible, absolute/percentage/target swipe, takeScreenshot, waitForAnimationToEnd, clearState, stopApp, and killApp (system-initiated process death: am kill on Android, stopApp alias elsewhere).', 'Scripts: ordered runScript file/env scripts with http.post, json, and output variables; evalScript inline expressions run flow-scoped JavaScript and write output.* leaves for later steps.', ] as const; diff --git a/packages/maestro/test/conformance/expected-divergence.ts b/packages/maestro/test/conformance/expected-divergence.ts index 52f96e4fc1..6d7a2e3997 100644 --- a/packages/maestro/test/conformance/expected-divergence.ts +++ b/packages/maestro/test/conformance/expected-divergence.ts @@ -48,11 +48,6 @@ export const FLOW_DIVERGENCES: Record = { reason: 'travel is outside the supported subset.', unsupported: ['travel'], }, - 'upstream/116_kill_app': { - classification: 'we-reject', - reason: 'Standalone killApp is outside the supported subset.', - unsupported: ['killApp'], - }, // --- Deliberately stricter than upstream --- 'invalid/duplicate-keys': { classification: 'we-reject', diff --git a/packages/platform-android/src/__tests__/app-lifecycle-open.test.ts b/packages/platform-android/src/__tests__/app-lifecycle-open.test.ts index 1abced327e..78b92e563a 100644 --- a/packages/platform-android/src/__tests__/app-lifecycle-open.test.ts +++ b/packages/platform-android/src/__tests__/app-lifecycle-open.test.ts @@ -1,6 +1,6 @@ import { test } from 'vitest'; import assert from 'node:assert/strict'; -import { closeAndroidApp, openAndroidApp } from '../app-lifecycle.ts'; +import { closeAndroidApp, killAndroidApp, openAndroidApp } from '../app-lifecycle.ts'; import { withAndroidAdbProvider } from '../adb-executor.ts'; import type { DeviceInfo } from '@agent-device/kernel/device'; import { AppError } from '@agent-device/kernel/errors'; @@ -162,6 +162,47 @@ test('closeAndroidApp waits until package process exits after force-stop', async ]); }); +test('killAndroidApp dispatches am kill rather than am force-stop', async () => { + const device: DeviceInfo = { + platform: 'android', + id: 'emulator-5554', + name: 'Pixel', + kind: 'emulator', + booted: true, + }; + const calls: (readonly string[])[] = []; + + await withAndroidAdbProvider( + { + exec: async (args) => { + calls.push(args); + if (args.join(' ') === 'shell dumpsys window windows') { + return { + stdout: 'mCurrentFocus=Window{43 u0 com.android.launcher/.Launcher}\n', + stderr: '', + exitCode: 0, + }; + } + return { stdout: '', stderr: '', exitCode: 0 }; + }, + reverse: { + ensure: async () => {}, + remove: async () => {}, + removeAllOwned: async () => {}, + }, + }, + { serial: 'emulator-5554' }, + async () => await killAndroidApp(device, 'com.example.app'), + ); + + assert.deepEqual(calls, [ + ['shell', 'am', 'kill', 'com.example.app'], + ['shell', 'dumpsys', 'window', 'windows'], + ['shell', 'pidof', 'com.example.app'], + ['shell', 'pidof', 'com.example.app'], + ]); +}); + test('openAndroidApp ensures Android reverse before localhost deep link launch', async () => { const device: DeviceInfo = { platform: 'android', diff --git a/packages/platform-android/src/app-lifecycle.ts b/packages/platform-android/src/app-lifecycle.ts index c5880ce5fb..c02cb89c0d 100644 --- a/packages/platform-android/src/app-lifecycle.ts +++ b/packages/platform-android/src/app-lifecycle.ts @@ -488,6 +488,26 @@ export async function closeAndroidApp(device: DeviceInfo, app: string): Promise< await waitForAndroidPackageStopped(device, resolved.value); } +/** + * Maestro `killApp` on Android: system-initiated process death (`am kill`), + * which only reaps a backgrounded/cached process — unlike `closeAndroidApp`'s + * `am force-stop`. Callers background the app first (e.g. `pressKey: Home`). + */ +export async function killAndroidApp(device: DeviceInfo, app: string): Promise { + const trimmed = app.trim(); + if (trimmed.toLowerCase() === 'settings') { + await runAndroidShell(device, ['am', 'kill', 'com.android.settings']); + await waitForAndroidPackageStopped(device, 'com.android.settings'); + return; + } + const resolved = await resolveAndroidApp(device, app); + if (resolved.type === 'intent') { + throw new AppError('INVALID_ARGS', 'Kill requires a package name, not an intent'); + } + await runAndroidShell(device, ['am', 'kill', resolved.value]); + await waitForAndroidPackageStopped(device, resolved.value); +} + async function waitForAndroidPackageStopped( device: DeviceInfo, packageName: string, diff --git a/packages/platform-android/src/lifecycle.ts b/packages/platform-android/src/lifecycle.ts index cab32f52f6..912c0dc606 100644 --- a/packages/platform-android/src/lifecycle.ts +++ b/packages/platform-android/src/lifecycle.ts @@ -67,6 +67,7 @@ export function bindAndroidApplicationLifecycle( device, interactor: await binding.resolveInteractor(input.execution, input.appBundleId), positionals: input.positionals, + mode: input.mode, }); }, finalizeApplicationClose: async (input) => { diff --git a/packages/platform-android/src/mechanics.ts b/packages/platform-android/src/mechanics.ts index 96efabf755..43b5dfa3f9 100644 --- a/packages/platform-android/src/mechanics.ts +++ b/packages/platform-android/src/mechanics.ts @@ -83,6 +83,7 @@ export async function listAndroidAppsWithAdb( export { closeAndroidApp, isAmStartError, + killAndroidApp, openAndroidApp, openAndroidDevice, parseAndroidLaunchComponent, diff --git a/scripts/fuzz/validation-arbitraries-maestro.ts b/scripts/fuzz/validation-arbitraries-maestro.ts index 1507debbcc..6b47745bb7 100644 --- a/scripts/fuzz/validation-arbitraries-maestro.ts +++ b/scripts/fuzz/validation-arbitraries-maestro.ts @@ -22,6 +22,8 @@ function validMaestroCommand(pick: number, salt: number): string[] { () => ['- back'], () => ['- hideKeyboard'], () => ['- stopApp'], + () => ['- killApp'], + () => [`- killApp: ${text}`], () => ['- clearState'], () => [`- clearState: ${text}`], () => ['- scroll'], diff --git a/src/core/interactors/android.ts b/src/core/interactors/android.ts index fff5d3f126..b52c1917d5 100644 --- a/src/core/interactors/android.ts +++ b/src/core/interactors/android.ts @@ -11,6 +11,7 @@ import { getAndroidKeyboardState, handleAndroidAlert, homeAndroid, + killAndroidApp, longPressAndroid, openAndroidApp, openAndroidDevice, @@ -69,6 +70,7 @@ export function createAndroidInteractor( }), openDevice: () => openAndroidDevice(device), close: (app) => closeAndroidApp(device, app), + kill: (app) => killAndroidApp(device, app), tap: (x, y) => pressAndroid(device, x, y), doubleTap: async (x, y) => { await pressAndroid(device, x, y); diff --git a/src/daemon/replay/internal/__tests__/session-replay-maestro-request.test.ts b/src/daemon/replay/internal/__tests__/session-replay-maestro-request.test.ts index 8102fab292..df2948d92a 100644 --- a/src/daemon/replay/internal/__tests__/session-replay-maestro-request.test.ts +++ b/src/daemon/replay/internal/__tests__/session-replay-maestro-request.test.ts @@ -102,6 +102,19 @@ test('folds Maestro settings app targeting into dispatch', () => { expect(request.dispatch).toEqual({ settingsAppBundleId: 'com.example.app' }); }); +test('folds Maestro killApp targeting into dispatch', () => { + const request = maestroOperationDispatchRequest( + { token: 'token', session: 'session', command: 'replay', positionals: [] }, + { + command: 'close', + positionals: ['com.example.app'], + dispatch: { closeAppOnly: true, killApp: true }, + }, + ); + + expect(request.dispatch).toEqual({ closeAppOnly: true, killApp: true }); +}); + test('leaves dispatch options the operation does not set untouched', () => { const viewport = { x: 0, y: 0, width: 402, height: 874 }; const request = maestroOperationDispatchRequest( diff --git a/src/daemon/replay/internal/session-replay-maestro-request.ts b/src/daemon/replay/internal/session-replay-maestro-request.ts index a95e077f2d..5aefbf320a 100644 --- a/src/daemon/replay/internal/session-replay-maestro-request.ts +++ b/src/daemon/replay/internal/session-replay-maestro-request.ts @@ -39,6 +39,7 @@ function maestroDispatchOptions( ): Pick { return stripUndefined({ closeAppOnly: dispatch?.closeAppOnly, + killApp: dispatch?.killApp, observationOnly: dispatch?.observationOnly, gestureViewport: dispatch?.gestureViewport, gestureExecutionProfile: dispatch?.gestureExecutionProfile, diff --git a/src/daemon/session-lifecycle/internal/__tests__/session-close-lifecycle-runtime.test.ts b/src/daemon/session-lifecycle/internal/__tests__/session-close-lifecycle-runtime.test.ts index 7ca47b926c..dcfc5631a6 100644 --- a/src/daemon/session-lifecycle/internal/__tests__/session-close-lifecycle-runtime.test.ts +++ b/src/daemon/session-lifecycle/internal/__tests__/session-close-lifecycle-runtime.test.ts @@ -319,6 +319,56 @@ test('a supported Android close clears admitted runtime hints exactly once', asy ); }); +test('an app-only kill close carries mode kill while a stop close carries none', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'android-app-only-kill-carries-mode'; + const device = { + platform: 'android' as const, + id: 'emulator-5554', + name: 'Pixel', + kind: 'emulator' as const, + target: 'mobile' as const, + booted: true, + }; + const session = makeSession(sessionName, device); + session.appBundleId = 'com.example.app'; + sessionStore.set(sessionName, session); + const seenModes: Array = []; + const baseBind = mockBindDeviceRuntime.getMockImplementation(); + mockBindDeviceRuntime.mockImplementation(async (boundDevice, use) => { + const binding = await baseBind!(boundDevice, use); + const innerClose = binding.operations.closeApplication; + if (!innerClose) return binding; + return { + ...binding, + operations: { + ...binding.operations, + closeApplication: async (input: Parameters[0]) => { + seenModes.push(input.mode); + return await innerClose(input); + }, + }, + }; + }); + + const killed = await close({ + sessionName, + sessionStore, + positionals: ['com.example.app'], + internal: { closeAppOnly: true, killApp: true }, + }); + const stopped = await close({ + sessionName, + sessionStore, + positionals: ['com.example.app'], + internal: { closeAppOnly: true }, + }); + + expect(killed?.ok).toBe(true); + expect(stopped?.ok).toBe(true); + expect(seenModes).toEqual(['kill', undefined]); +}); + test('close expires the ref frame immediately before its admitted platform mutation', async () => { const sessionStore = makeSessionStore(); const sessionName = 'close-ref-frame-seam'; diff --git a/src/daemon/session-lifecycle/internal/session-close.ts b/src/daemon/session-lifecycle/internal/session-close.ts index da1bd016e5..924a04a64a 100644 --- a/src/daemon/session-lifecycle/internal/session-close.ts +++ b/src/daemon/session-lifecycle/internal/session-close.ts @@ -149,6 +149,7 @@ async function dispatchTargetedPlatformClose(params: { outPath: req.flags?.out, appBundleId: session.appBundleId, surface: session.surface ?? 'app', + ...(req.internal?.killApp === true ? { mode: 'kill' as const } : {}), execution: applicationLifecycleExecutionFromRequest(req, logPath, session.trace?.outPath), }); return undefined; diff --git a/website/docs/docs/replay-e2e.md b/website/docs/docs/replay-e2e.md index 304ec141da..e86b3f66db 100644 --- a/website/docs/docs/replay-e2e.md +++ b/website/docs/docs/replay-e2e.md @@ -72,7 +72,7 @@ Supported subset: - Flows: `launchApp` (with `clearState`, `permissions`, and Apple-only launch arguments; `permissions` apply after state clearing but before launch, and a `launchApp` without `permissions` touches nothing — there is no silent `all: allow` default); `setPermissions` (mid-flow permission grants, denials, and resets; `all` resolves in the backend — one simctl call on iOS, the declared permissions on Android — with specific entries overriding after it); `runFlow` file/inline with platform, visibility, and limited boolean conditions; `onFlowStart`/`onFlowComplete`; `repeat.times` and retry. - Interactions: `tapOn`, `doubleTapOn`, `longPressOn`, `inputText` on the focused element, `eraseText`, `openLink`, `hideKeyboard`, basic `pressKey`, and `back`; selector targets poll until available and support recursive `index`, `childOf`, `above`, `below`, `leftOf`, `rightOf`, `containsChild`, `containsDescendants`, points, and `optional`; outer command labels are metadata, not target selectors. -- Assertions and navigation: `assertVisible`, `assertNotVisible`, `assertTrue` (literal values and `${VAR}` lookups only; `""`, `"false"`, `"0"`, `"null"`, and `"undefined"` are falsy, everything else is truthy), `extendedWaitUntil`, `scroll`, `scrollUntilVisible`, absolute/percentage/target `swipe`, `takeScreenshot`, `waitForAnimationToEnd`, `clearState`, and `stopApp`. +- Assertions and navigation: `assertVisible`, `assertNotVisible`, `assertTrue` (literal values and `${VAR}` lookups only; `""`, `"false"`, `"0"`, `"null"`, and `"undefined"` are falsy, everything else is truthy), `extendedWaitUntil`, `scroll`, `scrollUntilVisible`, absolute/percentage/target `swipe`, `takeScreenshot`, `waitForAnimationToEnd`, `clearState`, `stopApp`, and `killApp` (system-initiated process death: `am kill` on Android, `stopApp` alias elsewhere). - Scripts: ordered `runScript` file/env scripts with `http.post`, `json`, and `output` variables; `evalScript` inline expressions run flow-scoped JavaScript and write `output.*` leaves for later steps. Boundaries: From 369a51b35a8846983d4055658a73f4a075ec2cef Mon Sep 17 00:00:00 2001 From: Rohit <40559587+Rohit3523@users.noreply.github.com> Date: Tue, 22 Sep 2026 03:34:43 +0530 Subject: [PATCH 002/132] refactor(maestro): table-drive app-target lifecycle dispatch executeLifecycleCommand grew a sixth case for killApp and tripped the complexity gate. The stopApp/killApp/clearState legs share one shape, so dispatch them through a lookup instead of three switch cases. --- .../src/internal/runtime-port-commands.ts | 39 +++++++++---------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/packages/maestro/src/internal/runtime-port-commands.ts b/packages/maestro/src/internal/runtime-port-commands.ts index e398600ee1..7c71f57124 100644 --- a/packages/maestro/src/internal/runtime-port-commands.ts +++ b/packages/maestro/src/internal/runtime-port-commands.ts @@ -131,6 +131,19 @@ function dispatchMaestroRuntimeCommand( return handler(command, request, operations, context); } +type MaestroAppTargetLifecycleCommand = MaestroCommandOf<'stopApp' | 'killApp' | 'clearState'>; + +/** Lifecycle commands carrying only an app target share one dispatch shape. */ +const MAESTRO_APP_TARGET_OPERATIONS: { + [K in MaestroAppTargetLifecycleCommand['kind']]: ( + operations: MaestroRuntimeOperations, + ) => MaestroRuntimeOperations[K]; +} = { + stopApp: (operations) => operations.stopApp, + killApp: (operations) => operations.killApp, + clearState: (operations) => operations.clearState, +}; + async function executeLifecycleCommand( command: MaestroLifecycleCommand, request: MaestroRuntimeRequest, @@ -145,20 +158,6 @@ async function executeLifecycleCommand( context, 'invalidate', ); - case 'stopApp': - return await invokeOperation( - operations.stopApp, - { appId: command.appId ?? request.appId }, - context, - 'invalidate', - ); - case 'killApp': - return await invokeOperation( - operations.killApp, - { appId: command.appId ?? request.appId }, - context, - 'invalidate', - ); case 'setPermissions': return await invokeOperation( operations.setPermissions, @@ -169,17 +168,17 @@ async function executeLifecycleCommand( context, 'invalidate', ); - case 'clearState': + case 'openLink': return await invokeOperation( - operations.clearState, - { appId: command.appId ?? request.appId }, + operations.openLink, + { link: command.link }, context, 'invalidate', ); - case 'openLink': + default: return await invokeOperation( - operations.openLink, - { link: command.link }, + MAESTRO_APP_TARGET_OPERATIONS[command.kind](operations), + { appId: command.appId ?? request.appId }, context, 'invalidate', ); From 78727e3f1dac3be6c25c45faabcc2eec510dcf95 Mon Sep 17 00:00:00 2001 From: Rohit <40559587+Rohit3523@users.noreply.github.com> Date: Tue, 22 Sep 2026 03:43:30 +0530 Subject: [PATCH 003/132] fix(maestro): fail killApp loud when the target survives am kill only reaps background processes, so a foreground killApp used to succeed without killing anything. Refuse a foreground target naming the background-first precondition, and verify the process is gone after the kill instead of reporting an unproven success. --- .../src/__tests__/app-lifecycle-open.test.ts | 95 +++++++++++++++++++ .../platform-android/src/app-lifecycle.ts | 26 ++++- 2 files changed, 117 insertions(+), 4 deletions(-) diff --git a/packages/platform-android/src/__tests__/app-lifecycle-open.test.ts b/packages/platform-android/src/__tests__/app-lifecycle-open.test.ts index 78b92e563a..ac270f8fe5 100644 --- a/packages/platform-android/src/__tests__/app-lifecycle-open.test.ts +++ b/packages/platform-android/src/__tests__/app-lifecycle-open.test.ts @@ -196,13 +196,108 @@ test('killAndroidApp dispatches am kill rather than am force-stop', async () => ); assert.deepEqual(calls, [ + ['shell', 'dumpsys', 'window', 'windows'], ['shell', 'am', 'kill', 'com.example.app'], ['shell', 'dumpsys', 'window', 'windows'], ['shell', 'pidof', 'com.example.app'], ['shell', 'pidof', 'com.example.app'], + ['shell', 'pidof', 'com.example.app'], ]); }); +test('killAndroidApp refuses a foreground target before killing', async () => { + const device: DeviceInfo = { + platform: 'android', + id: 'emulator-5554', + name: 'Pixel', + kind: 'emulator', + booted: true, + }; + const calls: (readonly string[])[] = []; + + await withAndroidAdbProvider( + { + exec: async (args) => { + calls.push(args); + if (args.join(' ') === 'shell dumpsys window windows') { + return { + stdout: 'mCurrentFocus=Window{42 u0 com.example.app/.MainActivity}\n', + stderr: '', + exitCode: 0, + }; + } + return { stdout: '', stderr: '', exitCode: 0 }; + }, + reverse: { + ensure: async () => {}, + remove: async () => {}, + removeAllOwned: async () => {}, + }, + }, + { serial: 'emulator-5554' }, + async () => { + await assertRejectsAppError(() => killAndroidApp(device, 'com.example.app'), { + code: 'COMMAND_FAILED', + hint: /Background the app before killApp/, + }); + try { + await killAndroidApp(device, 'com.example.app'); + assert.fail('expected killAndroidApp to reject for a foreground app'); + } catch (error) { + assert.equal( + (error as InstanceType).details?.reason, + 'android-kill-requires-background-app', + ); + } + }, + ); + + assert.deepEqual(calls, [ + ['shell', 'dumpsys', 'window', 'windows'], + ['shell', 'dumpsys', 'window', 'windows'], + ]); +}); + +test('killAndroidApp fails when the process survives the kill', async () => { + const device: DeviceInfo = { + platform: 'android', + id: 'emulator-5554', + name: 'Pixel', + kind: 'emulator', + booted: true, + }; + + await withAndroidAdbProvider( + { + exec: async (args) => { + if (args.join(' ') === 'shell dumpsys window windows') { + return { + stdout: 'mCurrentFocus=Window{43 u0 com.android.launcher/.Launcher}\n', + stderr: '', + exitCode: 0, + }; + } + if (args.join(' ') === 'shell pidof com.example.app') { + return { stdout: '12345\n', stderr: '', exitCode: 0 }; + } + return { stdout: '', stderr: '', exitCode: 0 }; + }, + reverse: { + ensure: async () => {}, + remove: async () => {}, + removeAllOwned: async () => {}, + }, + }, + { serial: 'emulator-5554' }, + async () => { + await assertRejectsAppError(() => killAndroidApp(device, 'com.example.app'), { + code: 'COMMAND_FAILED', + hint: /Background the app before killApp/, + }); + }, + ); +}); + test('openAndroidApp ensures Android reverse before localhost deep link launch', async () => { const device: DeviceInfo = { platform: 'android', diff --git a/packages/platform-android/src/app-lifecycle.ts b/packages/platform-android/src/app-lifecycle.ts index c02cb89c0d..6b3072fb05 100644 --- a/packages/platform-android/src/app-lifecycle.ts +++ b/packages/platform-android/src/app-lifecycle.ts @@ -492,20 +492,38 @@ export async function closeAndroidApp(device: DeviceInfo, app: string): Promise< * Maestro `killApp` on Android: system-initiated process death (`am kill`), * which only reaps a backgrounded/cached process — unlike `closeAndroidApp`'s * `am force-stop`. Callers background the app first (e.g. `pressKey: Home`). + * A foreground target fails loud naming the precondition, and a process that + * survives the kill fails rather than reading as success. */ export async function killAndroidApp(device: DeviceInfo, app: string): Promise { const trimmed = app.trim(); if (trimmed.toLowerCase() === 'settings') { - await runAndroidShell(device, ['am', 'kill', 'com.android.settings']); - await waitForAndroidPackageStopped(device, 'com.android.settings'); + await killAndroidPackage(device, 'com.android.settings'); return; } const resolved = await resolveAndroidApp(device, app); if (resolved.type === 'intent') { throw new AppError('INVALID_ARGS', 'Kill requires a package name, not an intent'); } - await runAndroidShell(device, ['am', 'kill', resolved.value]); - await waitForAndroidPackageStopped(device, resolved.value); + await killAndroidPackage(device, resolved.value); +} + +async function killAndroidPackage(device: DeviceInfo, packageName: string): Promise { + const foreground = await readAndroidForegroundApp(device); + if (foreground?.package === packageName) { + throw new AppError('COMMAND_FAILED', `Cannot kill foreground app ${packageName}`, { + reason: 'android-kill-requires-background-app', + hint: 'Background the app before killApp (for example pressKey: Home): am kill only reaps background processes.', + }); + } + await runAndroidShell(device, ['am', 'kill', packageName]); + await waitForAndroidPackageStopped(device, packageName); + if (await isAndroidPackageProcessRunning(device, packageName)) { + throw new AppError('COMMAND_FAILED', `am kill did not stop ${packageName}`, { + reason: 'android-kill-requires-background-app', + hint: 'Background the app before killApp (for example pressKey: Home): am kill only reaps background processes.', + }); + } } async function waitForAndroidPackageStopped( From 690313034532600aab6696a8d234423b6f7fc308 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 22 Sep 2026 07:45:08 +0200 Subject: [PATCH 004/132] fix(ios): serve a simulator recording whose recorder died with its daemon (#2736) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(ios): serve a simulator recording whose recorder died with its daemon A simctl reattach answered `missing` as soon as the recorder process was gone, which is the ordinary state after daemon loss, so a retried `record stop` threw `resource-missing` forever even with the recorder's video on disk. The simctl descriptor now carries the caller-facing export coordinates, so a proven-gone recorder whose file still passes the container sniff reattaches as a handle that runs the same stop-and-export sequence the live recording would have run, and discloses the touch overlay whose events did not survive the daemon. A manifest written without those coordinates is answered exactly as it was before, because it cannot name an export it never recorded. * fix(ios): resume a recovered simulator export from what the first stop journaled A recorder that died with its daemon proved nothing about the export its file can still become, and neither did a manifest whose stop had already collected a copy or journaled a finalization. Reattach decided from the recorder's own path alone, so a retry after those steps found no file there and reported a loss the manifest contradicted. It now asks what a resumed stop would still have to read, which the shared stop sequence answers from the checkpoints the first attempt wrote. The coordinates a recovered export needs are the caller-facing keys of the live snapshot, so the facet is a Pick of it, encoded and restored by spread, and validated by the recording-facts validator Android's descriptor already needed — moved to capture-kit so both backends answer with the same strictness. A recovered handle refuses an overlay whose gesture events died with the daemon instead of running the overlay pass with nothing to burn in and then reporting it unavailable. * chore(gates): declare the capture-kit recording-facts subpath in the boundary enumeration --- packages/capture-kit/package.json | 4 + .../src/recording/recording-facts.test.ts | 47 +++ .../src/recording/recording-facts.ts | 56 ++++ .../src/recording/stop-sequence.ts | 7 + .../src/recording/manifest-validation.ts | 46 +-- .../src/recording/completion.ts | 14 +- .../src/recording/recovery.test.ts | 233 ++++++++++++++ .../platform-apple/src/recording/recovery.ts | 292 +++++++++++++++--- .../src/recording/runtime.fixtures.ts | 21 +- .../platform-apple/src/recording/runtime.ts | 97 +++++- scripts/layering/package-boundaries.test.ts | 1 + 11 files changed, 724 insertions(+), 94 deletions(-) create mode 100644 packages/capture-kit/src/recording/recording-facts.test.ts create mode 100644 packages/capture-kit/src/recording/recording-facts.ts diff --git a/packages/capture-kit/package.json b/packages/capture-kit/package.json index 05346e4968..139662c3d2 100644 --- a/packages/capture-kit/package.json +++ b/packages/capture-kit/package.json @@ -166,6 +166,10 @@ "types": "./src/recording/artifact.fixtures.ts", "default": "./src/recording/artifact.fixtures.ts" }, + "./recording-facts": { + "types": "./src/recording/recording-facts.ts", + "default": "./src/recording/recording-facts.ts" + }, "./recording-mp4-fixtures": { "types": "./src/recording/mp4.fixtures.ts", "default": "./src/recording/mp4.fixtures.ts" diff --git a/packages/capture-kit/src/recording/recording-facts.test.ts b/packages/capture-kit/src/recording/recording-facts.test.ts new file mode 100644 index 0000000000..51c679b483 --- /dev/null +++ b/packages/capture-kit/src/recording/recording-facts.test.ts @@ -0,0 +1,47 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { RECORDING_FACTS_KEYS, recordingFactsAreValid } from './recording-facts.ts'; + +const FACTS = Object.freeze({ + scope: 'device', + showTouches: true, + recordOnlySession: false, +}); + +function facet(overrides: Record = {}): Record { + return { ...FACTS, ...overrides }; +} + +test('accepts the facts a caller asked for, with or without the optional ones', () => { + assert.equal(recordingFactsAreValid(facet()), true); + assert.equal( + recordingFactsAreValid( + facet({ activeSessionApp: { bundleId: 'com.example.app', name: 'Example' } }), + ), + true, + ); + assert.equal(recordingFactsAreValid(facet({ exportQuality: 'high' })), true); +}); + +test('refuses a facet whose recording facts no start could have produced', () => { + assert.equal(recordingFactsAreValid(facet({ scope: 'window' })), false); + assert.equal(recordingFactsAreValid(facet({ showTouches: 'yes' })), false); + assert.equal(recordingFactsAreValid(facet({ recordOnlySession: 1 })), false); + assert.equal(recordingFactsAreValid({ ...FACTS, showTouches: undefined }), false); +}); + +test('refuses an optional fact that is present but unreadable rather than dropping it', () => { + assert.equal(recordingFactsAreValid(facet({ exportQuality: 'ultra' })), false); + assert.equal(recordingFactsAreValid(facet({ activeSessionApp: { bundleId: '' } })), false); + assert.equal(recordingFactsAreValid(facet({ activeSessionApp: { name: 'Example' } })), false); + assert.equal(recordingFactsAreValid(facet({ activeSessionApp: 'com.example.app' })), false); + assert.equal( + recordingFactsAreValid(facet({ activeSessionApp: { bundleId: 'a', name: '' } })), + false, + ); +}); + +test('names every key of the facet it validates', () => { + const declared = Object.keys({ ...FACTS, activeSessionApp: undefined, exportQuality: undefined }); + assert.deepEqual([...RECORDING_FACTS_KEYS].sort(), declared.sort()); +}); diff --git a/packages/capture-kit/src/recording/recording-facts.ts b/packages/capture-kit/src/recording/recording-facts.ts new file mode 100644 index 0000000000..57cb388331 --- /dev/null +++ b/packages/capture-kit/src/recording/recording-facts.ts @@ -0,0 +1,56 @@ +import { + isRecordingExportQuality, + isRecordingScope, + type RecordingAppIdentity, + type RecordingExportQuality, + type RecordingScope, +} from '@agent-device/contracts/recording'; +import { isRecord } from '@agent-device/kernel/record'; + +/** + * What a durable recording keeps about what its caller asked for, apart from where its files live + * (ADR 0024 2.3). Every backend's descriptor carries these facts because the export it owes the + * caller is described by them, so one validator decides whether a manifest's copy can be trusted + * rather than each backend inventing its own strictness. + */ +export type RecordingFacts = Readonly<{ + scope: RecordingScope; + showTouches: boolean; + recordOnlySession: boolean; + activeSessionApp?: RecordingAppIdentity; + exportQuality?: RecordingExportQuality; +}>; + +/** The keys of {@link RecordingFacts}, so a backend carries the facet without naming it twice. */ +export const RECORDING_FACTS_KEYS = [ + 'scope', + 'showTouches', + 'recordOnlySession', + 'activeSessionApp', + 'exportQuality', +] as const satisfies readonly (keyof RecordingFacts)[]; + +/** + * Whether a durable value carries whole recording facts. One unreadable field refuses the whole + * facet: a resumed stop that trusted half an overlay request would serve a video the caller never + * asked for, which is the failure an unreadable descriptor is supposed to prevent. + */ +export function recordingFactsAreValid(value: Record): value is RecordingFacts { + return ( + isRecordingScope(value.scope) && + typeof value.showTouches === 'boolean' && + typeof value.recordOnlySession === 'boolean' && + (value.exportQuality === undefined || isRecordingExportQuality(value.exportQuality)) && + isOptionalAppIdentity(value.activeSessionApp) + ); +} + +function isOptionalAppIdentity(value: unknown): value is RecordingAppIdentity | undefined { + if (value === undefined) return true; + if (!isRecord(value) || !isNonemptyText(value.bundleId)) return false; + return value.name === undefined || isNonemptyText(value.name); +} + +function isNonemptyText(value: unknown): value is string { + return typeof value === 'string' && value.length > 0; +} diff --git a/packages/capture-kit/src/recording/stop-sequence.ts b/packages/capture-kit/src/recording/stop-sequence.ts index 2c2970d3c3..a7a5787222 100644 --- a/packages/capture-kit/src/recording/stop-sequence.ts +++ b/packages/capture-kit/src/recording/stop-sequence.ts @@ -15,6 +15,13 @@ import { readStopCheckpoints, writeStopCheckpoint } from './stop-checkpoints.ts' /** Where a recorder writes when its file must stay separate from the export (ADR 0024 2.3). */ export { collectedRecordingPath, nativeRecordingPath } from './artifact-paths.ts'; +/** + * What an earlier attempt of this stop journaled. A backend deciding whether a lost recording is still + * recoverable reads the same checkpoints this sequence resumes from, so the two cannot disagree about + * what a retry would still have to do. + */ +export { readStopCheckpoints } from './stop-checkpoints.ts'; + /** What a backend learned while asking its recorder to stop (ADR 0024 2.2). */ export type RecorderStop = Readonly<{ observation: StopObservation; diff --git a/packages/platform-android/src/recording/manifest-validation.ts b/packages/platform-android/src/recording/manifest-validation.ts index 8d0960fa65..c8ad5fc90b 100644 --- a/packages/platform-android/src/recording/manifest-validation.ts +++ b/packages/platform-android/src/recording/manifest-validation.ts @@ -1,4 +1,5 @@ import { isNativePathDisposition } from '@agent-device/contracts/recording-native-path'; +import { recordingFactsAreValid } from '@agent-device/capture-kit/recording-facts'; import { isStopObservation } from '@agent-device/contracts/recording-stop-observation'; import type { ScreenRecordingChunk, @@ -53,20 +54,11 @@ function descriptorIdentityIsValid(value: Record): boolean { } function descriptorRecordingIsValid(value: Record): boolean { - return ( - (value.clientOutputPath === undefined || typeof value.clientOutputPath === 'string') && - isScope(value.scope) && - typeof value.showTouches === 'boolean' && - typeof value.recordOnlySession === 'boolean' - ); + return isOptionalText(value.clientOutputPath) && recordingFactsAreValid(value); } function descriptorOptionsAreValid(value: Record): boolean { - return ( - isTransportMode(value.transportMode) && - isOptionalQuality(value.exportQuality) && - isOptionalApp(value.activeSessionApp) - ); + return isTransportMode(value.transportMode); } function manifestIdentityIsValid(candidate: Partial): boolean { @@ -84,17 +76,13 @@ function manifestIdentityIsValid(candidate: Partial): boolean { function manifestRecordingIsValid(candidate: Partial): boolean { return ( typeof candidate.outputPath === 'string' && - (candidate.clientOutputPath === undefined || typeof candidate.clientOutputPath === 'string') && - isScope(candidate.scope) && - typeof candidate.showTouches === 'boolean' && - typeof candidate.recordOnlySession === 'boolean' + isOptionalText(candidate.clientOutputPath) && + recordingFactsAreValid(candidate) ); } function manifestOptionsAreValid(candidate: Partial): boolean { return ( - isOptionalApp(candidate.activeSessionApp) && - isOptionalQuality(candidate.exportQuality) && isTransportMode(candidate.transportMode) && (candidate.pendingRemotePath === undefined || isNativeRecordingPath(candidate.pendingRemotePath)) @@ -153,12 +141,7 @@ function completionIdentityIsValid(candidate: Partial } function completionRecordingIsValid(candidate: Partial): boolean { - return ( - isScope(candidate.scope) && - typeof candidate.showTouches === 'boolean' && - typeof candidate.recordOnlySession === 'boolean' && - isOptionalApp(candidate.activeSessionApp) - ); + return recordingFactsAreValid(candidate); } function isValidCompletionChunk(chunk: ScreenRecordingChunk, index: number): boolean { @@ -229,25 +212,12 @@ function isObject(value: unknown): value is Record { return typeof value === 'object' && value !== null; } -function isScope(value: unknown): value is 'app' | 'device' | 'system' { - return value === 'app' || value === 'device' || value === 'system'; -} - function isTransportMode(value: unknown): value is AndroidRecordingDescriptor['transportMode'] { return value === 'local' || value === 'transport-composed'; } -function isOptionalQuality(value: unknown): boolean { - return value === undefined || value === 'medium' || value === 'high'; -} - -function isOptionalApp(value: unknown): boolean { - return ( - value === undefined || - (isObject(value) && - typeof value.bundleId === 'string' && - (value.name === undefined || typeof value.name === 'string')) - ); +function isOptionalText(value: unknown): value is string | undefined { + return value === undefined || typeof value === 'string'; } function isNativeRecordingPath(value: unknown): value is string { diff --git a/packages/platform-apple/src/recording/completion.ts b/packages/platform-apple/src/recording/completion.ts index b3c89fce87..6fc7b50120 100644 --- a/packages/platform-apple/src/recording/completion.ts +++ b/packages/platform-apple/src/recording/completion.ts @@ -63,6 +63,10 @@ export async function completeAppleRecording(params: { * Turns the copy a stop collected into the export, and answers what became of the recorder's own * file (ADR 0024 2.3). The recorder's file is never finalized in place: the overlay and the telemetry * land on the export, and the recorder's file is retired only once that export exists. + * + * `overlayUnavailable` names a reason the caller was promised an overlay this export cannot carry, + * which is refused on the way in and disclosed on the way out rather than attempted with nothing to + * burn in and then apologised for. */ export async function finalizeAppleRecordingFromCollected( params: Readonly<{ @@ -72,6 +76,7 @@ export async function finalizeAppleRecordingFromCollected( collectedPath: string; exportPath: string; nativePath: string; + overlayUnavailable?: string; }>, ): Promise { const { host, snapshot, targetLabel, collectedPath, exportPath, nativePath } = params; @@ -80,13 +85,14 @@ export async function finalizeAppleRecordingFromCollected( if (snapshot.invalidatedReason && !snapshot.showTouches) { throw new Error(`recording invalidated: ${snapshot.invalidatedReason}`); } + const overlayUnavailability = params.overlayUnavailable ?? snapshot.invalidatedReason; let finalization: Awaited>; try { await host.screenRecording.outputs.copy({ from: collectedPath, to: exportPath }); finalization = await asAppErrorAsync(() => host.screenRecording.finalize.complete({ outputPath: exportPath, - showTouches: snapshot.invalidatedReason ? false : snapshot.showTouches, + showTouches: overlayUnavailability === undefined && snapshot.showTouches, gestureEvents: snapshot.gestureEvents, exportQuality: snapshot.exportQuality ?? 'medium', targetLabel, @@ -99,9 +105,9 @@ export async function finalizeAppleRecordingFromCollected( } return { ...finalization, - ...(snapshot.invalidatedReason - ? { overlayWarning: `overlay unavailable: ${snapshot.invalidatedReason}` } - : {}), + ...(overlayUnavailability === undefined + ? {} + : { overlayWarning: `overlay unavailable: ${overlayUnavailability}` }), nativePathDisposition: (await host.screenRecording.outputs.remove(nativePath)) === 'removed' ? 'retired' diff --git a/packages/platform-apple/src/recording/recovery.test.ts b/packages/platform-apple/src/recording/recovery.test.ts index 3c64cf26c8..70d880eef4 100644 --- a/packages/platform-apple/src/recording/recovery.test.ts +++ b/packages/platform-apple/src/recording/recovery.test.ts @@ -1,12 +1,21 @@ import { expect, test, vi } from 'vitest'; +import type { DurableCaptureProgress } from '@agent-device/contracts/durable-resource'; import { localRuntimeOwner } from '@agent-device/contracts/platform-runtime'; +import { recordingFileStore } from '@agent-device/capture-kit/recording-artifact-fixtures'; +import { + collectedRecordingPath, + nativeRecordingPath, +} from '@agent-device/capture-kit/recording-stop-sequence'; +import type { JsonObject } from '@agent-device/contracts/client'; import { createAppleScreenRecordingOperations } from './runtime.ts'; import { appleRecordingHost, coreDevice, processIdentity, recordingInput, + recordingOutputPath, simulator, + simulatorRecorderStart, } from './runtime.fixtures.ts'; test('daemon-loss cleanup distinguishes live, dead, replaced, and corrupt simulator identity', async () => { @@ -68,6 +77,230 @@ test('daemon-loss cleanup distinguishes live, dead, replaced, and corrupt simula expect(inspectProcess).not.toHaveBeenCalled(); }); +type SimulatorHostOptions = Parameters[0]; + +function simulatorHost( + files: ReturnType, + extra: SimulatorHostOptions = {}, +) { + return appleRecordingHost({ + ...extra, + files, + apple: { + ...simulatorRecorderStart(), + inspectProcess: async () => 'missing' as const, + ...extra.apple, + }, + }); +} + +function recoveryOperations(host: ReturnType) { + return createAppleScreenRecordingOperations({ + host, + device: simulator, + owner: localRuntimeOwner('apple'), + signal: new AbortController().signal, + }); +} + +async function startSimulatorRecording( + files: ReturnType, + overrides: Parameters[0] = {}, +) { + return await recoveryOperations(simulatorHost(files)).screenRecordingStart( + recordingInput(overrides), + ); +} + +test('a simulator recording whose recorder died with its daemon exports through the retried stop', async () => { + const files = recordingFileStore(); + const started = await startSimulatorRecording(files); + const nativePath = nativeRecordingPath(recordingOutputPath()); + + const reattached = await recoveryOperations(simulatorHost(files)).screenRecordingReattach({ + envelope: started.envelope, + }); + expect(reattached).toMatchObject({ status: 'active' }); + if (reattached.status !== 'active') throw new Error('expected an exportable recording'); + + await expect(reattached.handle.finish()).resolves.toMatchObject({ + status: 'completed', + result: { + backend: 'simctl recordVideo', + outPath: recordingOutputPath(), + scope: 'device', + stopObservation: { recorder: 'confirmed' }, + nativePathDisposition: 'retired', + }, + }); + expect(files.exists(recordingOutputPath())).toBe(true); + expect(files.exists(nativePath)).toBe(false); +}); + +test('a finalized export whose daemon died before the record closed still completes', async () => { + // What the first attempt left behind: the export it wrote, the recorder's file it retired, and the + // copy it discarded once the finalization below was journaled. + const files = recordingFileStore({ [recordingOutputPath()]: 'fake-video' }); + const started = await startSimulatorRecording(files); + await files.outputs.remove(nativeRecordingPath(recordingOutputPath())); + const complete = vi.fn(async () => ({})); + const envelope = stopJournalOn(started.envelope, { + stopObservation: { recorder: 'confirmed' }, + stoppedAtMs: 1_790_000_000_000, + collectedPath: collectedRecordingPath(recordingOutputPath()), + exportPath: recordingOutputPath(), + stopFinalization: { + telemetryPath: '/tmp/capture.gesture-telemetry.json', + nativePathDisposition: 'retired', + }, + }); + + const reattached = await recoveryOperations( + simulatorHost(files, { complete }), + ).screenRecordingReattach({ envelope }); + if (reattached.status !== 'active') throw new Error('expected a finalized export to replay'); + + await expect(reattached.handle.finish(stopProgressFor(envelope))).resolves.toMatchObject({ + status: 'completed', + result: { + outPath: recordingOutputPath(), + telemetryPath: '/tmp/capture.gesture-telemetry.json', + stopObservation: { recorder: 'confirmed' }, + nativePathDisposition: 'retired', + }, + }); + // The export was written by the stop that journaled it, so nothing here re-runs the finalizer. + expect(complete).not.toHaveBeenCalled(); +}); + +test('a collected copy is finalized again when the recorder no longer has its own file', async () => { + const files = recordingFileStore(); + const started = await startSimulatorRecording(files); + const collectedPath = collectedRecordingPath(recordingOutputPath()); + await files.outputs.copy({ from: nativeRecordingPath(recordingOutputPath()), to: collectedPath }); + await files.outputs.remove(nativeRecordingPath(recordingOutputPath())); + const copy = vi.fn(async ({ from, to }: Readonly<{ from: string; to: string }>) => { + const bytes = files.files.get(from); + if (bytes === undefined) throw new Error(`ENOENT: ${from}`); + files.files.set(to, bytes); + }); + const complete = vi.fn(async () => ({ + telemetryPath: '/tmp/capture.gesture-telemetry.json', + })); + const envelope = stopJournalOn(started.envelope, { + stopObservation: { recorder: 'confirmed' }, + stoppedAtMs: 1_790_000_000_000, + collectedPath, + }); + + const reattached = await recoveryOperations( + simulatorHost(files, { complete, outputs: { copy } }), + ).screenRecordingReattach({ envelope }); + if (reattached.status !== 'active') throw new Error('expected a collectable copy to finish'); + + await expect(reattached.handle.finish(stopProgressFor(envelope))).resolves.toMatchObject({ + status: 'completed', + result: { + outPath: recordingOutputPath(), + telemetryPath: '/tmp/capture.gesture-telemetry.json', + stopObservation: { recorder: 'confirmed' }, + }, + }); + expect(files.exists(recordingOutputPath())).toBe(true); + // The recorder's own file is gone. A stop that collected again would fail on its way to an export the + // manifest already describes. + expect(copy.mock.calls.map(([attempt]) => attempt.from)).toEqual([collectedPath]); +}); + +test('a journaled copy that cannot be read is answered as a loss instead of a fresh collect', async () => { + const started = await startSimulatorRecording(recordingFileStore()); + + await expect( + recoveryOperations(simulatorHost(recordingFileStore())).screenRecordingReattach({ + envelope: stopJournalOn(started.envelope, { + stopObservation: { recorder: 'confirmed' }, + collectedPath: collectedRecordingPath(recordingOutputPath()), + }), + }), + ).resolves.toEqual({ status: 'missing' }); +}); + +/** + * The checkpoints a first attempt journaled, under the keys the shared stop sequence writes them with, + * so `readStopCheckpoints` hands them back to the resumed stop. + */ +function stopJournalOn( + envelope: Awaited>['envelope'], + journal: JsonObject, +) { + return { ...envelope, metadata: { ...(envelope.metadata ?? {}), ...journal } }; +} + +/** How the durable record hands a stop what its earlier attempt journaled. */ +function stopProgressFor(envelope: ReturnType): DurableCaptureProgress { + const learned = { ...(envelope.metadata ?? {}) }; + return Object.freeze({ learned, record: (fact: JsonObject) => Object.assign(learned, fact) }); +} + +test('a recovered export discloses the touch overlay whose events died with the daemon', async () => { + const files = recordingFileStore(); + const started = await startSimulatorRecording(files, { showTouches: true }); + + const reattached = await recoveryOperations(simulatorHost(files)).screenRecordingReattach({ + envelope: started.envelope, + }); + if (reattached.status !== 'active') throw new Error('expected an exportable recording'); + const finished = await reattached.handle.finish(); + if (finished.status !== 'completed') throw new Error('expected a completed export'); + expect(finished.result.overlayWarning).toContain('overlay unavailable'); +}); + +test('a simulator recording left by a manifest without export coordinates is still a loss', async () => { + const files = recordingFileStore(); + const started = await startSimulatorRecording(files); + const { recording: _dropped, ...bodyWithoutCoordinates } = started.envelope.descriptor.body; + + await expect( + recoveryOperations(simulatorHost(files)).screenRecordingReattach({ + envelope: { + ...started.envelope, + descriptor: { ...started.envelope.descriptor, body: bodyWithoutCoordinates }, + }, + }), + ).resolves.toEqual({ status: 'missing' }); +}); + +test('a gone recorder with no readable video left behind is answered as a loss', async () => { + const started = await startSimulatorRecording(recordingFileStore()); + + await expect( + recoveryOperations(simulatorHost(recordingFileStore())).screenRecordingReattach({ + envelope: started.envelope, + }), + ).resolves.toEqual({ status: 'missing' }); +}); + +test('unreadable export coordinates refuse reattach before the recorder is probed', async () => { + const files = recordingFileStore(); + const started = await startSimulatorRecording(files); + const inspectProcess = vi.fn(async () => 'missing' as const); + + await expect( + recoveryOperations(simulatorHost(files, { apple: { inspectProcess } })).screenRecordingReattach( + { + envelope: { + ...started.envelope, + descriptor: { + ...started.envelope.descriptor, + body: { ...started.envelope.descriptor.body, recording: { outPath: 42 } }, + }, + }, + }, + ), + ).resolves.toMatchObject({ status: 'unreattachable', reason: 'descriptor-invalid' }); + expect(inspectProcess).not.toHaveBeenCalled(); +}); + test('runner recovery never stops a replacement session owner', async () => { const operations = createAppleScreenRecordingOperations({ host: appleRecordingHost(), diff --git a/packages/platform-apple/src/recording/recovery.ts b/packages/platform-apple/src/recording/recovery.ts index 7b1ae4cfcc..b0b4a91a94 100644 --- a/packages/platform-apple/src/recording/recovery.ts +++ b/packages/platform-apple/src/recording/recovery.ts @@ -1,17 +1,34 @@ import { deviceIdentity, isIosFamily, type DeviceInfo } from '@agent-device/kernel/device'; -import type { CleanupOutcome } from '@agent-device/contracts/durable-resource'; -import type { DurableDescriptorCodec } from '@agent-device/contracts/durable-resource-envelope'; +import type { + CleanupOutcome, + ReattachOutcome, + ResourceUnreattachableReason, +} from '@agent-device/contracts/durable-resource'; +import type { + DurableDescriptorCodec, + DurableResourceEnvelope, +} from '@agent-device/contracts/durable-resource-envelope'; import type { ManagedProcessIdentity, OwnedProcessRecordScope, } from '@agent-device/contracts/platform-runtime-host'; import type { RuntimeOwnerRef } from '@agent-device/contracts/platform-runtime'; import type { ScreenRecordingRuntimeHost } from '@agent-device/contracts/screen-recording-runtime-host'; +import { isRecord } from '@agent-device/kernel/record'; import { - type ScreenRecordingStartInput, SCREEN_RECORDING_RESOURCE_KIND, + type ScreenRecordingCompletion, + type ScreenRecordingLiveHandle, + type ScreenRecordingLiveSnapshot, + type ScreenRecordingStartInput, } from '@agent-device/contracts/screen-recording-runtime'; import { createDurableResourceEnvelope, encodeDurableDescriptor } from '@agent-device/capture-kit'; +import { + RECORDING_FACTS_KEYS, + recordingFactsAreValid, +} from '@agent-device/capture-kit/recording-facts'; +import type { RecordingStopProgress } from '@agent-device/contracts/recording-stop-progress'; +import { readStopCheckpoints } from '@agent-device/capture-kit/recording-stop-sequence'; export type AppleScreenRecordingOperationHost = Readonly<{ screenRecording: Pick< @@ -20,11 +37,34 @@ export type AppleScreenRecordingOperationHost = Readonly<{ >; }>; +/** + * The export a simulator recording owes its caller, durable so a `record stop` that lost its daemon + * mid-export can still produce it. These are the caller-facing facts of the live snapshot, never the + * recorder's: `outPath` is the path the caller asked for, which is not the descriptor's `outputPath` + * (that one is where `simctl` writes), and `startedAt` is the launch the duration is measured from. + * + * A manifest written before these coordinates existed cannot name an export it never recorded, so + * the field stays optional and its absence is answered exactly as such a manifest was answered then. + */ +/** The live-snapshot keys a recovered export is described by, which is the caller's own request. */ +const SIMULATOR_EXPORT_KEYS = [ + 'outPath', + 'startedAt', + 'clientOutPath', + ...RECORDING_FACTS_KEYS, +] as const; + +export type AppleSimulatorExportCoordinates = Pick< + ScreenRecordingLiveSnapshot, + (typeof SIMULATOR_EXPORT_KEYS)[number] +>; + export type AppleRecordingDescriptor = | Readonly<{ backend: 'simctl'; outputPath: string; processes: readonly ManagedProcessIdentity[]; + recording?: AppleSimulatorExportCoordinates; }> | Readonly<{ backend: 'runner'; @@ -46,6 +86,7 @@ const encodeAppleRecordingDescriptor: AppleRecordingDescriptorCodec['encode'] = backend: descriptor.backend, outputPath: descriptor.outputPath, processes: descriptor.processes.map((process) => ({ ...process })), + ...(descriptor.recording === undefined ? {} : { recording: { ...descriptor.recording } }), }; return encoded; } @@ -85,6 +126,21 @@ export function createAppleRecordingEnvelope(params: { }); } +/** + * The part of a live snapshot that outlives its daemon. What the recorder's own process held — gesture + * events, the touch reference frame, a runner's clock — is deliberately absent: recovery cannot invent + * it, and a recovered export states what it could not honour instead of pretending otherwise. + */ +export function simulatorExportCoordinates( + snapshot: ScreenRecordingLiveSnapshot, +): AppleSimulatorExportCoordinates { + const coordinates: Partial = {}; + for (const key of SIMULATOR_EXPORT_KEYS) { + if (snapshot[key] !== undefined) Object.assign(coordinates, { [key]: snapshot[key] }); + } + return coordinates as AppleSimulatorExportCoordinates; +} + export async function cleanupAppleRecording( host: AppleScreenRecordingOperationHost, device: DeviceInfo, @@ -171,48 +227,159 @@ async function cleanupRunner( } } +/** What a recovered simulator export needs besides the coordinates its manifest kept. */ +export type AppleSimulatorExportRestore = Readonly<{ + recording: AppleSimulatorExportCoordinates; + /** The file `simctl` wrote, or the copy a first attempt already collected. */ + nativePath: string; + cleanup(): Promise; +}>; + +/** + * What an envelope allows next. `restore-export` is the answer when the recorder is gone and the export + * is still reachable: recovery holds the facts and runs no stop, so the runtime that owns the stop + * sequence builds the handle from them. + */ +export type AppleRecordingReattachment = + | ReattachOutcome + | (AppleSimulatorExportRestore & Readonly<{ status: 'restore-export' }>); + export async function reattachAppleRecording( - host: AppleScreenRecordingOperationHost, - device: DeviceInfo, - body: Parameters[0], -) { - const decoded = descriptorCodec.decode(body); + params: Readonly<{ + host: AppleScreenRecordingOperationHost; + device: DeviceInfo; + envelope: DurableResourceEnvelope; + }>, +): Promise { + const { host, device, envelope } = params; + const decoded = descriptorCodec.decode(envelope.descriptor.body); if (decoded.status !== 'decoded') { - return { - status: 'unreattachable' as const, - reason: 'descriptor-invalid' as const, - message: decoded.message, - }; + return unreattachableAppleRecording('descriptor-invalid', decoded.message); } if (!descriptorMatchesAppleDevice(device, decoded.descriptor)) { - return { - status: 'unreattachable' as const, - reason: 'descriptor-invalid' as const, - message: 'Apple screen-recording descriptor does not match the bound device.', - }; + return unreattachableAppleRecording( + 'descriptor-invalid', + 'Apple screen-recording descriptor does not match the bound device.', + ); + } + return decoded.descriptor.backend === 'simctl' + ? await reattachSimulatorRecording(params, decoded.descriptor) + : await reattachRunnerRecording(host, device, decoded.descriptor); +} + +/** + * A `simctl` recorder that is proven gone is the ordinary state of a recording whose daemon died, + * and it says nothing about the file that recorder already wrote (ADR 0024 2.2). The manifest's + * coordinates plus that file are what a retried `record stop` still owes the caller, so this answers + * with a handle that finishes the export instead of with a loss nobody observed. + */ +/** + * A `simctl` recorder that is proven gone is the ordinary state of a recording whose daemon died, and it + * says nothing about what the export can still become (ADR 0024 2.2). So the answer comes from what a + * resumed stop would still have to read: nothing when the first attempt journaled a finalization, that + * copy when it journaled one, and otherwise the recorder's own file. Nothing reachable is reported as a + * loss. + */ +async function reattachSimulatorRecording( + params: Readonly<{ + host: AppleScreenRecordingOperationHost; + envelope: DurableResourceEnvelope; + }>, + descriptor: Extract, +): Promise { + const ownership = await Promise.all( + descriptor.processes.map( + async (marker) => await params.host.screenRecording.apple.inspectProcess(marker), + ), + ); + if (!ownership.every((value) => value === 'missing')) { + return unreattachableAppleRecording( + 'transport-not-reattachable', + ownership.includes('ownership-lost') + ? 'Apple recording ownership no longer matches the durable descriptor.' + : 'Apple screen recordings require exact cleanup after daemon restart.', + ); + } + const { recording } = descriptor; + const source = resumedExportSource( + readStopCheckpoints(params.envelope.metadata), + descriptor.outputPath, + ); + if ( + recording === undefined || + (source !== undefined && !(await recordingRemains(params.host, source))) + ) { + return { status: 'missing' }; } - const ownership = - decoded.descriptor.backend === 'simctl' - ? await Promise.all( - decoded.descriptor.processes.map( - async (marker) => await host.screenRecording.apple.inspectProcess(marker), - ), - ) - : [ - await host.screenRecording.apple.inspectRunner( - device, - decoded.descriptor.runnerSessionId, - decoded.descriptor.runnerAuthority, - ), - ]; - if (ownership.every((value) => value === 'missing')) return { status: 'missing' as const }; return { - status: 'unreattachable' as const, - reason: 'transport-not-reattachable' as const, - message: ownership.includes('ownership-lost') + status: 'restore-export', + recording, + nativePath: descriptor.outputPath, + cleanup: async () => + await cleanupSimulator(params.host, descriptor.processes, params.envelope.sessionId), + }; +} + +/** + * The file a resumed stop will still read, or `undefined` when it will read none. A journaled + * finalization is replayed as it stands and its copy is then discarded, a step that tolerates the copy + * already being gone; a journaled copy is what `finalize` runs from; and a stop that journaled neither + * collects from the recorder's own path again. + */ +function resumedExportSource( + learned: RecordingStopProgress, + nativePath: string, +): string | undefined { + if (learned.finalization !== undefined) return undefined; + return learned.collectedPath ?? nativePath; +} + +async function reattachRunnerRecording( + host: AppleScreenRecordingOperationHost, + device: DeviceInfo, + descriptor: Extract, +): Promise { + const ownership = await host.screenRecording.apple.inspectRunner( + device, + descriptor.runnerSessionId, + descriptor.runnerAuthority, + ); + if (ownership === 'missing') return { status: 'missing' }; + return unreattachableAppleRecording( + 'transport-not-reattachable', + ownership === 'ownership-lost' ? 'Apple recording ownership no longer matches the durable descriptor.' : 'Apple screen recordings require exact cleanup after daemon restart.', - }; + ); +} + +/** + * Whether the recorder's own file can still become an export. The container sniff is the read-only + * probe the stop itself runs on its collected copy, and it is the most this step can promise: a file + * that fails it is exactly the recording a retry would refuse, so nothing is offered for it. + */ +/** + * Whether a file the resumed stop would read can still become an export. The container sniff is the + * read-only probe the stop itself runs on that file, and it is the most this step can promise: a file + * that fails it is exactly the recording a retry would refuse. + */ +async function recordingRemains( + host: AppleScreenRecordingOperationHost, + outputPath: string, +): Promise { + try { + await host.screenRecording.finalize.sniff({ outputPath }); + return true; + } catch { + return false; + } +} + +function unreattachableAppleRecording( + reason: ResourceUnreattachableReason, + message: string, +): AppleRecordingReattachment { + return { status: 'unreattachable', reason, message }; } function decodeAppleRecordingDescriptor( @@ -227,12 +394,43 @@ function decodeAppleRecordingDescriptor( function decodeSimulatorDescriptor(body: Record, outputPath: string) { const processes = decodeProcessIdentities(body.processes); - return processes - ? ({ - status: 'decoded', - descriptor: Object.freeze({ backend: 'simctl', outputPath, processes }), - } as const) - : invalidDescriptor(); + const recording = readSimulatorExportCoordinates(body.recording); + if (!processes || recording === 'invalid') return invalidDescriptor(); + return { + status: 'decoded' as const, + descriptor: Object.freeze({ + backend: 'simctl' as const, + outputPath, + processes, + ...(recording === undefined ? {} : { recording }), + }), + } as const; +} + +/** + * What the durable coordinates have to say for themselves. `invalid` is a manifest whose recording + * facet cannot be trusted, which is answered exactly like any other unreadable descriptor: no + * reattach, no cleanup, the record stays for a human. + */ +function readSimulatorExportCoordinates( + value: unknown, +): AppleSimulatorExportCoordinates | undefined | 'invalid' { + if (value === undefined) return undefined; + return isRecord(value) && isWholeExportCoordinates(value) + ? Object.freeze(value as unknown as AppleSimulatorExportCoordinates) + : 'invalid'; +} + +/** The facts a recovered export computes on rather than repeats, whole or absent. */ +function isWholeExportCoordinates( + value: Record, +): value is AppleSimulatorExportCoordinates { + return ( + isNonemptyString(value.outPath) && + isFiniteNumber(value.startedAt) && + isOptionalText(value.clientOutPath) && + recordingFactsAreValid(value) + ); } function decodeRunnerDescriptor(body: Record, outputPath: string) { @@ -257,6 +455,14 @@ function isNonemptyString(value: unknown): value is string { return typeof value === 'string' && value.length > 0; } +function isOptionalText(value: unknown): value is string | undefined { + return value === undefined || isNonemptyString(value); +} + +function isFiniteNumber(value: unknown): value is number { + return typeof value === 'number' && Number.isFinite(value); +} + function isRunnerAuthority(value: unknown): value is 'local-lease' | 'scoped-provider' { return value === 'local-lease' || value === 'scoped-provider'; } diff --git a/packages/platform-apple/src/recording/runtime.fixtures.ts b/packages/platform-apple/src/recording/runtime.fixtures.ts index 190970e65c..90381e10e4 100644 --- a/packages/platform-apple/src/recording/runtime.fixtures.ts +++ b/packages/platform-apple/src/recording/runtime.fixtures.ts @@ -65,6 +65,19 @@ export function recordingInput( }; } +export function simulatorRecorderStart(): Pick< + ScreenRecordingRuntimeHost['apple'], + 'startSimulator' +> { + return { + startSimulator: async () => ({ + markers: [processIdentity], + wait: new Promise(() => {}), + terminate: async () => {}, + }), + }; +} + export function appleRecordingHost( options: { apple?: Partial; @@ -114,7 +127,13 @@ export function appleRecordingHost( apple, outputs: Object.assign({}, store.outputs, options.outputs), finalize: { - sniff: options.sniff ?? (async () => {}), + // The real sniff reads the file and refuses one that is not there or not a video, which is + // what lets a recovery ask whether a recorder's file can still become an export. + sniff: + options.sniff ?? + (async ({ outputPath }: Readonly<{ outputPath: string }>) => { + if (!store.exists(outputPath)) throw new Error(`no recording file at ${outputPath}`); + }), complete: options.complete ?? (async () => ({})), }, ownedProcesses: options.ownedProcesses ?? { replace: () => {}, clear: () => {} }, diff --git a/packages/platform-apple/src/recording/runtime.ts b/packages/platform-apple/src/recording/runtime.ts index 927ad959a9..e3e5deff41 100644 --- a/packages/platform-apple/src/recording/runtime.ts +++ b/packages/platform-apple/src/recording/runtime.ts @@ -28,12 +28,15 @@ import { cleanupAppleRecording, createAppleRecordingEnvelope, reattachAppleRecording, + simulatorExportCoordinates, type AppleRecordingDescriptor, type AppleScreenRecordingOperationHost, + type AppleSimulatorExportRestore, } from './recovery.ts'; import { validateAppleSimulatorRecording } from './validation.ts'; const SIMULATOR_TARGET_LABEL = 'iOS recording'; +const SIMULATOR_BACKEND_LABEL = 'simctl recordVideo'; export function appleScreenRecordingFacts(device: DeviceInfo) { if (device.appleOs === 'watchos') @@ -61,8 +64,12 @@ export function createAppleScreenRecordingOperations(params: { return Object.freeze({ screenRecordingStart: async (input) => await startAppleRecording({ host, device, owner, input, signal }), - screenRecordingReattach: async (input) => - await reattachAppleRecording(host, device, input.envelope.descriptor.body), + screenRecordingReattach: async (input) => { + const reattached = await reattachAppleRecording({ host, device, envelope: input.envelope }); + return reattached.status === 'restore-export' + ? { status: 'active', handle: simulatorExportHandle({ host, restored: reattached }) } + : reattached; + }, screenRecordingCleanup: async (input) => await cleanupAppleRecording( host, @@ -147,24 +154,27 @@ async function startAppleSimulatorRecording(params: AppleRecordingStartParams) { }), }; }; + const startedSnapshot = snapshot(input, SIMULATOR_BACKEND_LABEL, {}, clockAnchor); return startResult({ device, owner, input, - descriptor: { backend: 'simctl', outputPath: nativePath, processes }, - snapshot: snapshot(input, 'simctl recordVideo', {}, clockAnchor), + descriptor: { + backend: 'simctl', + outputPath: nativePath, + processes, + recording: simulatorExportCoordinates(startedSnapshot), + }, + snapshot: startedSnapshot, finish: (current, progress) => stopAndExportScreenRecording({ snapshot: current, progress, steps: { stop: stopSimulatorRecorder, - // The copy gets the container sniff before it is checkpointed; the full playability verdict - // runs once, on the export `finalize` writes from it. collect: async (collectedPath) => { try { - await host.screenRecording.outputs.copy({ from: nativePath, to: collectedPath }); - await host.screenRecording.finalize.sniff({ outputPath: collectedPath }); + await collectSimulatorRecording(host, nativePath, collectedPath); } catch (collectError) { throw recorderExitEndedTheRecording(collectError, recorderExit, recorderResult); } @@ -201,6 +211,77 @@ async function startAppleSimulatorRecording(params: AppleRecordingStartParams) { }); } +/** + * The stop a `simctl` recording gets when its recorder died with its daemon. There is no recorder left + * to signal and no gesture event left to burn in, and every other step is the one the first stop would + * have run — including which of them are still owed, which the shared sequence decides from the + * checkpoints the first attempt journaled. + */ +function simulatorExportHandle( + params: Readonly<{ + host: AppleScreenRecordingOperationHost; + restored: AppleSimulatorExportRestore; + }>, +): ScreenRecordingLiveHandle { + const { + host, + restored: { recording, nativePath, cleanup }, + } = params; + const snapshot: ScreenRecordingLiveSnapshot = Object.freeze({ + ...recording, + backend: SIMULATOR_BACKEND_LABEL, + gestureEvents: [], + }); + return createScreenRecordingLiveHandle(snapshot, { + finish: (current, progress) => + stopAndExportScreenRecording({ + snapshot: current, + progress, + steps: { + stop: async () => ({ + observation: { recorder: 'confirmed' as const }, + warning: + 'simctl recordVideo had already ended when record stop reattached to it; ' + + 'the video covers only what the recorder wrote before it stopped.', + }), + collect: async (collectedPath) => { + await collectSimulatorRecording(host, nativePath, collectedPath); + }, + finalize: async ({ collectedPath, exportPath }) => + await finalizeAppleRecordingFromCollected({ + host, + snapshot: current, + targetLabel: SIMULATOR_TARGET_LABEL, + collectedPath, + exportPath, + nativePath, + ...(recording.showTouches + ? { + overlayUnavailable: + 'the daemon that held the touch events ended before record stop', + } + : {}), + }), + discard: async (collectedPath) => { + await host.screenRecording.outputs.remove(collectedPath); + }, + }, + }), + forceCleanup: async () => await cleanup(), + }); +} + +async function collectSimulatorRecording( + host: AppleScreenRecordingOperationHost, + nativePath: string, + collectedPath: string, +): Promise { + // The copy gets the container sniff before it is checkpointed; the full playability verdict + // runs once, on the export `finalize` writes from it. + await host.screenRecording.outputs.copy({ from: nativePath, to: collectedPath }); + await host.screenRecording.finalize.sniff({ outputPath: collectedPath }); +} + async function startAppleRunnerRecording(params: AppleRecordingStartParams) { const { host, device, owner, input, signal } = params; const appBundleId = input.activeSessionApp?.bundleId; diff --git a/scripts/layering/package-boundaries.test.ts b/scripts/layering/package-boundaries.test.ts index f2f423170b..dd3402c096 100644 --- a/scripts/layering/package-boundaries.test.ts +++ b/scripts/layering/package-boundaries.test.ts @@ -423,6 +423,7 @@ test('the real tree parses, declares, and passes R11', () => { '@agent-device/capture-kit/quality-warnings', '@agent-device/capture-kit/react-native-overlay', '@agent-device/capture-kit/recording-artifact-fixtures', + '@agent-device/capture-kit/recording-facts', '@agent-device/capture-kit/recording-mp4-duration', '@agent-device/capture-kit/recording-mp4-fixtures', '@agent-device/capture-kit/recording-output-path', From ddc0d50b9e2e7ce0256d5f3142c672cb45870e9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 22 Sep 2026 08:56:09 +0200 Subject: [PATCH 005/132] test(ios): close the snapshot convergence evidence sweep at a final head (#2188) (#2750) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(ios-benchmark): wait out unmounted fixture screens during setup admission The proxy leg stopped on `fixture-anchor` for every deep-linked screen it reached: the persistent client reads the anchor right after `apps.open`, and that read lands before the app mounts its first tree. Measured on the fixture app, the snapshot taken immediately after an `open --relaunch --launch-url` returns the single Application node, while a read 1.5s later returns 31 nodes with the anchor. The fresh-process CLI leg hides the same gap behind its per-read process spawn. Setup reads are untimed, so the shared admission now re-observes until the expected anchor is exposed, bounded by FIXTURE_ANCHOR_ADMISSION_BUDGET_MS, and still stops the run on the same typed reason when a fixture never exposes it. An unsuccessful observation keeps failing immediately rather than polling. Also repairs the test module's stale `FixturePreparationResult` import: the scripts directory is erased rather than typechecked, so the rename to `FixtureOperationResult` went unnoticed. * docs(evidence): record the iOS snapshot convergence final head The sweep that gated #2188 on #2199 completed at `7c434b5758`: cold/cold-cold, first-interaction, warm/relaunch with the package-size leg, and the 0/20/80 ms proxy matrix, every leg clean and free of failed samples. The four Markdown summaries ride along; the record states what the numbers prove and the deviations that bound them — target and runtime, host load, the fixture app's changed trees, and the 196-commit span behind the package-size delta. Two findings that should not travel quietly: the warm snapshot daemon cost is about double the baseline under a controlled side-by-side diagnostic, and the #2198 slice corpora come from unmerged branch heads, so they are not parity baselines for anything. * chore(gates): pin the iOS snapshot convergence final corpus `PUBLISHED_EVIDENCE` becomes `PUBLISHED_CORPORA`: the pinned evidence is now two corpora measured at two revisions on two runtimes, each carrying its own tag and evidence commit, and `PUBLISHED_EVIDENCE` derives from them. A missing file is now fetched with a hint naming the corpus that actually holds it, instead of a single tag that is wrong for one of them. Disjointness, naming, and the tag and commit each corpus is cited by are tests rather than comments. Also stops exporting the setup admission's internals, which the dead-code audit rightly flags. --- ...s-snapshot-convergence-final-2026-09-21.md | 190 ++++++++++++++++++ .../ios-snapshot-benchmark/evidence.test.ts | 44 +++- scripts/ios-snapshot-benchmark/evidence.ts | 103 ++++++++-- .../ios-snapshot-benchmark/evidence/README.md | 43 +++- .../ios-snapshot-cold-local-7c434b575.md | 34 ++++ ...pshot-first-interaction-local-7c434b575.md | 28 +++ .../evidence/ios-snapshot-proxy-7c434b575.md | 58 ++++++ ...-snapshot-warm-relaunch-local-7c434b575.md | 37 ++++ .../fixture-admission.test.ts | 79 +++++++- .../fixture-admission.ts | 79 ++++++-- .../proxy-client-support.test.ts | 12 ++ .../proxy-client-support.ts | 56 +++--- 12 files changed, 681 insertions(+), 82 deletions(-) create mode 100644 docs/evidence/ios-snapshot-convergence-final-2026-09-21.md create mode 100644 scripts/ios-snapshot-benchmark/evidence/ios-snapshot-cold-local-7c434b575.md create mode 100644 scripts/ios-snapshot-benchmark/evidence/ios-snapshot-first-interaction-local-7c434b575.md create mode 100644 scripts/ios-snapshot-benchmark/evidence/ios-snapshot-proxy-7c434b575.md create mode 100644 scripts/ios-snapshot-benchmark/evidence/ios-snapshot-warm-relaunch-local-7c434b575.md diff --git a/docs/evidence/ios-snapshot-convergence-final-2026-09-21.md b/docs/evidence/ios-snapshot-convergence-final-2026-09-21.md new file mode 100644 index 0000000000..37be6760cf --- /dev/null +++ b/docs/evidence/ios-snapshot-convergence-final-2026-09-21.md @@ -0,0 +1,190 @@ +# iOS snapshot convergence final-head evidence + +- Issues: #2188 (gate), #2199 (measurement), #2189 (baseline corpus), #1571/#1626 (corpus lineage) +- Observed: 2026-09-21, 19:03–22:52 CEST (legs sequential, one subprocess-backed gate per host) +- Revision: `7c434b5758` (`docs/ios-snapshot-final-evidence`, clean tree at every leg; `revision.dirty: false`) +- Target: `bench-2188-final` iPhone 17 Pro Simulator, iOS 26.2, UDID `5FEADD02-98E4-4F01-861C-07003C1A3291` +- Host: MacBook Pro `Mac16,8`, Apple M4 Pro, 12 cores, Xcode 26.2 (17C52), Node v26.8.1 +- App: `Agent Device Tester` (`com.callstack.agentdevicelab`) from the CI artifact + `fingerprint.d6b4c1f5e022e54074df584e332d1e0f100c7a0f.ios` (run `35530285864`), repacked with the + measured head's own JavaScript bundle +- Corpora: tag `evidence/ios-snapshot/7c434b575`, evidence commit `96d4951c19`, four files listed + with their SHA-256 digests in [`scripts/ios-snapshot-benchmark/evidence/README.md`](../../scripts/ios-snapshot-benchmark/evidence/README.md) +- Baseline: tag `evidence/ios-snapshot/71fb2483f`, evidence commit `2d4baf461aa`, measured at + `71fb2483f3` on `bench-golden-v2` (iOS 27.0) + +## What this closes + +#2188 was gated on #2199's final exact-head evidence sweep: the package-size delta against #2189, the +conformance/fuzz/property corpus, provider contracts, the Simulator and proxy legs, the no-regrowth +enforcement, and a release record at a named head. All of it now exists at `7c434b5758`. + +The proxy leg was the last blocker, and it was blocked by the harness, not by the product. The +anchor admission the harness gained at `71fb2483f3` reads the tree exactly once after `apps.open`. +On this target the first tree lands later than that read: the snapshot taken immediately after an +`open --relaunch --launch-url` returns the single `Application` node, while a read 1.5 s later +returns 31 nodes with the `Catalog` anchor. Every deep-linked screen the leg reached stopped with +`fixture-anchor`. `45e4c594a1..7c434b5758` puts a bounded wait into that untimed setup admission +(`FIXTURE_ANCHOR_ADMISSION_BUDGET_MS`, 30 s, 500 ms polling) and keeps the same typed stop when a +fixture never exposes its anchor. No `src/`, `packages/`, app, or CLI runtime file differs between +those two commits; the measured binary is the measured head's own `pnpm build` output. + +## Deviations from the baseline corpus + +Stated in full, because they bound what these numbers can prove. + +| Deviation | This corpus | Baseline corpus | +| --- | --- | --- | +| Simulator and runtime | `bench-2188-final`, iOS 26.2 | `bench-golden-v2`, iOS 27.0 | +| Why | `bench-golden-v2`'s iOS 27.0 runtime was not installed on this host at measurement time | — | +| Host load (1-min average, sampled per leg) | warm/relaunch 4.2–31.2, proxy 6.4–30.3, cold 5.3–96.7 | quiet | +| Fixture app trees | catalog 31 nodes, nested-scroll 23, checkout 25, alert 26, Settings 18, inert 4 | catalog 35, nested-scroll 25, checkout 29, alert 28, Settings 18, inert 6 | +| Package-size span | `71fb2483f3..7c434b5758`: 196 commits of unrelated product work | — | + +Because the fixture trees differ, response sizes are not comparable cell for cell, and any timing +delta on a screen whose tree changed is only partly attributable to the tooling. Timings are bounded +observations under uncontrolled host load, not a general performance guarantee. + +## Result — cold and cold-cold (10 samples per cell, fresh CLI process) + +| State | Screen | Wall median | Wall median (base) | Daemon median | Daemon median (base) | +| --- | --- | ---: | ---: | ---: | ---: | +| cold-cold | quiet | 7,159 | 17,544 | 5,709 | 15,161 | +| cold-cold | list | 7,580 | 19,170 | 5,881 | 17,191 | +| cold-cold | nested-scroll | 8,091 | 19,407 | 6,374 | 17,773 | +| cold-cold | alert | 7,461 | 19,448 | 5,689 | 17,436 | +| cold-cold | system-surface | 6,553 | 20,235 | 4,961 | 17,974 | +| cold-cold | xctest-stress | 7,273 | 20,811 | 5,761 | 18,506 | +| cold | quiet | 5,743 | 6,643 | 4,409 | 5,886 | +| cold | list | 5,847 | 6,851 | 4,420 | 5,897 | +| cold | nested-scroll | 6,058 | 6,713 | 4,619 | 5,884 | +| cold | alert | 5,706 | 6,674 | 4,310 | 5,868 | +| cold | system-surface | 5,028 | 6,819 | 3,662 | 5,957 | +| cold | xctest-stress | 5,679 | 6,714 | 4,323 | 5,874 | + +Milliseconds. `cold-cold` shuts the Simulator down, clears benchmark-owned derived data, and boots it +again before every sample; `cold` stops the daemon and terminates the app. Cold-cold wall medians +fall by 57–68% and daemon medians by 62–72% against the baseline, measured while this host carried +substantially more load than the baseline's. Cold (booted Simulator, cold runner) improves by 9–26%. + +## Result — warm, relaunch, and first interaction + +Warm and relaunch carry 20 samples per cell; first interaction carries 10 and has no baseline-corpus +counterpart, so it is reported on its own. + +| Screen | Warm wall | Warm wall (base) | Warm daemon | Warm daemon (base) | Relaunch wall | Relaunch wall (base) | First interaction wall | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| quiet | 169 | 154 | 89 | 50 | 3,876 | 3,645 | 928 | +| list | 291 | 312 | 202 | 209 | 3,888 | 4,378 | 1,436 | +| nested-scroll | 250 | 187 | 167 | 83 | 3,872 | 4,313 | 741 | +| alert | 224 | 207 | 139 | 104 | 3,896 | 4,370 | 1,005 | +| system-surface | 208 | 210 | 130 | 105 | 3,576 | 4,917 | 1,119 | +| xctest-stress | 213 | 208 | 128 | 105 | 3,868 | 4,260 | 1,302 | + +Milliseconds. Relaunch is at or better than the baseline on five of six screens. Warm wall is within +±10% of the baseline on five of six screens; warm daemon duration is uniformly higher, which is the +finding below. + +## Result — proxy transport (20 samples per screen at each added RTT) + +The headline of the convergence work: a warm snapshot crossed the tunnel with far fewer round trips +than at the baseline, so added latency costs far less. + +| Screen | Transport | 0 ms | 20 ms | 80 ms | 0 ms (base) | 20 ms (base) | 80 ms (base) | +| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | +| quiet | fresh CLI | 184 | 293 | 556 | 233 | 1,057 | 1,801 | +| quiet | persistent client | 103 | 158 | 282 | 273 | 562 | 806 | +| list | fresh CLI | 304 | 415 | 691 | 431 | 1,221 | 1,328 | +| list | persistent client | 214 | 280 | 408 | 254 | 719 | 729 | +| nested-scroll | fresh CLI | 260 | 377 | 643 | 188 | 1,116 | 1,108 | +| nested-scroll | persistent client | 179 | 240 | 381 | 138 | 590 | 616 | +| alert | fresh CLI | 241 | 343 | 615 | 210 | 1,113 | 1,362 | +| alert | persistent client | 148 | 203 | 351 | 108 | 622 | 637 | +| system-surface | fresh CLI | 217 | 319 | 595 | 209 | 1,119 | 1,145 | +| system-surface | persistent client | 128 | 183 | 330 | 105 | 537 | 638 | +| xctest-stress | fresh CLI | 229 | 340 | 562 | 225 | 1,136 | 1,395 | +| xctest-stress | persistent client | 136 | 197 | 314 | 127 | 612 | 619 | + +Milliseconds, wall-clock medians. Going from 0 to 80 ms of added RTT costs 178–387 ms per snapshot +here, against 475–1,568 ms at the baseline. On the quiet fixture over the fresh CLI transport that +difference is about twenty RTT-bound exchanges per snapshot at the baseline against about five here; +over the persistent client it is about seven against about two. Every cell completed with zero failed +samples at every RTT, so the retry/timeout path is not what improved. + +## Result — package size against #2189 + +| Measurement | `7c434b5758` | `71fb2483f3` | Delta | +| --- | ---: | ---: | ---: | +| Packed tarball | 1,409,306 B | 982,960 B | +43.4% | +| Clean-installed package | 4,723,895 B / 543 files | 3,347,347 B / 436 files | +41.1% / +107 | +| Bundled JavaScript | 3,775,024 B raw, 1,258,950 B gzip, 410 files | 2,480,947 B raw, 835,279 B gzip, 330 files | +52.2% raw, +50.7% gzip | + +The span covers 196 commits of unrelated product work, so this delta cannot be attributed to the +snapshot convergence alone; it is recorded as the size at the measured head, and the leg reproduces +to the byte across independent runs (the same `1,409,306` tarball twice). + +## Result — deterministic corpus and no-regrowth enforcement + +| Gate | Command | Result | +| --- | --- | --- | +| Swift/TypeScript presentation differential | `pnpm test:ios-snapshot-differential` | pass (`swift test --package-path apple/snapshot-presentation`, plus 3 differential tests) | +| Engine conformance, properties, transitions, tree | `vitest run packages/capture-kit/src/ios-snapshot-engine` | pass (10 files, 67 tests) | +| Fuzz corpus replay | `vitest run scripts/fuzz/corpus-replay.test.ts` | pass (11 cases) | +| Fuzz worker lane | `pnpm test:fuzz-worker` | pass (11 cases) | +| Provider contracts | `pnpm test:integration:provider` | pass (66 files, 212 tests) | +| No-regrowth rules R72/R73/R74 | `pnpm check:layering` | pass (`ios-snapshot-engine-ownership`, `provider-snapshot-presentation-ownership`, `snapshot-assembly-presentation-neutrality`, wired in `scripts/layering/check.ts`) | +| Benchmark harness unit suite | `vitest run scripts/ios-snapshot-benchmark` | pass (17 files, 50 tests) | +| Deep-button controls | inside every corpus (`deepButtonEvidence`) | the shallow-rule control exits 1 with "changed descendant was omitted by shallow observation"; the safe control exits 0 | +| Affected gate | `pnpm check:affected --run` | see the PR; GitHub stays authoritative for provider integration, coverage, native builds, and device lanes | + +## Finding — warm snapshot daemon cost is about double the baseline + +Recorded here rather than hidden, and not explained by target, runtime, app, or load. + +An off-corpus controlled diagnostic: same Simulator, same installed app, same ambient load +(1-minute average ~21), back-to-back runs of ten warm `interactiveOnly` snapshots of the smallest +fixture (4 nodes, ~3.4 KB), reading the daemon's own per-step duration. + +| CLI under test | Durations (ms) | Median | +| --- | --- | ---: | +| `71fb2483f3` (baseline corpus head) | 68, 47, 46, 44, 43, 43, 43, 45, 47, 46 | 45.5 | +| `7c434b5758` (this head), round 1 | 97, 86, 84, 90, 84, 86, 86, 92, 83, 82 | 86 | +| `7c434b5758` (this head), round 2 | 98, 133, 96, 96, 88, 90, 87, 94, 107, 103 | 96 | + +The published warm cells agree with the diagnostic (89–202 ms against the baseline's 50–209 ms), and +a repeat at a 1-minute load average of 3.8 still measured 86 ms, so contention is not the cause. The +shape is a roughly fixed ~40 ms per snapshot rather than work proportional to tree size. Candidates +in `71fb2483f3..7c434b5758` that add work to every capture: #2644 (reading the `NotEnabled` trait per +node), #2670 (publishing the captured keyboard band as a fact), #2693 (disclosing target +re-activation on every capture-consuming command), and #2661 (the single geometry-normalization +pass). This does not weaken the claims the corpora were measured for — one converged snapshot path, +no presentation branching in the assembly, the RTT and cold-cold improvements above — but it is a +real cost this head carries and it needs its own follow-up. + +## Correction — the #2198 slice corpora are not parity baselines + +The slice corpora published under `2198-slice-a-7616ba222d/`, +`2198-slice-a-first-interaction-fixed-e729321dcc/`, and `2198-slice-b-rtt-318d510769/` were measured +on branch heads (`7616ba222d`, `9189275dcb`, `e729321dcc`) that are **not ancestors** of `main`, and +two of their first-interaction cells carry ten failed samples each (the ambiguous-anchor harness bug +those slices themselves fixed). They are cited here as branch-lineage observations only. The pinned +`71fb2483f3` corpus is the comparison this record rests on. + +## Reproduction + +```sh +# one dedicated simulator, app installed, nothing else running a device +pnpm bench:ios-snapshot -- --udid --screen quiet,list,nested-scroll,alert,system-surface,xctest-stress \ + --mode local --state cold-cold,cold --samples 10 --skip-package-size --out .tmp/cold.json +pnpm bench:ios-snapshot -- --udid --screen quiet,list,nested-scroll,alert,system-surface,xctest-stress \ + --mode local --state first-interaction --samples 10 --skip-package-size --out .tmp/first.json +pnpm bench:ios-snapshot -- --udid --screen quiet,list,nested-scroll,alert,system-surface,xctest-stress \ + --mode local --state warm,relaunch --samples 20 --out .tmp/warm-relaunch.json +pnpm bench:ios-snapshot -- --udid --screen quiet,list,nested-scroll,alert,system-surface,xctest-stress \ + --mode proxy --rtt 0,20,80 --samples 20 --bandwidth-kbps unlimited --packet-loss 0 --skip-package-size \ + --out .tmp/proxy.json +pnpm bench:ios-snapshot:evidence # schema and hash check against PUBLISHED_CORPORA +``` + +Legs must not overlap: one subprocess-backed gate per host, otherwise the samples contend and the +cell admission stops the run rather than reporting mixed timings. diff --git a/scripts/ios-snapshot-benchmark/evidence.test.ts b/scripts/ios-snapshot-benchmark/evidence.test.ts index dc69e244f1..b34a16868e 100644 --- a/scripts/ios-snapshot-benchmark/evidence.test.ts +++ b/scripts/ios-snapshot-benchmark/evidence.test.ts @@ -6,6 +6,7 @@ import { afterEach, test } from 'vitest'; import { DEFAULT_EVIDENCE_DIR, EVIDENCE_FIXTURE_PATH, + PUBLISHED_CORPORA, PUBLISHED_EVIDENCE, checkEvidenceCorpus, fetchEvidenceCommand, @@ -80,13 +81,32 @@ test('the evidence README cites every published hash and the fetch recipe', () = assert.ok(readme.includes(fetchEvidenceCommand())); }); -function oneOfThreePublishedFiles(): EvidenceFile { - const [file, sha256] = Object.entries(PUBLISHED_EVIDENCE)[0]!; +test('each published corpus is disjoint, named for its revision, and cited by tag and commit', () => { + const readme = fs.readFileSync(path.join(DEFAULT_EVIDENCE_DIR, 'README.md'), 'utf8'); + const seen = new Set(); + for (const corpus of PUBLISHED_CORPORA) { + assert.match(corpus.tag, /^refs\/tags\/evidence\/ios-snapshot\/[0-9a-f]{9,}$/); + assert.match(corpus.commit, /^[0-9a-f]{40}$/); + assert.ok(readme.includes(corpus.tag), `README does not cite ${corpus.tag}`); + assert.ok(readme.includes(corpus.commit), `README does not cite ${corpus.commit}`); + for (const file of Object.keys(corpus.files)) { + assert.ok(!seen.has(file), `${file} is published by two corpora`); + seen.add(file); + assert.ok( + file.endsWith(`-${corpus.tag.split('/').pop()}.json`), + `${file} is not named for ${corpus.tag}`, + ); + } + } +}); + +function onePublishedFile(): EvidenceFile { + const [file, sha256] = Object.entries(PUBLISHED_CORPORA[0]!.files)[0]!; return { file, sha256, published: 'match', - revision: '71fb2483f30d90e615e949601c836aeebbf450c5', + revision: PUBLISHED_CORPORA[0]!.revision, status: 'completed', cells: 2, errors: [], @@ -95,15 +115,23 @@ function oneOfThreePublishedFiles(): EvidenceFile { test('the default evidence directory must hold the complete published corpus, named files and all', () => { const dir = temporaryEvidenceDir({}); - assert.throws( - () => checkEvidenceCorpus(dir, [oneOfThreePublishedFiles()], true), - /is missing published evidence file\(s\): ios-snapshot-warm-relaunch-local-71fb2483f\.json, ios-snapshot-proxy-71fb2483f\.json/, - ); + const present = onePublishedFile(); + let message = ''; + try { + checkEvidenceCorpus(dir, [present], true); + } catch (error) { + message = error instanceof Error ? error.message : String(error); + } + assert.match(message, /is missing published evidence file\(s\)/); + for (const file of Object.keys(PUBLISHED_EVIDENCE)) { + if (file === present.file) continue; + assert.ok(message.includes(file), `the error omits ${file}`); + } }); test('an explicit --evidence-dir stays permissive: a partial corpus does not fail completeness', () => { const dir = temporaryEvidenceDir({}); - assert.doesNotThrow(() => checkEvidenceCorpus(dir, [oneOfThreePublishedFiles()], false)); + assert.doesNotThrow(() => checkEvidenceCorpus(dir, [onePublishedFile()], false)); }); test('fetched evidence under the in-tree directory matches the published corpus', (context) => { diff --git a/scripts/ios-snapshot-benchmark/evidence.ts b/scripts/ios-snapshot-benchmark/evidence.ts index eb52b04cd6..778b4203fe 100644 --- a/scripts/ios-snapshot-benchmark/evidence.ts +++ b/scripts/ios-snapshot-benchmark/evidence.ts @@ -6,25 +6,63 @@ import { validateRawResult } from './schema.ts'; import type { BenchmarkResult } from './types.ts'; /** - * The evidence/ios-snapshot branch tip is mutable; this annotated tag and the full commit SHA - * below are the durable, immutable ref the corpus is pinned to. Re-tag (a new suffix, a new - * commit) if the corpus is ever re-measured — never move this tag. + * Each corpus is pinned by an annotated tag on the mutable `evidence/ios-snapshot` branch tip plus + * the full evidence commit below. Re-tag (a new suffix, a new commit) if a corpus is ever + * re-measured — never move these tags. */ -const EVIDENCE_TAG = 'refs/tags/evidence/ios-snapshot/71fb2483f'; -const EVIDENCE_COMMIT = '2d4baf461aa8897d49c6d4683cd16d8f43588ae8'; -export const DEFAULT_EVIDENCE_DIR = path.join(import.meta.dirname, 'evidence'); -export const EVIDENCE_FIXTURE_PATH = path.join(import.meta.dirname, 'evidence-fixture.v1.json'); +export type PublishedCorpus = { + /** Revision the leg measured, as a full SHA. */ + revision: string; + /** Durable ref the corpus is fetched from. */ + tag: string; + /** Evidence-branch commit holding the files. */ + commit: string; + /** Corpus file name -> sha256. */ + files: Readonly>; +}; + +const BASELINE_CORPUS: PublishedCorpus = { + revision: '71fb2483f30d90e615e949601c836aeebbf450c5', + tag: 'refs/tags/evidence/ios-snapshot/71fb2483f', + commit: '2d4baf461aa8897d49c6d4683cd16d8f43588ae8', + files: { + 'ios-snapshot-cold-local-71fb2483f.json': + '532a83247bfbf8ee47039f80ac429f067c84679e92c781768c1044da1ae6e9bf', + 'ios-snapshot-warm-relaunch-local-71fb2483f.json': + '6d299e8baec69662dca2c1ad8f1348e4361d5afaa781080e9a6b9b3dac362cbf', + 'ios-snapshot-proxy-71fb2483f.json': + 'b11b7a07be9e4dcf003f3af66943682a6733c6f21f5f43d3d9e88b3fb37b51a7', + }, +}; -/** Raw results published on the evidence branch, keyed by file name; measured at 71fb2483f. */ -export const PUBLISHED_EVIDENCE: Readonly> = { - 'ios-snapshot-cold-local-71fb2483f.json': - '532a83247bfbf8ee47039f80ac429f067c84679e92c781768c1044da1ae6e9bf', - 'ios-snapshot-warm-relaunch-local-71fb2483f.json': - '6d299e8baec69662dca2c1ad8f1348e4361d5afaa781080e9a6b9b3dac362cbf', - 'ios-snapshot-proxy-71fb2483f.json': - 'b11b7a07be9e4dcf003f3af66943682a6733c6f21f5f43d3d9e88b3fb37b51a7', +const CONVERGENCE_FINAL_CORPUS: PublishedCorpus = { + revision: '7c434b575837e3291c51315bf9bb8b54c8ce7568', + tag: 'refs/tags/evidence/ios-snapshot/7c434b575', + commit: '96d4951c19fbe009ba19d192a9774668edcc3f56', + files: { + 'ios-snapshot-cold-local-7c434b575.json': + '4663897ee5104569ad54e2ac803c216b284280c1c70fd82a8b2cf7b675d8a8bd', + 'ios-snapshot-first-interaction-local-7c434b575.json': + '87c686336f5581e3f18111e160cf7b733cd726b41e79ed6d8e5b53e2ab40c3fb', + 'ios-snapshot-warm-relaunch-local-7c434b575.json': + 'd49df3c3c943178f016a2b958449b257c6d46a44c8ffdf8fab75c1634ae1ebce', + 'ios-snapshot-proxy-7c434b575.json': + '5b5353831851f3a0f60e19d6bfd0cf50db47c3c4db52a283024c10bb17e71573', + }, }; +export const PUBLISHED_CORPORA: readonly PublishedCorpus[] = [ + BASELINE_CORPUS, + CONVERGENCE_FINAL_CORPUS, +]; + +/** Raw results published on the evidence branch, keyed by file name across every corpus. */ +export const PUBLISHED_EVIDENCE: Readonly> = Object.fromEntries( + PUBLISHED_CORPORA.flatMap((corpus) => Object.entries(corpus.files)), +); +export const DEFAULT_EVIDENCE_DIR = path.join(import.meta.dirname, 'evidence'); +export const EVIDENCE_FIXTURE_PATH = path.join(import.meta.dirname, 'evidence-fixture.v1.json'); + export type EvidenceFile = { file: string; sha256: string; @@ -35,9 +73,31 @@ export type EvidenceFile = { errors: string[]; }; -export function fetchEvidenceCommand(file = ''): string { +function corpusFetchCommand(corpus: PublishedCorpus, file: string): string { const destination = path.posix.join('scripts/ios-snapshot-benchmark/evidence', file); - return `git fetch origin ${EVIDENCE_TAG} && git show ${EVIDENCE_COMMIT}:${file} > ${destination}`; + return `git show ${corpus.commit}:${file} > ${destination}`; +} + +/** + * The fetch recipe for one corpus file. An unknown name falls back to the newest corpus, which is + * what a generic hint should recommend. + */ +export function fetchEvidenceCommand(file = ''): string { + const corpus = + PUBLISHED_CORPORA.find((candidate) => file in candidate.files) ?? PUBLISHED_CORPORA.at(-1)!; + return `git fetch origin ${corpus.tag} && ${corpusFetchCommand(corpus, file)}`; +} + +/** One shell line per corpus that holds any of `files`, so a hint never names the wrong commit. */ +function fetchEvidenceCommands(files: readonly string[]): string { + const lines: string[] = []; + for (const corpus of PUBLISHED_CORPORA) { + const missing = files.filter((file) => file in corpus.files); + if (missing.length === 0) continue; + const fetches = missing.map((file) => corpusFetchCommand(corpus, file)); + lines.push(`git fetch origin ${corpus.tag} && ${fetches.join(' && ')}`); + } + return lines.join('\n'); } export function listEvidenceFiles(dir: string): string[] { @@ -133,8 +193,8 @@ export function checkEvidenceCorpus(dir: string, files: EvidenceFile[], isDefaul const missing = missingPublishedEvidence(files); if (missing.length > 0) { throw new Error( - `${dir} is missing published evidence file(s): ${missing.join(', ')}. Fetch them with: ` + - `${fetchEvidenceCommand()}`, + `${dir} is missing published evidence file(s): ${missing.join(', ')}.\n` + + `Fetch them with:\n${fetchEvidenceCommands(missing)}`, ); } } @@ -144,7 +204,10 @@ function runEvidenceReport(argv: string[]): void { const { dir, isDefault } = readEvidenceDirOption(argv); const files = readEvidenceDir(dir); if (files.length === 0) { - throw new Error(`${dir} holds no evidence; fetch it with: ${fetchEvidenceCommand()}`); + throw new Error( + `${dir} holds no evidence; fetch it with:\n` + + `${fetchEvidenceCommands(Object.keys(PUBLISHED_EVIDENCE))}`, + ); } process.stdout.write(renderEvidenceReport(dir, files)); checkEvidenceCorpus(dir, files, isDefault); diff --git a/scripts/ios-snapshot-benchmark/evidence/README.md b/scripts/ios-snapshot-benchmark/evidence/README.md index b91bfaae2e..2945b81618 100644 --- a/scripts/ios-snapshot-benchmark/evidence/README.md +++ b/scripts/ios-snapshot-benchmark/evidence/README.md @@ -1,12 +1,16 @@ # iOS snapshot benchmark evidence -The raw `pnpm bench:ios-snapshot` results measured at commit -`71fb2483f30d90e615e949601c836aeebbf450c5` on `bench-golden-v2` (iPhone 17 Pro, iOS 27.0) live on -the orphan branch `evidence/ios-snapshot`, not in this tree. The branch tip is mutable; the durable -ref is the annotated tag `evidence/ios-snapshot/71fb2483f`, pinned to commit -`2d4baf461aa8897d49c6d4683cd16d8f43588ae8` — fetch and read from the tag and that full SHA, never -from the branch tip. Only Markdown summaries are kept here. The published hashes are declared in -[`../evidence.ts`](../evidence.ts) as `PUBLISHED_EVIDENCE`: +The raw `pnpm bench:ios-snapshot` results live on the orphan branch `evidence/ios-snapshot`, not in +this tree. The branch tip is mutable; the durable refs are the annotated tags and full commits +below — fetch from those, never from the branch tip. Only Markdown summaries are kept here. The +published hashes are declared in [`../evidence.ts`](../evidence.ts) as `PUBLISHED_CORPORA`, and +[`../evidence.ts`](../evidence.ts) derives `PUBLISHED_EVIDENCE` from them. + +## Baseline corpus — `71fb2483f` + +Measured at commit `71fb2483f30d90e615e949601c836aeebbf450c5` on `bench-golden-v2` (iPhone 17 Pro, +iOS 27.0). Tag `refs/tags/evidence/ios-snapshot/71fb2483f`, commit +`2d4baf461aa8897d49c6d4683cd16d8f43588ae8`. | File | sha256 | | --- | --- | @@ -14,12 +18,28 @@ from the branch tip. Only Markdown summaries are kept here. The published hashes | `ios-snapshot-warm-relaunch-local-71fb2483f.json` | `6d299e8baec69662dca2c1ad8f1348e4361d5afaa781080e9a6b9b3dac362cbf` | | `ios-snapshot-proxy-71fb2483f.json` | `b11b7a07be9e4dcf003f3af66943682a6733c6f21f5f43d3d9e88b3fb37b51a7` | +## Convergence final corpus — `7c434b575` + +Measured at commit `7c434b575837e3291c51315bf9bb8b54c8ce7568` on `bench-2188-final` (iPhone 17 Pro, +iOS 26.2). Tag `refs/tags/evidence/ios-snapshot/7c434b575`, commit +`96d4951c19fbe009ba19d192a9774668edcc3f56`. This is the corpus that closed the evidence sweep +gating #2188 on #2199; [`../../../docs/evidence/ios-snapshot-convergence-final-2026-09-21.md`](../../../docs/evidence/ios-snapshot-convergence-final-2026-09-21.md) +reads it against the baseline, including the target, runtime, host-load, app, and harness +deviations between the two corpora. + +| File | sha256 | +| --- | --- | +| `ios-snapshot-cold-local-7c434b575.json` | `4663897ee5104569ad54e2ac803c216b284280c1c70fd82a8b2cf7b675d8a8bd` | +| `ios-snapshot-first-interaction-local-7c434b575.json` | `87c686336f5581e3f18111e160cf7b733cd726b41e79ed6d8e5b53e2ab40c3fb` | +| `ios-snapshot-warm-relaunch-local-7c434b575.json` | `d49df3c3c943178f016a2b958449b257c6d46a44c8ffdf8fab75c1634ae1ebce` | +| `ios-snapshot-proxy-7c434b575.json` | `5b5353831851f3a0f60e19d6bfd0cf50db47c3c4db52a283024c10bb17e71573` | + ## Fetch One file, from the repository root: ```sh -git fetch origin refs/tags/evidence/ios-snapshot/71fb2483f && git show 2d4baf461aa8897d49c6d4683cd16d8f43588ae8: > scripts/ios-snapshot-benchmark/evidence/ +git fetch origin refs/tags/evidence/ios-snapshot/7c434b575 && git show 96d4951c19fbe009ba19d192a9774668edcc3f56: > scripts/ios-snapshot-benchmark/evidence/ ``` The whole corpus, then a schema and hash check: @@ -31,6 +51,13 @@ for f in ios-snapshot-cold-local-71fb2483f.json \ ios-snapshot-proxy-71fb2483f.json; do git show 2d4baf461aa8897d49c6d4683cd16d8f43588ae8:$f > scripts/ios-snapshot-benchmark/evidence/$f done +git fetch origin refs/tags/evidence/ios-snapshot/7c434b575 +for f in ios-snapshot-cold-local-7c434b575.json \ + ios-snapshot-first-interaction-local-7c434b575.json \ + ios-snapshot-warm-relaunch-local-7c434b575.json \ + ios-snapshot-proxy-7c434b575.json; do + git show 96d4951c19fbe009ba19d192a9774668edcc3f56:$f > scripts/ios-snapshot-benchmark/evidence/$f +done pnpm bench:ios-snapshot:evidence ``` diff --git a/scripts/ios-snapshot-benchmark/evidence/ios-snapshot-cold-local-7c434b575.md b/scripts/ios-snapshot-benchmark/evidence/ios-snapshot-cold-local-7c434b575.md new file mode 100644 index 0000000000..76fc7a93cf --- /dev/null +++ b/scripts/ios-snapshot-benchmark/evidence/ios-snapshot-cold-local-7c434b575.md @@ -0,0 +1,34 @@ +# iOS snapshot convergence benchmark + +- Status: **completed** +- Revision: 7c434b575837e3291c51315bf9bb8b54c8ce7568 +- Host: MacBook Pro (Mac16,8; Apple M4 Pro, 12 cores) +- Target: bench-2188-final (5FEADD02-98E4-4F01-861C-07003C1A3291, com.apple.CoreSimulator.SimRuntime.iOS-26-2) +- Generated: 2026-09-21T20:51:31.618Z + +| State | Screen | Transport | Execution | N | Wall median | Wall p95 | Daemon median | Response median | Failures | +|---|---|---|---|---:|---:|---:|---:|---:|---:| +| cold-cold | quiet | local | fresh-process-cli | 10 | 7159.1 | 9060.7 | 5709.0 | 4365.0 | 0 | +| cold-cold | list | local | fresh-process-cli | 10 | 7580.2 | 9145.6 | 5881.0 | 18206.0 | 0 | +| cold-cold | nested-scroll | local | fresh-process-cli | 10 | 8091.3 | 9339.9 | 6374.0 | 13423.0 | 0 | +| cold-cold | alert | local | fresh-process-cli | 10 | 7460.8 | 10034.7 | 5689.0 | 16035.0 | 0 | +| cold-cold | system-surface | local | fresh-process-cli | 10 | 6553.0 | 9585.4 | 4961.0 | 11342.0 | 0 | +| cold-cold | xctest-stress | local | fresh-process-cli | 10 | 7273.4 | 9747.5 | 5761.0 | 16106.0 | 0 | +| cold | quiet | local | fresh-process-cli | 10 | 5743.0 | 6533.9 | 4409.0 | 4337.0 | 0 | +| cold | list | local | fresh-process-cli | 10 | 5847.5 | 6055.3 | 4420.0 | 17277.0 | 0 | +| cold | nested-scroll | local | fresh-process-cli | 10 | 6057.6 | 6525.3 | 4619.0 | 12214.0 | 0 | +| cold | alert | local | fresh-process-cli | 10 | 5706.1 | 5874.5 | 4310.0 | 15105.0 | 0 | +| cold | system-surface | local | fresh-process-cli | 10 | 5028.5 | 5436.4 | 3662.0 | 12586.0 | 0 | +| cold | xctest-stress | local | fresh-process-cli | 10 | 5679.0 | 5870.3 | 4323.0 | 14364.0 | 0 | + +## Package size + +Not measured. + +## Deep-button control + +- Fixture artifact: deep-button-fixture.v1.json (depth 72) +- Red control: pnpm bench:ios-snapshot:deep-button -- --rule invalid-shallow (exit 1) + - AssertionError: changed descendant was omitted by shallow observation; no-effect claim is invalid. +- Safe control: pnpm bench:ios-snapshot:deep-button -- --rule safe-full (exit 0) + - full observation changed and includes the changed descendant. diff --git a/scripts/ios-snapshot-benchmark/evidence/ios-snapshot-first-interaction-local-7c434b575.md b/scripts/ios-snapshot-benchmark/evidence/ios-snapshot-first-interaction-local-7c434b575.md new file mode 100644 index 0000000000..3bf86628d2 --- /dev/null +++ b/scripts/ios-snapshot-benchmark/evidence/ios-snapshot-first-interaction-local-7c434b575.md @@ -0,0 +1,28 @@ +# iOS snapshot convergence benchmark + +- Status: **completed** +- Revision: 7c434b575837e3291c51315bf9bb8b54c8ce7568 +- Host: MacBook Pro (Mac16,8; Apple M4 Pro, 12 cores) +- Target: bench-2188-final (5FEADD02-98E4-4F01-861C-07003C1A3291, com.apple.CoreSimulator.SimRuntime.iOS-26-2) +- Generated: 2026-09-21T19:20:51.180Z + +| State | Screen | Transport | Execution | N | Wall median | Wall p95 | Daemon median | Response median | Failures | +|---|---|---|---|---:|---:|---:|---:|---:|---:| +| first-interaction | quiet | local | fresh-process-cli | 10 | 928.2 | 1189.3 | – | 567.0 | 0 | +| first-interaction | list | local | fresh-process-cli | 10 | 1435.7 | 1707.0 | – | 615.0 | 0 | +| first-interaction | nested-scroll | local | fresh-process-cli | 10 | 741.1 | 1241.7 | – | 579.0 | 0 | +| first-interaction | alert | local | fresh-process-cli | 10 | 1005.4 | 1018.1 | – | 546.0 | 0 | +| first-interaction | system-surface | local | fresh-process-cli | 10 | 1119.4 | 2051.8 | – | 548.0 | 0 | +| first-interaction | xctest-stress | local | fresh-process-cli | 10 | 1302.4 | 1341.0 | – | 539.0 | 0 | + +## Package size + +Not measured. + +## Deep-button control + +- Fixture artifact: deep-button-fixture.v1.json (depth 72) +- Red control: pnpm bench:ios-snapshot:deep-button -- --rule invalid-shallow (exit 1) + - AssertionError: changed descendant was omitted by shallow observation; no-effect claim is invalid. +- Safe control: pnpm bench:ios-snapshot:deep-button -- --rule safe-full (exit 0) + - full observation changed and includes the changed descendant. diff --git a/scripts/ios-snapshot-benchmark/evidence/ios-snapshot-proxy-7c434b575.md b/scripts/ios-snapshot-benchmark/evidence/ios-snapshot-proxy-7c434b575.md new file mode 100644 index 0000000000..3a827fb98b --- /dev/null +++ b/scripts/ios-snapshot-benchmark/evidence/ios-snapshot-proxy-7c434b575.md @@ -0,0 +1,58 @@ +# iOS snapshot convergence benchmark + +- Status: **completed** +- Revision: 7c434b575837e3291c51315bf9bb8b54c8ce7568 +- Host: MacBook Pro (Mac16,8; Apple M4 Pro, 12 cores) +- Target: bench-2188-final (5FEADD02-98E4-4F01-861C-07003C1A3291, com.apple.CoreSimulator.SimRuntime.iOS-26-2) +- Generated: 2026-09-21T20:02:38.671Z + +| State | Screen | Transport | Execution | N | Wall median | Wall p95 | Daemon median | Response median | Failures | +|---|---|---|---|---:|---:|---:|---:|---:|---:| +| warm | quiet | proxy | persistent-client | 20 | 102.9 | 109.9 | 94.0 | 1843.0 | 0 | +| warm | quiet | proxy | fresh-process-cli | 20 | 183.8 | 196.2 | 93.0 | 1841.0 | 0 | +| warm | list | proxy | persistent-client | 20 | 214.4 | 230.7 | 205.0 | 8869.0 | 0 | +| warm | list | proxy | fresh-process-cli | 20 | 304.5 | 315.8 | 208.0 | 8869.0 | 0 | +| warm | nested-scroll | proxy | persistent-client | 20 | 179.4 | 283.4 | 171.0 | 5784.0 | 0 | +| warm | nested-scroll | proxy | fresh-process-cli | 20 | 260.2 | 299.2 | 168.0 | 5787.0 | 0 | +| warm | alert | proxy | persistent-client | 20 | 148.4 | 154.1 | 140.0 | 7961.0 | 0 | +| warm | alert | proxy | fresh-process-cli | 20 | 240.7 | 251.6 | 147.0 | 7961.0 | 0 | +| warm | system-surface | proxy | persistent-client | 20 | 127.9 | 159.7 | 119.0 | 5758.0 | 0 | +| warm | system-surface | proxy | fresh-process-cli | 20 | 217.3 | 246.0 | 125.0 | 5758.0 | 0 | +| warm | xctest-stress | proxy | persistent-client | 20 | 135.8 | 149.7 | 128.0 | 7252.0 | 0 | +| warm | xctest-stress | proxy | fresh-process-cli | 20 | 228.6 | 235.2 | 139.0 | 7252.0 | 0 | +| warm | quiet | proxy | persistent-client | 20 | 157.6 | 164.2 | 101.0 | 1842.0 | 0 | +| warm | quiet | proxy | fresh-process-cli | 20 | 293.1 | 300.6 | 105.0 | 1844.0 | 0 | +| warm | list | proxy | persistent-client | 20 | 280.4 | 292.0 | 222.0 | 8869.0 | 0 | +| warm | list | proxy | fresh-process-cli | 20 | 415.5 | 429.1 | 221.0 | 8869.0 | 0 | +| warm | nested-scroll | proxy | persistent-client | 20 | 239.9 | 250.6 | 183.0 | 5785.0 | 0 | +| warm | nested-scroll | proxy | fresh-process-cli | 20 | 376.8 | 392.2 | 182.0 | 5785.0 | 0 | +| warm | alert | proxy | persistent-client | 20 | 203.2 | 221.5 | 148.0 | 7961.0 | 0 | +| warm | alert | proxy | fresh-process-cli | 20 | 343.4 | 356.7 | 150.0 | 7961.0 | 0 | +| warm | system-surface | proxy | persistent-client | 20 | 183.3 | 200.8 | 127.0 | 5758.0 | 0 | +| warm | system-surface | proxy | fresh-process-cli | 20 | 319.3 | 336.4 | 133.0 | 5758.0 | 0 | +| warm | xctest-stress | proxy | persistent-client | 20 | 197.3 | 208.7 | 140.0 | 7252.0 | 0 | +| warm | xctest-stress | proxy | fresh-process-cli | 20 | 339.6 | 353.2 | 148.0 | 7252.0 | 0 | +| warm | quiet | proxy | persistent-client | 20 | 281.9 | 303.0 | 104.0 | 1842.0 | 0 | +| warm | quiet | proxy | fresh-process-cli | 20 | 555.6 | 569.7 | 116.0 | 1840.0 | 0 | +| warm | list | proxy | persistent-client | 20 | 407.8 | 437.1 | 227.0 | 8838.0 | 0 | +| warm | list | proxy | fresh-process-cli | 20 | 691.4 | 708.9 | 236.0 | 8838.0 | 0 | +| warm | nested-scroll | proxy | persistent-client | 20 | 380.8 | 432.7 | 199.0 | 5785.0 | 0 | +| warm | nested-scroll | proxy | fresh-process-cli | 20 | 642.7 | 667.0 | 195.0 | 5784.0 | 0 | +| warm | alert | proxy | persistent-client | 20 | 350.9 | 361.8 | 169.0 | 7961.0 | 0 | +| warm | alert | proxy | fresh-process-cli | 20 | 615.3 | 626.3 | 170.0 | 7961.0 | 0 | +| warm | system-surface | proxy | persistent-client | 20 | 329.8 | 346.5 | 150.0 | 5758.0 | 0 | +| warm | system-surface | proxy | fresh-process-cli | 20 | 595.2 | 611.2 | 152.0 | 5758.0 | 0 | +| warm | xctest-stress | proxy | persistent-client | 20 | 314.0 | 344.5 | 134.0 | 7252.0 | 0 | +| warm | xctest-stress | proxy | fresh-process-cli | 20 | 562.4 | 586.0 | 128.0 | 7252.0 | 0 | + +## Package size + +Not measured. + +## Deep-button control + +- Fixture artifact: deep-button-fixture.v1.json (depth 72) +- Red control: pnpm bench:ios-snapshot:deep-button -- --rule invalid-shallow (exit 1) + - AssertionError: changed descendant was omitted by shallow observation; no-effect claim is invalid. +- Safe control: pnpm bench:ios-snapshot:deep-button -- --rule safe-full (exit 0) + - full observation changed and includes the changed descendant. diff --git a/scripts/ios-snapshot-benchmark/evidence/ios-snapshot-warm-relaunch-local-7c434b575.md b/scripts/ios-snapshot-benchmark/evidence/ios-snapshot-warm-relaunch-local-7c434b575.md new file mode 100644 index 0000000000..29ef025a6f --- /dev/null +++ b/scripts/ios-snapshot-benchmark/evidence/ios-snapshot-warm-relaunch-local-7c434b575.md @@ -0,0 +1,37 @@ +# iOS snapshot convergence benchmark + +- Status: **completed** +- Revision: 7c434b575837e3291c51315bf9bb8b54c8ce7568 +- Host: MacBook Pro (Mac16,8; Apple M4 Pro, 12 cores) +- Target: bench-2188-final (5FEADD02-98E4-4F01-861C-07003C1A3291, com.apple.CoreSimulator.SimRuntime.iOS-26-2) +- Generated: 2026-09-21T19:53:30.433Z + +| State | Screen | Transport | Execution | N | Wall median | Wall p95 | Daemon median | Response median | Failures | +|---|---|---|---|---:|---:|---:|---:|---:|---:| +| warm | quiet | local | fresh-process-cli | 20 | 169.4 | 183.3 | 89.0 | 3448.0 | 0 | +| warm | list | local | fresh-process-cli | 20 | 291.2 | 309.3 | 202.0 | 18354.0 | 0 | +| warm | nested-scroll | local | fresh-process-cli | 20 | 249.8 | 268.2 | 167.0 | 12593.0 | 0 | +| warm | alert | local | fresh-process-cli | 20 | 224.0 | 238.6 | 139.0 | 16256.0 | 0 | +| warm | system-surface | local | fresh-process-cli | 20 | 207.7 | 225.2 | 130.0 | 11566.0 | 0 | +| warm | xctest-stress | local | fresh-process-cli | 20 | 213.1 | 233.1 | 128.0 | 14955.0 | 0 | +| relaunch | quiet | local | fresh-process-cli | 20 | 3876.5 | 3981.8 | 2851.0 | 2757.0 | 0 | +| relaunch | list | local | fresh-process-cli | 20 | 3888.4 | 3970.5 | 2871.0 | 2754.0 | 0 | +| relaunch | nested-scroll | local | fresh-process-cli | 20 | 3871.9 | 3958.4 | 2844.0 | 2800.0 | 0 | +| relaunch | alert | local | fresh-process-cli | 20 | 3896.4 | 4058.7 | 2881.0 | 2763.0 | 0 | +| relaunch | system-surface | local | fresh-process-cli | 20 | 3576.4 | 3640.7 | 2536.0 | 11302.0 | 0 | +| relaunch | xctest-stress | local | fresh-process-cli | 20 | 3867.8 | 4071.6 | 2855.0 | 2796.0 | 0 | + +## Package size + +- Packed tarball: 1409306 bytes +- Packed unpacked tree: 4723895 bytes +- Clean-installed package tree: 4723895 bytes (543 files) +- Bundled JavaScript: 3775024 raw / 1258950 gzip bytes + +## Deep-button control + +- Fixture artifact: deep-button-fixture.v1.json (depth 72) +- Red control: pnpm bench:ios-snapshot:deep-button -- --rule invalid-shallow (exit 1) + - AssertionError: changed descendant was omitted by shallow observation; no-effect claim is invalid. +- Safe control: pnpm bench:ios-snapshot:deep-button -- --rule safe-full (exit 0) + - full observation changed and includes the changed descendant. diff --git a/scripts/ios-snapshot-benchmark/fixture-admission.test.ts b/scripts/ios-snapshot-benchmark/fixture-admission.test.ts index cc328e75ff..f107a812ee 100644 --- a/scripts/ios-snapshot-benchmark/fixture-admission.test.ts +++ b/scripts/ios-snapshot-benchmark/fixture-admission.test.ts @@ -4,8 +4,8 @@ import { BenchmarkCellAdmissionError } from './lifecycle.ts'; import { prepareFixture, requireFixtureAnchor, + type FixtureOperationResult, type FixturePreparationDriver, - type FixturePreparationResult, } from './fixture-admission.ts'; import type { ScreenFixture } from './types.ts'; @@ -18,7 +18,9 @@ const alertFixture: ScreenFixture = { setupAction: 'open-alert', }; -function observed(anchor: string): FixturePreparationResult { +const shortAnchorOptions = { anchorBudgetMs: 40, anchorPollMs: 5 }; + +function observed(anchor: string): FixtureOperationResult { return { ok: true, payload: { @@ -61,10 +63,79 @@ test('shares opening, setup, and post-setup admission across fixture drivers', a test('turns a wrong post-setup screen into a typed fixture-anchor stop', async () => { await assert.rejects( - () => prepareFixture(alertFixture, successfulDriver(['Automation lab', 'Settings'])), + () => + prepareFixture( + alertFixture, + successfulDriver(['Automation lab', 'Settings']), + shortAnchorOptions, + ), (error: unknown) => - error instanceof BenchmarkCellAdmissionError && error.reason === 'fixture-anchor', + error instanceof BenchmarkCellAdmissionError && + error.reason === 'fixture-anchor' && + error.message.includes('within 40ms'), + ); +}); + +test('waits an unmounted opening screen out, then continues setup', async () => { + const calls: string[] = []; + const anchors = ['Settings', 'Settings', 'Automation lab', 'Automation confirmation']; + const driver: FixturePreparationDriver = { + observe: () => { + calls.push('observe'); + return observed(anchors.shift() ?? 'Unexpected extra observation'); + }, + scrollToBottom: () => { + calls.push('scroll'); + return { ok: true, payload: {} }; + }, + openAlert: () => { + calls.push('open-alert'); + return { ok: true, payload: {} }; + }, + }; + + await prepareFixture(alertFixture, driver, shortAnchorOptions); + + assert.deepEqual(calls, ['observe', 'observe', 'observe', 'scroll', 'open-alert', 'observe']); + assert.deepEqual(anchors, []); +}); + +test('stops at the opening anchor once the admission budget elapses', async () => { + const unmounted: FixturePreparationDriver = { + observe: () => observed('Settings'), + scrollToBottom: () => ({ ok: true, payload: {} }), + openAlert: () => ({ ok: true, payload: {} }), + }; + const started = Date.now(); + await assert.rejects( + () => prepareFixture(alertFixture, unmounted, shortAnchorOptions), + (error: unknown) => + error instanceof BenchmarkCellAdmissionError && + error.reason === 'fixture-anchor' && + error.message.includes('within 40ms'), + ); + assert.ok(Date.now() - started < 5_000, 'the anchor budget was not honored'); +}); + +test('fails an unsuccessful observation immediately instead of polling', async () => { + let observations = 0; + const driver: FixturePreparationDriver = { + observe: () => { + observations += 1; + return { ok: false, payload: {} }; + }, + scrollToBottom: () => ({ ok: true, payload: {} }), + openAlert: () => ({ ok: true, payload: {} }), + }; + + await assert.rejects( + () => prepareFixture(alertFixture, driver, { anchorBudgetMs: 5_000, anchorPollMs: 1 }), + (error: unknown) => + error instanceof BenchmarkCellAdmissionError && + error.reason === 'fixture-anchor' && + error.message.includes('failed'), ); + assert.equal(observations, 1); }); test('checks the expected post-setup anchor for direct client batch results', () => { diff --git a/scripts/ios-snapshot-benchmark/fixture-admission.ts b/scripts/ios-snapshot-benchmark/fixture-admission.ts index 673a37ab3f..c9df0549f3 100644 --- a/scripts/ios-snapshot-benchmark/fixture-admission.ts +++ b/scripts/ios-snapshot-benchmark/fixture-admission.ts @@ -5,6 +5,14 @@ import type { Failure, ScreenFixture } from './types.ts'; export type FixtureAnchorPhase = 'opened' | 'prepared' | 'sample'; +/** + * Setup reads are untimed, so an anchor read that lands before the app has mounted its first tree + * is waited out instead of stopping the run. The budget must stay far below the operation timeout + * it precedes; a fixture that never exposes its anchor still stops the run. + */ +const FIXTURE_ANCHOR_ADMISSION_BUDGET_MS = 30_000; +const FIXTURE_ANCHOR_POLL_INTERVAL_MS = 500; + export type FixtureOperationResult = { ok: boolean; payload: unknown; @@ -19,30 +27,62 @@ export type FixturePreparationDriver = { openAlert: () => FixtureOperationResult | Promise; }; +export type FixturePreparationOptions = { + anchorBudgetMs?: number; + anchorPollMs?: number; +}; + export async function prepareFixture( fixture: ScreenFixture, driver: FixturePreparationDriver, + options: FixturePreparationOptions = {}, ): Promise { - const opened = await driver.observe(); - requireFixtureOperationSuccess( - opened, - `${fixture.id} semantic anchor observation`, - 'fixture-anchor', - ); - requireFixtureAnchor(opened.payload, fixture, 'opened', opened.command); + await observeFixtureAnchor(fixture, driver, 'opened', options); if (fixture.setupAction !== 'open-alert') return; const scrolled = await driver.scrollToBottom(); requireFixtureOperationSuccess(scrolled, `${fixture.id} setup scroll`, 'cell-state'); const alert = await driver.openAlert(); requireFixtureOperationSuccess(alert, `${fixture.id} setup action`, 'cell-state'); - const prepared = await driver.observe(); - requireFixtureOperationSuccess( - prepared, - `${fixture.id} post-setup semantic anchor observation`, - 'fixture-anchor', - ); - requireFixtureAnchor(prepared.payload, fixture, 'prepared', prepared.command); + await observeFixtureAnchor(fixture, driver, 'prepared', options); +} + +async function observeFixtureAnchor( + fixture: ScreenFixture, + driver: FixturePreparationDriver, + phase: Exclude, + options: FixturePreparationOptions, +): Promise { + const budgetMs = options.anchorBudgetMs ?? FIXTURE_ANCHOR_ADMISSION_BUDGET_MS; + const pollMs = options.anchorPollMs ?? FIXTURE_ANCHOR_POLL_INTERVAL_MS; + const operation = + phase === 'opened' + ? `${fixture.id} semantic anchor observation` + : `${fixture.id} post-setup semantic anchor observation`; + const deadline = Date.now() + budgetMs; + for (;;) { + const result = await driver.observe(); + requireFixtureOperationSuccess(result, operation, 'fixture-anchor'); + if (hasFixtureAnchor(result.payload, fixture, phase)) return result; + const remainingMs = deadline - Date.now(); + if (remainingMs <= 0) { + throw new BenchmarkCellAdmissionError( + 'fixture-anchor', + `Fixture ${fixture.id} ${phase} did not expose the exact anchor ` + + `${JSON.stringify(expectedAnchor(fixture, phase))} within ${budgetMs}ms.`, + result.command, + ); + } + await sleep(Math.min(pollMs, remainingMs)); + } +} + +function hasFixtureAnchor( + payload: unknown, + fixture: ScreenFixture, + phase: FixtureAnchorPhase, +): boolean { + return snapshotHasAnchor(payload, expectedAnchor(fixture, phase)); } export function requireFixtureAnchor( @@ -51,15 +91,20 @@ export function requireFixtureAnchor( phase: FixtureAnchorPhase, command = 'agent-device snapshot', ): void { - const anchor = expectedAnchor(fixture, phase); - if (snapshotHasAnchor(payload, anchor)) return; + if (hasFixtureAnchor(payload, fixture, phase)) return; throw new BenchmarkCellAdmissionError( 'fixture-anchor', - `Fixture ${fixture.id} ${phase} did not expose the exact anchor ${JSON.stringify(anchor)}.`, + `Fixture ${fixture.id} ${phase} did not expose the exact anchor ${JSON.stringify( + expectedAnchor(fixture, phase), + )}.`, command, ); } +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + export function fixtureOperationFromCli( result: CliResult, command: string, diff --git a/scripts/ios-snapshot-benchmark/proxy-client-support.test.ts b/scripts/ios-snapshot-benchmark/proxy-client-support.test.ts index 825b81a2f3..b84c87fa6e 100644 --- a/scripts/ios-snapshot-benchmark/proxy-client-support.test.ts +++ b/scripts/ios-snapshot-benchmark/proxy-client-support.test.ts @@ -75,8 +75,20 @@ test('persistent-client setup rejects a wrong prepared screen as fixture-anchor' clientWithSnapshots(['Automation lab', 'Settings'], []), alertFixture, 'simulator', + { anchorBudgetMs: 40, anchorPollMs: 5 }, ), (error: unknown) => error instanceof BenchmarkCellAdmissionError && error.reason === 'fixture-anchor', ); }); + +test('persistent-client setup waits an unmounted opening screen out', async () => { + const calls: string[] = []; + await openClientFixture( + clientWithSnapshots(['Settings', 'Automation lab', 'Automation confirmation'], calls), + alertFixture, + 'simulator', + { anchorBudgetMs: 2_000, anchorPollMs: 1 }, + ); + assert.deepEqual(calls, ['open', 'snapshot', 'snapshot', 'scroll', 'click', 'snapshot']); +}); diff --git a/scripts/ios-snapshot-benchmark/proxy-client-support.ts b/scripts/ios-snapshot-benchmark/proxy-client-support.ts index 6d8c1b9de2..4c26ea85b2 100644 --- a/scripts/ios-snapshot-benchmark/proxy-client-support.ts +++ b/scripts/ios-snapshot-benchmark/proxy-client-support.ts @@ -19,6 +19,7 @@ import { fixtureOperationFromClient, prepareFixture, requireFixtureAnchor, + type FixturePreparationOptions, } from './fixture-admission.ts'; import { type NetworkConditioner, type ProxyRpcRecord } from './proxy-conditioner.ts'; import type { ProxyStartup } from './proxy-process.ts'; @@ -108,6 +109,7 @@ export async function openClientFixture( client: AgentClient, fixture: ScreenFixture, udid: string, + preparation: FixturePreparationOptions = {}, ): Promise { await client.apps.open({ app: fixture.app, @@ -117,31 +119,35 @@ export async function openClientFixture( relaunch: true, foreground: true, }); - await prepareFixture(fixture, { - observe: async () => - fixtureOperationFromClient( - await client.batch.run(snapshotBatchOptions()), - 'agent-device client batch --steps snapshot', - ), - scrollToBottom: async () => - fixtureOperationFromClient( - await client.interactions.scroll({ - direction: 'bottom', - platform: 'ios', - udid, - }), - 'agent-device client scroll bottom', - ), - openAlert: async () => - fixtureOperationFromClient( - await client.interactions.click({ - target: { kind: 'selector', selector: 'id="automation-open-alert"' }, - platform: 'ios', - udid, - }), - 'agent-device client click id="automation-open-alert"', - ), - }); + await prepareFixture( + fixture, + { + observe: async () => + fixtureOperationFromClient( + await client.batch.run(snapshotBatchOptions()), + 'agent-device client batch --steps snapshot', + ), + scrollToBottom: async () => + fixtureOperationFromClient( + await client.interactions.scroll({ + direction: 'bottom', + platform: 'ios', + udid, + }), + 'agent-device client scroll bottom', + ), + openAlert: async () => + fixtureOperationFromClient( + await client.interactions.click({ + target: { kind: 'selector', selector: 'id="automation-open-alert"' }, + platform: 'ios', + udid, + }), + 'agent-device client click id="automation-open-alert"', + ), + }, + preparation, + ); } export async function captureClientSample( From 9208b90491498a2156165e5133518015ce8bfaab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 22 Sep 2026 08:57:59 +0200 Subject: [PATCH 006/132] fix(fold): report screen dimensions in their native-panel coordinate space (#2737) The fold response's `screen.widthPt`/`heightPt` are the lit panel's native points (pixels divided by its point scale, never rotated), not the next snapshot's viewport. Add the required `screen.coordinateSpace: "native-panel"` discriminator at the construction site, export it from the owning contract so the Apple owner and the MCP schema share one value, and correct every doc and message that implied these numbers could place a tap. A fresh snapshot stays the only source of app-viewport coordinates: on an open Duo the 669x951 inner panel hosts a 951x669 app window. Closes #2729 --- CHANGELOG.md | 9 ++- packages/contracts/src/fold-runtime.ts | 13 +++- packages/contracts/src/navigation.ts | 5 +- .../platform-apple/src/foldable/pose.test.ts | 59 ++++++++++++++++++- packages/platform-apple/src/foldable/pose.ts | 8 ++- src/commands/schema/cli-help.ts | 4 +- src/daemon/__tests__/fold-runtime.test.ts | 11 +++- src/daemon/fold-runtime.ts | 2 +- .../__tests__/command-tools-parity.test.ts | 34 +++++++++++ src/mcp/command-output-schemas.ts | 12 +++- website/docs/docs/commands.md | 2 +- 11 files changed, 141 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d8783a9c6..5870d0eeee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,13 +41,18 @@ the pose control in the Xcode Device Hub window through macOS accessibility, and it is confirmed the way ADR 0025 asked for, by reading the hinge angle back from CoreDevice (`devicectl device motion hinge-angle`) until it agrees with the request. The response reports - the verified pose, the hinge angle, and the panel the device now lights with its point size, so an - agent can see that its refs and coordinates are stale without another capture. The macOS helper + the verified pose, the hinge angle, and the panel the device now lights with its native panel point + size, so an agent can see the lit panel changed without another capture. The macOS helper gained a `device-hub pose` subcommand that finds Device Hub in the process table (LaunchServices registers the trampolined app with no pid), reopens its window when it shows none, and selects the simulator through the sidebar row keyed by its UDID so two simulators sharing a name cannot be confused. Simulator-only; a single-panel simulator refuses with `UNSUPPORTED_OPERATION`, and every other platform states its own refusal cell. +- Fixed (ios): the `fold` response's `screen` dimensions now match their coordinate-space contract. They + are the lit panel's own native points (pixels divided by its point scale, never rotated), so the + response carries the `coordinateSpace: "native-panel"` discriminator and the docs no longer claim + they are the next snapshot's coordinates. They cannot place a tap; take a fresh snapshot for the app + viewport. - Added (diff): `diff screenshot` accepts a JPEG baseline or current image. Both inputs had to be PNG, so a capture exported by another tool had to be converted first and a HarmonyOS capture — which the platform serves as JPEG under whatever name the command was given — could never be compared. Each diff --git a/packages/contracts/src/fold-runtime.ts b/packages/contracts/src/fold-runtime.ts index 2aeab8c1ea..c20b3480e8 100644 --- a/packages/contracts/src/fold-runtime.ts +++ b/packages/contracts/src/fold-runtime.ts @@ -8,13 +8,22 @@ import type { RuntimeOperationFact } from './platform-runtime.ts'; */ export type SetFoldPoseInput = Readonly<{ pose: FoldPose }>; +/** Single source of truth for the discriminator the Apple owner sets and the MCP schema advertises. */ +export const FOLD_SCREEN_COORDINATE_SPACE = 'native-panel' as const; + /** - * The lit panel after the pose settled, in the points the next snapshot will use. Reported so an - * agent can see that the coordinate space changed without a second capture. + * The panel the device lights after the pose settled, in that panel's own native points: its pixel + * size divided by its point scale, never rotated. `coordinateSpace` is always + * {@link FOLD_SCREEN_COORDINATE_SPACE}, and these numbers are NOT snapshot coordinates — the active + * app window can differ from the panel (iPhone Duo: a 669x951 inner panel hosts a 951x669 app + * window), so they cannot place a tap. A caller that needs the app viewport must take a fresh + * snapshot. */ export type FoldScreenReport = Readonly<{ /** The CoreDevice display name of the panel the device now lights. */ display: string; + /** Marks these dimensions as the panel's native points, never a snapshot's app viewport. */ + coordinateSpace: typeof FOLD_SCREEN_COORDINATE_SPACE; widthPt: number; heightPt: number; }>; diff --git a/packages/contracts/src/navigation.ts b/packages/contracts/src/navigation.ts index 861efb3cce..3d5c6c1d62 100644 --- a/packages/contracts/src/navigation.ts +++ b/packages/contracts/src/navigation.ts @@ -53,8 +53,9 @@ export type OrientationCommandResult = { * * Unlike `orientation`, there is no unconfirmed variant: the Apple owner reads the hinge angle * back from CoreDevice after pressing the Device Hub pose control, and reports a pose only when - * that reading agrees with the request. `screen` names the panel the device lights afterwards, in - * points, because a pose change moves the app to a different coordinate space (ADR 0025). + * that reading agrees with the request. `screen` names the panel the device lights afterwards and + * that panel's native point size (ADR 0025); it is the panel's geometry, not the app viewport, so a + * caller must take a fresh snapshot before placing a tap. */ export type FoldCommandResult = { action: 'fold'; diff --git a/packages/platform-apple/src/foldable/pose.test.ts b/packages/platform-apple/src/foldable/pose.test.ts index d0121eb759..5a7b83413e 100644 --- a/packages/platform-apple/src/foldable/pose.test.ts +++ b/packages/platform-apple/src/foldable/pose.test.ts @@ -99,7 +99,7 @@ test('presses the Device Hub control for the pose and reports the pose CoreDevic await expect(setAppleFoldPose(duo, 'open')).resolves.toEqual({ pose: 'open', hingeAngleDegrees: 180, - screen: { display: 'LCD-1', widthPt: 669, heightPt: 951 }, + screen: { display: 'LCD-1', coordinateSpace: 'native-panel', widthPt: 669, heightPt: 951 }, }); expect(mockPress).toHaveBeenCalledWith({ @@ -111,6 +111,62 @@ test('presses the Device Hub control for the pose and reports the pose CoreDevic expect(mockHinge).toHaveBeenCalledTimes(2); }); +test('reports the closed outer panel in native points, not rotated to a snapshot viewport', async () => { + // The outer panel is 1398x2034 px at scale 3, i.e. 466x678 native points; the pose must not + // rotate that into the app window's shape, because a caller cannot place a tap from it. + mockInventory + .mockResolvedValueOnce(duoInventory('outer')) + .mockResolvedValueOnce(duoInventory('outer')); + mockHinge.mockResolvedValue(0); + + await expect(setAppleFoldPose(duo, 'closed')).resolves.toEqual({ + pose: 'closed', + hingeAngleDegrees: 0, + screen: { display: 'LCD', coordinateSpace: 'native-panel', widthPt: 466, heightPt: 678 }, + }); +}); + +test("reports native panel points regardless of the display's own currentOrientation", async () => { + // The same inner panel (2007x2853 px at scale 3) with a portrait orientation tag still measures + // 669x951: the report divides by point scale only and never swaps on `currentOrientation`. + const rot0Inner = buildInventory([ + panel({ power: 'dark' }), + panel({ + name: 'LCD-1', + displayId: 3, + primary: false, + power: 'lit', + widthPx: 2007, + heightPx: 2853, + currentOrientation: 'rot0', + }), + ]); + mockInventory.mockResolvedValueOnce(duoInventory('inner')).mockResolvedValueOnce(rot0Inner); + mockHinge.mockResolvedValue(180); + + const result = await setAppleFoldPose(duo, 'open'); + expect(result.screen).toEqual({ + display: 'LCD-1', + coordinateSpace: 'native-panel', + widthPt: 669, + heightPt: 951, + }); +}); + +test('omits the screen report when panel selection is ambiguous, never inventing a viewport', async () => { + // Two lit panels make the inventory ambiguous, so `readLitPanel` refuses to name one; the pose is + // still verified but carries no `screen` rather than guessing the app viewport. + const bothLit = buildInventory([panel({}), panel({ name: 'LCD-1', displayId: 3 })]); + expect(bothLit.ambiguous).toBe(true); + mockInventory.mockResolvedValueOnce(bothLit).mockResolvedValueOnce(bothLit); + mockHinge.mockResolvedValue(180); + + await expect(setAppleFoldPose(duo, 'open')).resolves.toEqual({ + pose: 'open', + hingeAngleDegrees: 180, + }); +}); + test('maps half-open onto the Book preset and reports it only once the hinge has stopped', async () => { mockInventory .mockResolvedValueOnce(duoInventory('inner')) @@ -122,6 +178,7 @@ test('maps half-open onto the Book preset and reports it only once the hinge has await expect(setAppleFoldPose(duo, 'half-open')).resolves.toMatchObject({ pose: 'half-open', hingeAngleDegrees: 130, + screen: { display: 'LCD-1', coordinateSpace: 'native-panel', widthPt: 669, heightPt: 951 }, }); expect(mockPress).toHaveBeenCalledWith(expect.objectContaining({ pose: 'book' })); expect(mockHinge).toHaveBeenCalledTimes(3); diff --git a/packages/platform-apple/src/foldable/pose.ts b/packages/platform-apple/src/foldable/pose.ts index 2da21bb9c5..9cab525785 100644 --- a/packages/platform-apple/src/foldable/pose.ts +++ b/packages/platform-apple/src/foldable/pose.ts @@ -1,5 +1,9 @@ import { foldPoseForHingeAngle, type FoldPose } from '@agent-device/contracts/device'; -import type { FoldScreenReport, SetFoldPoseResult } from '@agent-device/contracts/fold-runtime'; +import { + FOLD_SCREEN_COORDINATE_SPACE, + type FoldScreenReport, + type SetFoldPoseResult, +} from '@agent-device/contracts/fold-runtime'; import type { DeviceInfo } from '@agent-device/kernel/device'; import { AppError } from '@agent-device/kernel/errors'; import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; @@ -196,9 +200,11 @@ async function readLitPanel( return inventory.activeDisplay; } +/** The panel's native points: pixels divided by its own point scale, never rotated by orientation. */ function screenReport(display: AppleDeviceDisplay): FoldScreenReport { return { display: display.name, + coordinateSpace: FOLD_SCREEN_COORDINATE_SPACE, widthPt: Math.round(display.widthPx / display.pointScale), heightPt: Math.round(display.heightPx / display.pointScale), }; diff --git a/src/commands/schema/cli-help.ts b/src/commands/schema/cli-help.ts index f54c968afe..f49706c7ee 100644 --- a/src/commands/schema/cli-help.ts +++ b/src/commands/schema/cli-help.ts @@ -742,12 +742,12 @@ A foldable Apple device (iPhone Duo) carries two integrated panels, which Apple Screens are handled for you: Each iOS simulator capture resolves the CoreDevice display table, captures the lit panel explicitly, and normalizes density with that panel's own point scale. Do not add a screen flag to the normal loop; there is none, because the lit panel is always the only capturable one: the dark panel yields an all-black PNG. - The two panels are different sizes and different coordinate spaces (iPhone Duo: 466x678 points closed on the outer panel, 669x951 open on the inner). A pose change therefore invalidates every ref and coordinate. Re-snapshot after any pose change and never carry coordinates or refs across one. + The two panels are different sizes (iPhone Duo: 466x678 points closed on the outer panel, 669x951 open on the inner). A pose change therefore invalidates every ref and coordinate. Re-snapshot after any pose change and never carry coordinates or refs across one. Check which panel is lit before trusting a geometry claim: agent-device screenshot reports its point size, and 466x678 versus 669x951 says which panel you captured. Changing the pose: agent-device fold closed | half-open | open - fold presses the pose control in the Xcode Device Hub window for this simulator (Closed, Book, Open) and then reads the hinge angle back from CoreDevice until it agrees: closed is 0 degrees, open is 180, and half-open is any angle between them (Device Hub's Book preset, 130 degrees on iOS 27.1). An angle in that interval only proves the category, so half-open is reported once two consecutive readings both fall inside it and agree within 0.5 degrees. The response reports the verified pose, the hinge angle, and the panel the device now lights with its point size. A hinge whose last reading is some other pose fails with COMMAND_FAILED and reason fold-pose-unverified, naming the angle CoreDevice still reports; a hinge seen half-open but never at rest fails with reason fold-pose-unsettled, naming the observed and previous angles. A single-panel simulator fails with UNSUPPORTED_OPERATION. + fold presses the pose control in the Xcode Device Hub window for this simulator (Closed, Book, Open) and then reads the hinge angle back from CoreDevice until it agrees: closed is 0 degrees, open is 180, and half-open is any angle between them (Device Hub's Book preset, 130 degrees on iOS 27.1). An angle in that interval only proves the category, so half-open is reported once two consecutive readings both fall inside it and agree within 0.5 degrees. The response reports the verified pose, the hinge angle, and the panel the device now lights with its native panel point size, marked coordinateSpace "native-panel". That point size is the panel's own geometry, not the next snapshot's viewport, so it cannot place a tap: the active app window can differ (a 669x951 inner panel hosts a 951x669 window). A hinge whose last reading is some other pose fails with COMMAND_FAILED and reason fold-pose-unverified, naming the angle CoreDevice still reports; a hinge seen half-open but never at rest fails with reason fold-pose-unsettled, naming the observed and previous angles. A single-panel simulator fails with UNSUPPORTED_OPERATION. Expect a fold to take 10-16 seconds: each hinge read is a five-second devicectl stream, and half-open waits for the hinge to stop moving. Re-snapshot after every fold; refs and coordinates from before it are stale, and the command's message says so. Requirements: an iOS simulator session on a foldable device, Xcode 27.1 or newer with Device Hub, and Accessibility permission for the host (agent-device settings permission grant accessibility --platform macos). The command launches Device Hub if needed, reopens its window when it shows none, and selects the simulator through its sidebar by UDID, so no operator step is needed. No official host API sets the pose; the app under test still reads it as UIHinge.status. If a task asserts behavior for more than one pose, fold to each pose and re-snapshot, and report which poses the run covered.`, diff --git a/src/daemon/__tests__/fold-runtime.test.ts b/src/daemon/__tests__/fold-runtime.test.ts index a6ca846e63..3cbc78f7ed 100644 --- a/src/daemon/__tests__/fold-runtime.test.ts +++ b/src/daemon/__tests__/fold-runtime.test.ts @@ -88,7 +88,12 @@ test('resolves one admitted binding and reports the pose the owner read back', a const setFoldPose = vi.fn(async () => ({ pose: 'open' as const, hingeAngleDegrees: 180, - screen: { display: 'LCD-1', widthPt: 669, heightPt: 951 }, + screen: { + display: 'LCD-1', + coordinateSpace: 'native-panel' as const, + widthPt: 669, + heightPt: 951, + }, })); const harness = runtimeHarness( foldRuntimeOperationFacts({ fold: available }).setFoldPose, @@ -109,9 +114,9 @@ test('resolves one admitted binding and reports the pose the owner read back', a action: 'fold', pose: 'open', hingeAngleDegrees: 180, - screen: { display: 'LCD-1', widthPt: 669, heightPt: 951 }, + screen: { display: 'LCD-1', coordinateSpace: 'native-panel', widthPt: 669, heightPt: 951 }, message: - 'Folded to open (hinge 180°, LCD-1 lit at 669x951pt); refs from before the pose change are stale', + 'Folded to open (hinge 180°, LCD-1 native panel 669x951pt, not snapshot coordinates); refs from before the pose change are stale', }); expect(setFoldPose).toHaveBeenCalledWith({ pose: 'open' }); }); diff --git a/src/daemon/fold-runtime.ts b/src/daemon/fold-runtime.ts index 8a214a4a66..179ea15a73 100644 --- a/src/daemon/fold-runtime.ts +++ b/src/daemon/fold-runtime.ts @@ -55,7 +55,7 @@ async function executeSetFoldPose( ...(screen ? { screen } : {}), ...successText( screen - ? `Folded to ${result.pose} (hinge ${result.hingeAngleDegrees}°, ${screen.display} lit at ${screen.widthPt}x${screen.heightPt}pt); refs from before the pose change are stale` + ? `Folded to ${result.pose} (hinge ${result.hingeAngleDegrees}°, ${screen.display} native panel ${screen.widthPt}x${screen.heightPt}pt, not snapshot coordinates); refs from before the pose change are stale` : `Folded to ${result.pose} (hinge ${result.hingeAngleDegrees}°); refs from before the pose change are stale`, ), }; diff --git a/src/mcp/__tests__/command-tools-parity.test.ts b/src/mcp/__tests__/command-tools-parity.test.ts index b0c4a6f7f6..113ffcc363 100644 --- a/src/mcp/__tests__/command-tools-parity.test.ts +++ b/src/mcp/__tests__/command-tools-parity.test.ts @@ -166,6 +166,40 @@ test('MCP fill projects target-bound unconfirmed verification through its advert assert.notDeepEqual(validateAgainstSchema(missingTarget, fillTool.outputSchema), []); }); +test('MCP fold advertises the native-panel discriminator and rejects a screen missing it', async () => { + const foldResult = { + action: 'fold', + pose: 'open', + hingeAngleDegrees: 180, + screen: { display: 'LCD-1', coordinateSpace: 'native-panel', widthPt: 669, heightPt: 951 }, + message: + 'Folded to open (hinge 180°, LCD-1 native panel 669x951pt, not snapshot coordinates); refs from before the pose change are stale', + } satisfies CommandExecutionResult<'fold'>; + const executor = createCommandToolExecutor({ + createClient: () => ({}) as AgentDeviceClient, + runCommand: async () => foldResult, + }); + + const foldTool = listCommandTools().find((tool) => tool.name === 'fold'); + assert.ok(foldTool?.outputSchema); + assert.equal(foldTool.outputSchema, COMMAND_OUTPUT_SCHEMAS.fold); + + const result = await executor.execute('fold', { pose: 'open' }); + assert.deepEqual(result.structuredContent, foldResult); + assert.deepEqual(validateAgainstSchema(result.structuredContent, foldTool.outputSchema), []); + + // A panel report that drops the discriminator no longer validates: the coordinate-space contract + // is required, not inferable from the numbers alone. + const { coordinateSpace: _coordinateSpace, ...screenWithoutDiscriminator } = foldResult.screen; + assert.notDeepEqual( + validateAgainstSchema( + { ...foldResult, screen: screenWithoutDiscriminator }, + foldTool.outputSchema, + ), + [], + ); +}); + test('MCP applies config-backed command defaults; explicit operator input is refused', async () => { const home = mkdtempForTestSync('agent-device-mcp-config-'); temporaryDirectory = home; diff --git a/src/mcp/command-output-schemas.ts b/src/mcp/command-output-schemas.ts index 094232fb2f..1f64e203ed 100644 --- a/src/mcp/command-output-schemas.ts +++ b/src/mcp/command-output-schemas.ts @@ -6,6 +6,7 @@ import { BACK_MODES } from '@agent-device/contracts/back-mode'; import { NATIVE_PATH_DISPOSITION_VALUES } from '@agent-device/contracts/recording-native-path'; import { RECORDER_OBSERVATION_VALUES } from '@agent-device/contracts/recording-stop-observation'; import { DEVICE_ROTATIONS, FOLD_POSES } from '@agent-device/contracts/device'; +import { FOLD_SCREEN_COORDINATE_SPACE } from '@agent-device/contracts/fold-runtime'; import { SESSION_SURFACES } from '@agent-device/contracts/session'; import { TV_REMOTE_BUTTONS } from '@agent-device/contracts/tv-remote'; import { DEVICE_TARGETS, PUBLIC_PLATFORMS } from '@agent-device/kernel/device'; @@ -525,10 +526,15 @@ const BASE_COMMAND_OUTPUT_SCHEMAS = { screen: objectSchema( { display: stringSchema('CoreDevice name of the panel the device now lights.'), - widthPt: numberSchema(), - heightPt: numberSchema(), + coordinateSpace: constSchema(FOLD_SCREEN_COORDINATE_SPACE), + widthPt: numberSchema( + 'Panel width in native panel points (pixels divided by point scale), NOT snapshot coordinates; take a fresh snapshot to place a tap.', + ), + heightPt: numberSchema( + 'Panel height in native panel points (pixels divided by point scale), NOT snapshot coordinates; take a fresh snapshot to place a tap.', + ), }, - ['display', 'widthPt', 'heightPt'], + ['display', 'coordinateSpace', 'widthPt', 'heightPt'], ), message: stringSchema(), }, diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index f3a0629246..44c99a6af7 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -91,7 +91,7 @@ agent-device fold open - `action-button` asks the device whether it has the button before pressing it. A target whose model has none — an iPhone SE beside an iPhone 15, or most iPad simulators — fails with `UNSUPPORTED_OPERATION` rather than reporting a press that never happened. - `action-button` does not activate or relaunch the session's app, and it takes no `--settle`: pressing a hardware button is not a navigation, so the app stays where it was. - `action-button` reports that the press was dispatched, not what the system did with it. Simulators run no Shortcuts and no App Intents, so what a press triggers can only be verified on a physical iPhone; on a Simulator the command proves the press was accepted and that the session app was not brought forward. -- `fold ` puts a foldable iPhone simulator (iPhone Duo) into a hinge pose. No official host API sets a pose (ADR 0025), so the command presses the pose control in the Xcode Device Hub window through macOS accessibility, then reads the hinge angle back with `devicectl device motion hinge-angle` and reports the pose only when that reading agrees: `closed` is 0°, `open` is 180°, and `half-open` is any angle between them (Device Hub's Book preset, 130° on iOS 27.1). An angle inside that interval only proves the category, so `half-open` is reported once two consecutive readings both fall inside it and agree within 0.5°. The response names the panel the device now lights and its point size, because a pose change moves the app to a different coordinate space: re-snapshot afterwards, and never carry refs or coordinates across a `fold`. +- `fold ` puts a foldable iPhone simulator (iPhone Duo) into a hinge pose. No official host API sets a pose (ADR 0025), so the command presses the pose control in the Xcode Device Hub window through macOS accessibility, then reads the hinge angle back with `devicectl device motion hinge-angle` and reports the pose only when that reading agrees: `closed` is 0°, `open` is 180°, and `half-open` is any angle between them (Device Hub's Book preset, 130° on iOS 27.1). An angle inside that interval only proves the category, so `half-open` is reported once two consecutive readings both fall inside it and agree within 0.5°. The response names the panel the device now lights and its native panel point size, marked `coordinateSpace: "native-panel"`; that size is the panel's own geometry, not the next snapshot's viewport (a 669x951 inner panel can host a 951x669 app window), so it cannot place a tap. Re-snapshot afterwards, and never carry refs or coordinates across a `fold`. - `fold` is simulator-only and needs Xcode 27.1 or newer with Device Hub. The host needs Accessibility permission (`settings permission grant accessibility --platform macos`), and Device Hub must list the simulator; the command reopens Device Hub's window when it shows none, selects the device through the sidebar row keyed by its UDID, and restores the sidebar afterwards. A single-panel simulator such as an iPhone 17 fails with `UNSUPPORTED_OPERATION`; Android, web, Linux, HarmonyOS, Vega, physical devices, and the tvOS, macOS, and visionOS leaves refuse it. - `fold` costs one bounded hinge stream per read, and devicectl's smallest stream is five seconds: `closed` and `open` take about ten seconds, `half-open` about sixteen, because the hinge animates and the command waits for it to stop. A hinge whose last reading is some other pose fails with `COMMAND_FAILED` and `reason: fold-pose-unverified`, naming the angle CoreDevice still reports. A hinge seen `half-open` but never at rest fails with `reason: fold-pose-unsettled`, naming the observed and previous angles: the requested category was observed, and what is missing is a pose the hinge holds (#2730). - `action-button` is not a cheap command to loop. On an iPhone 17 Pro Simulator the press itself spent about five seconds inside XCUITest, while `home` and `app-switcher` on the same session took under two seconds each. From 68a5e64cc8eb3885616755bec2c981e1bfc794e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 22 Sep 2026 10:33:11 +0200 Subject: [PATCH 007/132] fix(apple): harden Device Hub discovery and sidebar recovery (#2733) --- .../DeviceHubWindowInventory.swift | 65 ++++++ .../DeviceHubPose.swift | 185 +++++------------- .../DeviceHubSidebar.swift | 57 ++++++ .../DeviceHubWindowDiscovery.swift | 108 ++++++++++ .../DeviceHubWindowInventoryTests.swift | 65 ++++++ docs/adr/0025-foldable-apple-panels.md | 43 ++-- .../src/os/macos/helper.test.ts | 48 ++++- website/docs/docs/commands.md | 3 +- 8 files changed, 414 insertions(+), 160 deletions(-) create mode 100644 apple/macos-helper/Sources/AgentDeviceMacOSDeviceHub/DeviceHubWindowInventory.swift create mode 100644 apple/macos-helper/Sources/AgentDeviceMacOSHelper/DeviceHubSidebar.swift create mode 100644 apple/macos-helper/Sources/AgentDeviceMacOSHelper/DeviceHubWindowDiscovery.swift create mode 100644 apple/macos-helper/Tests/AgentDeviceMacOSDeviceHubTests/DeviceHubWindowInventoryTests.swift diff --git a/apple/macos-helper/Sources/AgentDeviceMacOSDeviceHub/DeviceHubWindowInventory.swift b/apple/macos-helper/Sources/AgentDeviceMacOSDeviceHub/DeviceHubWindowInventory.swift new file mode 100644 index 0000000000..066c86f340 --- /dev/null +++ b/apple/macos-helper/Sources/AgentDeviceMacOSDeviceHub/DeviceHubWindowInventory.swift @@ -0,0 +1,65 @@ +import ApplicationServices + +package enum DeviceHubWindowRead { + case available([AXUIElement]) + case failed(Int32) +} + +package struct DeviceHubWindowCandidate { + package let processID: Int32 + package let window: AXUIElement +} + +package enum DeviceHubRowLookup { + case identified(AXUIElement, restoreSidebar: Bool) + case sidebarUnavailable + case unconfirmed + case timedOut + + package var diagnostic: String { + switch self { + case .identified: return "identified" + case .sidebarUnavailable: return "sidebar-unavailable" + case .unconfirmed: return "row-unconfirmed" + case .timedOut: return "timed-out" + } + } +} + +package struct DeviceHubWindowInventory { + package private(set) var candidates: [DeviceHubWindowCandidate] = [] + package private(set) var emptyProcessIDs: [Int32] = [] + package private(set) var failures: [Int32: Int32] = [:] + package private(set) var candidateOutcomes: [String] = [] + package private(set) var budgetExpired = false + + package init() {} + + package mutating func record(processID: Int32, read: DeviceHubWindowRead) { + switch read { + case .available(let windows): + if windows.isEmpty { emptyProcessIDs.append(processID) } + candidates.append(contentsOf: windows.map { DeviceHubWindowCandidate(processID: processID, window: $0) }) + case .failed(let status): + failures[processID] = status + } + } + + package mutating func record(candidate: Int, outcome: DeviceHubRowLookup) { + candidateOutcomes.append("\(candidates[candidate].processID):\(candidate):\(outcome.diagnostic)") + } + + package mutating func expireBudget() { budgetExpired = true } + + package func processesToReopen(attempted: Set) -> [Int32] { + budgetExpired ? [] : emptyProcessIDs.filter { !attempted.contains($0) } + } + + package func failureReason(reopenFailures: [Int32: String] = [:]) -> String { + if !failures.isEmpty { return "device-hub-window-read-failed" } + if budgetExpired { return "device-hub-discovery-timeout" } + if !candidates.isEmpty { return "device-hub-identity-unconfirmed" } + if let reason = reopenFailures.sorted(by: { $0.key < $1.key }).first?.value { return reason } + return "device-hub-window-missing" + } +} diff --git a/apple/macos-helper/Sources/AgentDeviceMacOSHelper/DeviceHubPose.swift b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/DeviceHubPose.swift index c33f4a69a4..05a6cce6e7 100644 --- a/apple/macos-helper/Sources/AgentDeviceMacOSHelper/DeviceHubPose.swift +++ b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/DeviceHubPose.swift @@ -4,12 +4,7 @@ import ApplicationServices import Darwin import Foundation -/// The Xcode Device Hub application. Its device windows carry an action bar whose pose controls -/// are the only public seam onto the private channel that folds a simulator. -private let deviceHubBundleId = "com.apple.dt.Devices" - -/// How long a reopen gets to restore a window, and a sidebar selection to switch the window's -/// device, before the press is refused. +/// How long sidebar selection and pose-control discovery may each wait before refusing a press. private let deviceHubSettleDeadline: TimeInterval = 8 private let deviceHubPoll: TimeInterval = 0.25 @@ -51,16 +46,15 @@ func handleDeviceHub(arguments: [String]) throws -> any Encodable { details: ["reason": "accessibility-permission", "permission": "accessibility"] ) } - guard let pid = deviceHubProcessIdentifier() else { - throw HelperError.commandFailed( - "Xcode Device Hub is not running", - details: ["reason": "device-hub-not-running", "bundleId": deviceHubBundleId] - ) + let discovery = try discoverDeviceHubWindow(udid: udid, deviceName: deviceName) + let window = discovery.window + defer { + if discovery.revealedSidebar { + _ = setDeviceHubSidebar(visible: false, window: window, + deadline: ProcessInfo.processInfo.systemUptime + 0.5) + } } - - let appElement = AXUIElementCreateApplication(pid) - let (window, reopened) = try deviceHubWindow(in: appElement, pid: pid, deviceName: deviceName) - let selected = try selectDevice(udid: udid, deviceName: deviceName, in: window, appElement: appElement) + let selected = try selectDevice(row: discovery.row, deviceName: deviceName, in: window) let windowTitle = stringAttribute(window, attribute: kAXTitleAttribute as String) ?? "" guard let button = awaitPoseButton(in: window, description: control) else { throw HelperError.commandFailed( @@ -84,64 +78,12 @@ func handleDeviceHub(arguments: [String]) throws -> any Encodable { pose: pose.rawValue, control: control, windowTitle: windowTitle, - reopened: reopened, + reopened: discovery.reopened, selected: selected ) ) } -/// Device Hub is launched through a trampoline, so LaunchServices registers it with no process -/// identifier (`NSRunningApplication.processIdentifier` is -1) and an accessibility element built -/// from that identifier is invalid. The process table is the only place its real pid appears. -private func deviceHubProcessIdentifier() -> pid_t? { - let executableSuffix = deviceHubExecutableSuffix - let byteCount = proc_listpids(UInt32(PROC_ALL_PIDS), 0, nil, 0) - guard byteCount > 0 else { return nil } - var pids = [pid_t](repeating: 0, count: Int(byteCount) / MemoryLayout.size + 64) - let filled = proc_listpids( - UInt32(PROC_ALL_PIDS), 0, &pids, Int32(pids.count * MemoryLayout.size) - ) - guard filled > 0 else { return nil } - var path = [CChar](repeating: 0, count: 4096) - for pid in pids.prefix(Int(filled) / MemoryLayout.size) where pid > 0 { - guard proc_pidpath(pid, &path, UInt32(path.count)) > 0 else { continue } - if String(cString: path).hasSuffix(executableSuffix) { - return pid - } - } - return nil -} - -/// A device window to drive: the one already showing this device when there is one, otherwise -/// any device window, since its sidebar can switch it to the device. A Device Hub with no window -/// at all — a simulator booted headlessly leaves it that way — is asked to reopen one. -private func deviceHubWindow( - in appElement: AXUIElement, - pid: pid_t, - deviceName: String -) throws -> (window: AXUIElement, reopened: Bool) { - if let window = preferredWindow(in: appElement, deviceName: deviceName) { - return (window, false) - } - try sendReopenEvent(to: pid) - let deadline = Date().addingTimeInterval(deviceHubSettleDeadline) - while Date() < deadline { - Thread.sleep(forTimeInterval: deviceHubPoll) - if let window = preferredWindow(in: appElement, deviceName: deviceName) { - return (window, true) - } - } - throw HelperError.commandFailed( - "Device Hub shows no device window to drive", - details: ["reason": "device-hub-window-missing", "deviceName": deviceName] - ) -} - -private func preferredWindow(in appElement: AXUIElement, deviceName: String) -> AXUIElement? { - let candidates = windows(of: appElement) - return candidates.first { windowShows(deviceName: deviceName, $0) } ?? candidates.first -} - private func windowShows(deviceName: String, _ window: AXUIElement) -> Bool { return deviceHubWindowShows( deviceName: deviceName, @@ -152,7 +94,7 @@ private func windowShows(deviceName: String, _ window: AXUIElement) -> Bool { /// `kAEReopenApplication` is what the Dock sends when an app with no open windows is clicked, and /// it is the one event Device Hub answers by restoring the device window. Sent to the process /// directly, because LaunchServices cannot address a trampolined app by bundle identifier. -private func sendReopenEvent(to pid: pid_t) throws { +func sendDeviceHubReopenEvent(to pid: pid_t, timeout: TimeInterval) throws { let target = NSAppleEventDescriptor(processIdentifier: pid) let event = NSAppleEventDescriptor( eventClass: AEEventClass(kCoreEventClass), @@ -162,7 +104,7 @@ private func sendReopenEvent(to pid: pid_t) throws { transactionID: AETransactionID(kAnyTransactionID) ) do { - _ = try event.sendEvent(options: [.noReply], timeout: 5) + _ = try event.sendEvent(options: [.noReply], timeout: min(5, max(0.001, timeout))) } catch { let code = (error as NSError).code throw HelperError.commandFailed( @@ -178,66 +120,43 @@ private func sendReopenEvent(to pid: pid_t) throws { /// Switches the window to the device through its sidebar row, whose accessibility identifier is /// `TableRow.Device.` — the one place Device Hub exposes a device identity that two /// simulators sharing a name cannot confuse. A window already titled with the device still gets -/// the selection when the row is visible, because the title alone cannot tell such twins apart. +/// the selection, because the title alone cannot tell such twins apart. private func selectDevice( - udid: String, + row: AXUIElement, deviceName: String, - in window: AXUIElement, - appElement: AXUIElement + in window: AXUIElement ) throws -> Bool { - var shownSidebar = false - var row = deviceRow(udid: udid, in: window) - if row == nil, showSidebar(appElement: appElement) { - shownSidebar = true - let deadline = Date().addingTimeInterval(deviceHubSettleDeadline) - while row == nil, Date() < deadline { - Thread.sleep(forTimeInterval: deviceHubPoll) - row = deviceRow(udid: udid, in: window) - } - } - defer { - if shownSidebar { _ = pressMenuItem(appElement: appElement, menu: "View", item: "Hide Sidebar") } - } - guard let row else { - if windowShows(deviceName: deviceName, window) { - return false - } - throw HelperError.commandFailed( - "Device Hub lists no device \(udid) in its sidebar", - details: ["reason": "device-hub-device-missing", "udid": udid, "deviceName": deviceName] - ) - } + let deadline = ProcessInfo.processInfo.systemUptime + deviceHubSettleDeadline + AXUIElementSetMessagingTimeout(row, 0.25) + AXUIElementSetMessagingTimeout(window, 0.25) let status = AXUIElementSetAttributeValue(row, kAXSelectedAttribute as CFString, kCFBooleanTrue) - guard status == .success else { + guard status == .success || status == .cannotComplete else { throw HelperError.commandFailed( "Device Hub refused to select \(deviceName) in its sidebar", details: ["reason": "device-hub-select-failed", "status": "\(status.rawValue)"] ) } - let deadline = Date().addingTimeInterval(deviceHubSettleDeadline) - while !windowShows(deviceName: deviceName, window), Date() < deadline { + while ProcessInfo.processInfo.systemUptime < deadline { + var selected: CFTypeRef? + if AXUIElementCopyAttributeValue(row, kAXSelectedAttribute as CFString, &selected) == .success, + (selected as? Bool) == true, windowShows(deviceName: deviceName, window) { return true } Thread.sleep(forTimeInterval: deviceHubPoll) } - guard windowShows(deviceName: deviceName, window) else { - throw HelperError.commandFailed( - "Device Hub did not switch its window to \(deviceName)", - details: [ - "reason": "device-hub-select-unconfirmed", - "windowTitle": stringAttribute(window, attribute: kAXTitleAttribute as String) ?? "", - ] - ) - } - return true + throw HelperError.commandFailed( + "Device Hub did not confirm selection of \(deviceName)", + details: ["reason": "device-hub-select-unconfirmed", "status": "\(status.rawValue)"] + ) } -private func deviceRow(udid: String, in window: AXUIElement) -> AXUIElement? { - guard let label = findElement(root: window, depth: 0, where: { +func deviceHubDeviceRow(udid: String, in window: AXUIElement, deadline: TimeInterval) -> AXUIElement? { + guard let label = findDeviceHubElement(root: window, depth: 0, deadline: deadline, where: { stringAttribute($0, attribute: "AXIdentifier") == deviceHubDeviceRowIdentifier(udid: udid) }) else { return nil } var current: AXUIElement? = label - while let element = current { + while let element = current, ProcessInfo.processInfo.systemUptime < deadline { + AXUIElementSetMessagingTimeout(element, Float(max(0.001, min(0.25, deadline - ProcessInfo.processInfo.systemUptime)))) if stringAttribute(element, attribute: kAXRoleAttribute as String) == "AXRow" { return element } @@ -246,36 +165,16 @@ private func deviceRow(udid: String, in window: AXUIElement) -> AXUIElement? { return nil } -private func showSidebar(appElement: AXUIElement) -> Bool { - return pressMenuItem(appElement: appElement, menu: "View", item: "Show Sidebar") -} - -private func pressMenuItem(appElement: AXUIElement, menu: String, item: String) -> Bool { - guard let menuBar = elementAttribute(appElement, attribute: kAXMenuBarAttribute as String) else { - return false - } - for menuBarItem in children(of: menuBar) - where stringAttribute(menuBarItem, attribute: kAXTitleAttribute as String) == menu { - for submenu in children(of: menuBarItem) { - for menuItem in children(of: submenu) - where stringAttribute(menuItem, attribute: kAXTitleAttribute as String) == item { - return AXUIElementPerformAction(menuItem, kAXPressAction as CFString) == .success - } - } - } - return false -} - /// The action bar is rebuilt for the device the window shows, so right after a sidebar selection /// the window already carries the new title while the pose controls are still being laid out. /// The controls are therefore awaited, not looked up once. private func awaitPoseButton(in window: AXUIElement, description: String) -> AXUIElement? { - let deadline = Date().addingTimeInterval(deviceHubSettleDeadline) + let deadline = ProcessInfo.processInfo.systemUptime + deviceHubSettleDeadline while true { - if let button = poseButton(in: window, description: description) { + if let button = poseButton(in: window, description: description, deadline: deadline) { return button } - guard Date() < deadline else { return nil } + guard ProcessInfo.processInfo.systemUptime < deadline else { return nil } Thread.sleep(forTimeInterval: deviceHubPoll) } } @@ -283,29 +182,35 @@ private func awaitPoseButton(in window: AXUIElement, description: String) -> AXU /// The pose controls sit in the window's action bar as `AXButton`s described by their preset /// name; the simulated screen inside the same window is an iOS content group whose own buttons /// carry app labels, never these three. -private func poseButton(in window: AXUIElement, description: String) -> AXUIElement? { - return findElement(root: window, depth: 0) { +private func poseButton(in window: AXUIElement, description: String, deadline: TimeInterval) -> AXUIElement? { + return findDeviceHubElement(root: window, depth: 0, deadline: deadline) { stringAttribute($0, attribute: kAXRoleAttribute as String) == "AXButton" && stringAttribute($0, attribute: kAXDescriptionAttribute as String) == description } } -private func findElement( +func findDeviceHubElement( root: AXUIElement, depth: Int, + deadline: TimeInterval, where matches: (AXUIElement) -> Bool ) -> AXUIElement? { - if depth > 14 { + let remaining = deadline - ProcessInfo.processInfo.systemUptime + if depth > 14 || remaining <= 0 { return nil } + AXUIElementSetMessagingTimeout(root, Float(min(0.25, remaining))) for child in children(of: root) { + let remaining = deadline - ProcessInfo.processInfo.systemUptime + guard remaining > 0 else { return nil } + AXUIElementSetMessagingTimeout(child, Float(min(0.25, remaining))) if matches(child) { return child } if stringAttribute(child, attribute: kAXSubroleAttribute as String) == "iOSContentGroup" { continue } - if let nested = findElement(root: child, depth: depth + 1, where: matches) { + if let nested = findDeviceHubElement(root: child, depth: depth + 1, deadline: deadline, where: matches) { return nested } } diff --git a/apple/macos-helper/Sources/AgentDeviceMacOSHelper/DeviceHubSidebar.swift b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/DeviceHubSidebar.swift new file mode 100644 index 0000000000..efebba26ea --- /dev/null +++ b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/DeviceHubSidebar.swift @@ -0,0 +1,57 @@ +import AgentDeviceMacOSDeviceHub +import ApplicationServices +import Foundation + +func identifyDeviceHubRow( + udid: String, + candidate: DeviceHubWindowCandidate, + deadline: TimeInterval +) -> DeviceHubRowLookup { + let now = ProcessInfo.processInfo.systemUptime + let visibleRowDeadline = now + max(0, deadline - now) / 3 + if let row = deviceHubDeviceRow(udid: udid, in: candidate.window, deadline: visibleRowDeadline) { + return .identified(row, restoreSidebar: false) + } + let searchDeadline = deadline - 0.25 + guard let button = deviceHubSidebarButton(visible: true, window: candidate.window, deadline: searchDeadline) else { + return ProcessInfo.processInfo.systemUptime >= searchDeadline ? .timedOut : .sidebarUnavailable + } + var transferCleanup = false + defer { + if !transferCleanup { + _ = setDeviceHubSidebar(visible: false, window: candidate.window, deadline: deadline) + } + } + let status = pressDeviceHubSidebarButton(button, deadline: searchDeadline) + guard status == .success || status == .cannotComplete else { return .unconfirmed } + repeat { + if let row = deviceHubDeviceRow(udid: udid, in: candidate.window, deadline: searchDeadline) { + transferCleanup = true + return .identified(row, restoreSidebar: true) + } + let remaining = searchDeadline - ProcessInfo.processInfo.systemUptime + if remaining <= 0 { return .timedOut } + Thread.sleep(forTimeInterval: min(0.05, remaining)) + } while ProcessInfo.processInfo.systemUptime < searchDeadline + return .timedOut +} + +func setDeviceHubSidebar(visible: Bool, window: AXUIElement, deadline: TimeInterval) -> Bool { + guard let button = deviceHubSidebarButton(visible: visible, window: window, deadline: deadline) else { return false } + return pressDeviceHubSidebarButton(button, deadline: deadline) == .success +} + +private func deviceHubSidebarButton(visible: Bool, window: AXUIElement, deadline: TimeInterval) -> AXUIElement? { + let description = visible ? "Show Sidebar" : "Hide Sidebar" + return findDeviceHubElement(root: window, depth: 0, deadline: deadline) { + stringAttribute($0, attribute: kAXRoleAttribute as String) == "AXButton" + && stringAttribute($0, attribute: kAXDescriptionAttribute as String) == description + } +} + +private func pressDeviceHubSidebarButton(_ button: AXUIElement, deadline: TimeInterval) -> AXError { + let remaining = deadline - ProcessInfo.processInfo.systemUptime + guard remaining > 0 else { return .cannotComplete } + AXUIElementSetMessagingTimeout(button, Float(min(0.1, remaining))) + return AXUIElementPerformAction(button, kAXPressAction as CFString) +} diff --git a/apple/macos-helper/Sources/AgentDeviceMacOSHelper/DeviceHubWindowDiscovery.swift b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/DeviceHubWindowDiscovery.swift new file mode 100644 index 0000000000..efc125c6da --- /dev/null +++ b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/DeviceHubWindowDiscovery.swift @@ -0,0 +1,108 @@ +import AgentDeviceMacOSDeviceHub +import AppKit +import ApplicationServices +import Darwin + +struct DiscoveredDeviceHubWindow { + let window: AXUIElement + let reopened: Bool + let row: AXUIElement + let revealedSidebar: Bool +} + +func discoverDeviceHubWindow(udid: String, deviceName: String) throws -> DiscoveredDeviceHubWindow { + let deadline = ProcessInfo.processInfo.systemUptime + 8 + var reopened = Set() + var reopenFailures: [pid_t: String] = [:] + var inventory = DeviceHubWindowInventory() + var processIDs: [pid_t] = [] + repeat { + processIDs = deviceHubProcessIdentifiers() + if processIDs.isEmpty { + throw HelperError.commandFailed( + "Xcode Device Hub is not running", + details: ["reason": "device-hub-not-running", "bundleId": "com.apple.dt.Devices"] + ) + } + inventory = DeviceHubWindowInventory() + for pid in processIDs { + let remaining = deadline - ProcessInfo.processInfo.systemUptime + guard remaining > 0 else { inventory.expireBudget(); break } + let app = AXUIElementCreateApplication(pid) + AXUIElementSetMessagingTimeout(app, Float(min(0.5, remaining))) + var value: CFTypeRef? + let status = AXUIElementCopyAttributeValue(app, kAXWindowsAttribute as CFString, &value) + if status == .success, let windows = value as? [AXUIElement] { + for window in windows { AXUIElementSetMessagingTimeout(window, 0.5) } + inventory.record(processID: pid, read: .available(windows)) + } else { + inventory.record(processID: pid, read: .failed( + status == .success ? AXError.illegalArgument.rawValue : status.rawValue + )) + } + } + for (index, candidate) in inventory.candidates.enumerated() { + let now = ProcessInfo.processInfo.systemUptime + guard now < deadline else { inventory.expireBudget(); break } + let candidateDeadline = now + (deadline - now) / Double(inventory.candidates.count - index) / 2 + let outcome = identifyDeviceHubRow(udid: udid, candidate: candidate, deadline: candidateDeadline) + inventory.record(candidate: index, outcome: outcome) + if case .identified(let row, let restoreSidebar) = outcome { + return DiscoveredDeviceHubWindow(window: candidate.window, + reopened: reopened.contains(candidate.processID), row: row, revealedSidebar: restoreSidebar) + } + } + let attempted = reopened.union(reopenFailures.keys) + for pid in inventory.processesToReopen(attempted: attempted) { + guard ProcessInfo.processInfo.systemUptime < deadline else { inventory.expireBudget(); break } + do { + try sendDeviceHubReopenEvent(to: pid, timeout: deadline - ProcessInfo.processInfo.systemUptime) + reopened.insert(pid) + } catch let error as HelperError { + guard case .commandFailed(_, let details) = error else { throw error } + reopenFailures[pid] = details["reason"] ?? "device-hub-reopen-failed" + } + } + let remaining = deadline - ProcessInfo.processInfo.systemUptime + if remaining <= 0 { inventory.expireBudget(); break } + Thread.sleep(forTimeInterval: min(0.25, remaining)) + } while ProcessInfo.processInfo.systemUptime < deadline + + let statuses = inventory.failures.sorted { $0.key < $1.key } + .map { "\($0.key):\($0.value)" }.joined(separator: ",") + throw HelperError.commandFailed( + "Could not confirm a Device Hub window for the requested device", + details: [ + "reason": inventory.failureReason(reopenFailures: reopenFailures), + "reopenFailures": reopenFailures.sorted { $0.key < $1.key }.map { "\($0.key):\($0.value)" }.joined(separator: ","), + "candidateOutcomes": inventory.candidateOutcomes.joined(separator: ","), + "udid": udid, + "deviceName": deviceName, + "processIDs": processIDs.map(String.init).joined(separator: ","), + "axWindowReadStatuses": statuses, + "hostDisplayIDs": deviceHubHostDisplays().map { String($0) }.joined(separator: ","), + ] + ) +} + +private func deviceHubProcessIdentifiers() -> [pid_t] { + let byteCount = proc_listpids(UInt32(PROC_ALL_PIDS), 0, nil, 0) + guard byteCount > 0 else { return [] } + var pids = [pid_t](repeating: 0, count: Int(byteCount) / MemoryLayout.size + 64) + let filled = proc_listpids(UInt32(PROC_ALL_PIDS), 0, &pids, Int32(pids.count * MemoryLayout.size)) + guard filled > 0 else { return [] } + return pids.prefix(Int(filled) / MemoryLayout.size).filter { pid in + guard pid > 0 else { return false } + var path = [CChar](repeating: 0, count: 4096) + guard proc_pidpath(pid, &path, UInt32(path.count)) > 0 else { return false } + return String(cString: path).hasSuffix(deviceHubExecutableSuffix) + }.sorted() +} + +private func deviceHubHostDisplays() -> [CGDirectDisplayID] { + var count: UInt32 = 0 + guard CGGetActiveDisplayList(0, nil, &count) == .success else { return [] } + var displays = [CGDirectDisplayID](repeating: 0, count: Int(count)) + guard CGGetActiveDisplayList(count, &displays, &count) == .success else { return [] } + return Array(displays.prefix(Int(count))) +} diff --git a/apple/macos-helper/Tests/AgentDeviceMacOSDeviceHubTests/DeviceHubWindowInventoryTests.swift b/apple/macos-helper/Tests/AgentDeviceMacOSDeviceHubTests/DeviceHubWindowInventoryTests.swift new file mode 100644 index 0000000000..9ab682dcde --- /dev/null +++ b/apple/macos-helper/Tests/AgentDeviceMacOSDeviceHubTests/DeviceHubWindowInventoryTests.swift @@ -0,0 +1,65 @@ +import ApplicationServices +import XCTest +@testable import AgentDeviceMacOSDeviceHub + +final class DeviceHubWindowInventoryTests: XCTestCase { + func testLaterWindowKeepsItsProcessAndDoesNotPreventEmptyProcessReopen() { + var inventory = DeviceHubWindowInventory() + let window = AXUIElementCreateApplication(20) + inventory.record(processID: 10, read: .available([])) + inventory.record(processID: 20, read: .available([window])) + XCTAssertEqual(inventory.candidates.map(\.processID), [20]) + XCTAssertTrue(CFEqual(inventory.candidates[0].window, window)) + inventory.record(candidate: 0, outcome: .sidebarUnavailable) + XCTAssertEqual(inventory.processesToReopen(attempted: []), [10]) + XCTAssertEqual(inventory.failureReason(), "device-hub-identity-unconfirmed") + XCTAssertEqual(inventory.candidateOutcomes, ["20:0:sidebar-unavailable"]) + } + + func testFailedReadsAreNotReopenedOrMisreportedAsMissingWindows() { + var inventory = DeviceHubWindowInventory() + inventory.record(processID: 10, read: .failed(-25204)) + inventory.record(processID: 20, read: .available([])) + XCTAssertEqual(inventory.processesToReopen(attempted: []), [20]) + XCTAssertEqual(inventory.failures, [10: -25204]) + XCTAssertEqual(inventory.failureReason(), "device-hub-window-read-failed") + } + + func testSuccessfulEmptyReadIsMissingAndReopenIsAttemptedOnlyOnce() { + var inventory = DeviceHubWindowInventory() + inventory.record(processID: 10, read: .available([])) + XCTAssertEqual(inventory.failureReason(), "device-hub-window-missing") + XCTAssertEqual(inventory.processesToReopen(attempted: []), [10]) + XCTAssertEqual(inventory.processesToReopen(attempted: [10]), []) + } + + func testSharedBudgetExhaustionStopsReopenWithoutClaimingMissingDevice() { + var inventory = DeviceHubWindowInventory() + inventory.record(processID: 10, read: .available([])) + inventory.expireBudget() + XCTAssertEqual(inventory.processesToReopen(attempted: []), []) + XCTAssertEqual(inventory.failureReason(), "device-hub-discovery-timeout") + } + + func testCandidateOutcomesRetainUncertaintyAndOwningProcess() { + var inventory = DeviceHubWindowInventory() + inventory.record(processID: 10, read: .available([AXUIElementCreateApplication(10)])) + inventory.record(processID: 20, read: .available([AXUIElementCreateApplication(20)])) + inventory.record(candidate: 0, outcome: .timedOut) + inventory.record(candidate: 1, outcome: .unconfirmed) + XCTAssertEqual(inventory.candidateOutcomes, ["10:0:timed-out", "20:1:row-unconfirmed"]) + XCTAssertEqual(inventory.failureReason(), "device-hub-identity-unconfirmed") + } + func testUnrelatedReopenFailureDoesNotMaskCandidateUncertainty() { + var inventory = DeviceHubWindowInventory() + inventory.record(processID: 10, read: .available([])) + let failures: [Int32: String] = [10: "automation-permission"] + XCTAssertEqual(inventory.failureReason(reopenFailures: failures), "automation-permission") + inventory.record(processID: 20, read: .available([AXUIElementCreateApplication(20)])) + inventory.record(candidate: 0, outcome: .timedOut) + XCTAssertEqual(inventory.failureReason(reopenFailures: failures), "device-hub-identity-unconfirmed") + inventory.expireBudget() + XCTAssertEqual(inventory.failureReason(reopenFailures: failures), "device-hub-discovery-timeout") + } + +} diff --git a/docs/adr/0025-foldable-apple-panels.md b/docs/adr/0025-foldable-apple-panels.md index b8031df053..1be5ff2ac3 100644 --- a/docs/adr/0025-foldable-apple-panels.md +++ b/docs/adr/0025-foldable-apple-panels.md @@ -116,10 +116,26 @@ size, because the point size is what tells an agent its refs are stale. Requirements the command states in its own errors: Accessibility permission for the host (`accessibility-permission`), a running Device Hub (`fold` launches it in the background the way `open` does), a device window it can reopen (`device-hub-window-missing`), and a sidebar row for -the UDID (`device-hub-device-missing`). A single-panel simulator is refused before anything is +the UDID (otherwise `device-hub-identity-unconfirmed`). A single-panel simulator is refused before anything is pressed (`single-panel-device`), and the leaf fact refuses physical devices and every non-iPhone simulator OS. +Window discovery reads `AXWindows` from every matching Device Hub process within one shared +retry budget. These are application-wide windows; discovery does not click through monitors or +move windows to the main display. A failed accessibility read retains its AX status and returns +`device-hub-window-read-failed`, rather than masquerading as an empty window list. Only successful +empty reads trigger a reopen. A matching UDID sidebar row is required before pressing, including +when a window title already matches. An unrelated window does not stop attempts on other +windows or empty processes. A hidden sidebar is revealed through that window’s own toolbar button; +application-wide menus cannot redirect the action to another window. Sidebar reads and +restoration use bounded AX calls. Cleanup is registered before a sidebar press, since an AX reply +can time out after the action applied. Selection verifies the row's `AXSelected` state and the +window title even after an uncertain reply. Discovery passes the proven row to selection instead +of searching twice. Inconclusive candidates report `device-hub-identity-unconfirmed` with each +candidate's outcome; exhausting the shared budget before finishing work reports +`device-hub-discovery-timeout`. Neither claims a device is absent. Active host display IDs stay +in discovery error details; display geometry does not participate in AX window selection. + ## Amendment: an observed half-open angle is not a settled pose (issue #2730) The rule above first ended the other way: `half-open` was also reported when the four-read budget @@ -353,24 +369,17 @@ explicit. was measured, so that path stays unverified. - **Quarter-turn detection.** Both Duo panels report `currentOrientation: rot90`, and no available path rotates a foldable, so the orientation half of the inventory is carried but never exercised against a changed value. - **Pose control on a - second Device Hub instance.** `fold` drives the first `DeviceHub` process in the process table. - Two Xcodes each running a Device Hub is not a state this was verified in. - **A hinge that + second Device Hub instance.** Discovery checks every matching process, but simultaneous Xcodes + and the secondary-display fold matrix remain unverified on live hardware. - **A hinge that settles slowly.** Four reads is twenty seconds of streams, and a Duo that needs longer to come to rest inside `half-open` is now refused where the superseded rule would have reported a pose. Every Duo run observed for this change settled inside the budget; no simulator that needs longer was seen, so the budget stays as it is rather than growing on a hypothesis. -- **Pose control when two simulators share a name.** `fold` picks the device by the sidebar row's - `AXIdentifier`, but `selectDevice` in `DeviceHubPose.swift` confirms the switch through - `windowShows(deviceName:)`, which reads the window title, and it skips the selection altogether - when no row was found and that title already matches — and two simulators named `iPhone Duo` are - both titled `iPhone Duo – iOS 27.1`. Observed while pressing the poses for the captures above, - and consistent with that path, a `fold open` aimed at one Duo pressed the pose control of the - other, which unfolded while the intended device stayed closed; the `apple_fold_pose_pressed` - diagnostic reports `selected` and nothing gates the press on it. The hinge read-back is what - stopped the command reporting a pose it had not achieved, yet the untargeted device had already - moved. Selecting that device's row before pressing routed every later pose correctly, so the row - is the only device identity Device Hub offers and an unconfirmed selection leaves the press aimed - at whatever it happens to display. - **Physical foldables.** Device Hub poses simulators only; - the leaf fact refuses a physical device, and the hinge stream on one was not exercised. - +- **Pose control when two simulators share a name.** The earlier title-only fallback could press + a different Duo when the intended device's row was unavailable; hinge verification refused the + result only after the other device had moved. Discovery now requires the UDID row, and selection + confirms `AXSelected` as well as the window title before pressing. Live wrong-UDID refusal and + hidden-sidebar recovery were checked; the simultaneous same-name simulator matrix remains open. +- **Physical foldables.** Device Hub poses simulators only; the leaf fact refuses a physical device, + and the hinge stream on one was not exercised. diff --git a/packages/platform-apple/src/os/macos/helper.test.ts b/packages/platform-apple/src/os/macos/helper.test.ts index eb8d24b0e9..41f4def72a 100644 --- a/packages/platform-apple/src/os/macos/helper.test.ts +++ b/packages/platform-apple/src/os/macos/helper.test.ts @@ -1,7 +1,12 @@ import assert from 'node:assert/strict'; -import { test } from 'vitest'; +import { expect, test } from 'vitest'; import { createLocalAppleToolProvider, withAppleToolProvider } from '../../core/tool-provider.ts'; -import { macOsClickScheduleMs, runMacOsPressAction, runMacOsSnapshotAction } from './helper.ts'; +import { + macOsClickScheduleMs, + runMacOsPressAction, + runMacOsSnapshotAction, + runMacOsDeviceHubPoseAction, +} from './helper.ts'; test('macOS helper snapshot passes cancellation to the helper process', async () => { const controller = new AbortController(); @@ -207,3 +212,42 @@ test('macOS helper press stays a single held click when nothing is repeated', as assert.equal(receivedArgs.includes('--hold-ms'), false); assert.equal(receivedArgs.includes('--interval-ms'), false); }); + +test('Device Hub window read failures preserve AX status and process diagnostics', async () => { + const provider = createLocalAppleToolProvider({ + macosHelper: { + run: async () => ({ + exitCode: 1, + stdout: JSON.stringify({ + ok: false, + error: { + message: "Could not read Device Hub's accessible windows", + details: { + reason: 'device-hub-window-read-failed', + processIDs: '10,20', + axWindowReadStatuses: '10:-25204', + hostDisplayIDs: '1,2', + }, + }, + }), + stderr: '', + }), + }, + }); + await withAppleToolProvider(provider, async () => { + await assert.rejects( + runMacOsDeviceHubPoseAction({ udid: 'duo', deviceName: 'iPhone Duo', pose: 'open' }), + (error: unknown) => { + expect(error).toMatchObject({ + details: { + reason: 'device-hub-window-read-failed', + processIDs: '10,20', + axWindowReadStatuses: '10:-25204', + hostDisplayIDs: '1,2', + }, + }); + return true; + }, + ); + }); +}); diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 44c99a6af7..08f8c42f3f 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -92,7 +92,8 @@ agent-device fold open - `action-button` does not activate or relaunch the session's app, and it takes no `--settle`: pressing a hardware button is not a navigation, so the app stays where it was. - `action-button` reports that the press was dispatched, not what the system did with it. Simulators run no Shortcuts and no App Intents, so what a press triggers can only be verified on a physical iPhone; on a Simulator the command proves the press was accepted and that the session app was not brought forward. - `fold ` puts a foldable iPhone simulator (iPhone Duo) into a hinge pose. No official host API sets a pose (ADR 0025), so the command presses the pose control in the Xcode Device Hub window through macOS accessibility, then reads the hinge angle back with `devicectl device motion hinge-angle` and reports the pose only when that reading agrees: `closed` is 0°, `open` is 180°, and `half-open` is any angle between them (Device Hub's Book preset, 130° on iOS 27.1). An angle inside that interval only proves the category, so `half-open` is reported once two consecutive readings both fall inside it and agree within 0.5°. The response names the panel the device now lights and its native panel point size, marked `coordinateSpace: "native-panel"`; that size is the panel's own geometry, not the next snapshot's viewport (a 669x951 inner panel can host a 951x669 app window), so it cannot place a tap. Re-snapshot afterwards, and never carry refs or coordinates across a `fold`. -- `fold` is simulator-only and needs Xcode 27.1 or newer with Device Hub. The host needs Accessibility permission (`settings permission grant accessibility --platform macos`), and Device Hub must list the simulator; the command reopens Device Hub's window when it shows none, selects the device through the sidebar row keyed by its UDID, and restores the sidebar afterwards. A single-panel simulator such as an iPhone 17 fails with `UNSUPPORTED_OPERATION`; Android, web, Linux, HarmonyOS, Vega, physical devices, and the tvOS, macOS, and visionOS leaves refuse it. +- `fold` is simulator-only and needs Xcode 27.1 or newer with Device Hub. The host needs Accessibility permission (`settings permission grant accessibility --platform macos`), and Device Hub must list the simulator; the command reopens Device Hub's window when it shows none, selects the device through the sidebar row keyed by its UDID, and attempts to restore any sidebar it revealed afterwards within a bounded cleanup budget. Sidebar actions use the selected window’s toolbar button. A single-panel simulator such as an iPhone 17 fails with `UNSUPPORTED_OPERATION`; Android, web, Linux, HarmonyOS, Vega, physical devices, and the tvOS, macOS, and visionOS leaves refuse it. +- `fold` discovers windows across running Device Hub processes without requiring main-display placement. Failed accessibility reads report `device-hub-window-read-failed` with AX status and process/display diagnostics; they do not count as an empty window list. Inconclusive row lookup reports `device-hub-identity-unconfirmed` with candidate outcomes; an interrupted discovery reports `device-hub-discovery-timeout`. Active host display IDs are included only in discovery errors. - `fold` costs one bounded hinge stream per read, and devicectl's smallest stream is five seconds: `closed` and `open` take about ten seconds, `half-open` about sixteen, because the hinge animates and the command waits for it to stop. A hinge whose last reading is some other pose fails with `COMMAND_FAILED` and `reason: fold-pose-unverified`, naming the angle CoreDevice still reports. A hinge seen `half-open` but never at rest fails with `reason: fold-pose-unsettled`, naming the observed and previous angles: the requested category was observed, and what is missing is a pose the hinge holds (#2730). - `action-button` is not a cheap command to loop. On an iPhone 17 Pro Simulator the press itself spent about five seconds inside XCUITest, while `home` and `app-switcher` on the same session took under two seconds each. - On iOS devices, `http(s)://` URLs open in Safari when no app is active. Custom scheme URLs require an active app in the session. From d6ba1059287c9b28ce0775fbf23103950c322163 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 22 Sep 2026 13:08:17 +0200 Subject: [PATCH 008/132] refactor(capture-kit): resolve recording helper scripts from swift-cache (#2743) --- .../src/recording/__tests__/overlay.test.ts | 22 +------ packages/capture-kit/src/recording/overlay.ts | 61 ++++--------------- .../src/recording/swift-cache.test.ts | 26 +++++++- .../capture-kit/src/recording/swift-cache.ts | 54 ++++++++++++++++ 4 files changed, 93 insertions(+), 70 deletions(-) diff --git a/packages/capture-kit/src/recording/__tests__/overlay.test.ts b/packages/capture-kit/src/recording/__tests__/overlay.test.ts index aabd5b73e8..47b1b34180 100644 --- a/packages/capture-kit/src/recording/__tests__/overlay.test.ts +++ b/packages/capture-kit/src/recording/__tests__/overlay.test.ts @@ -28,7 +28,7 @@ vi.mock('../video.ts', () => ({ waitForPlayableVideo: vi.fn(async () => {}), })); -import { buildRecordingScriptPathCandidates, overlayRecordingTouches } from '../overlay.ts'; +import { overlayRecordingTouches } from '../overlay.ts'; import { AppError } from '@agent-device/kernel/errors'; import { runCmd } from '@agent-device/host-kit/command'; @@ -130,23 +130,3 @@ test('overlay forwards the requested high export preset', async () => { expect.arrayContaining(['--events', telemetryPath, '--quality', 'high']), ); }); - -test('recording script candidates include packaged dist apple-runner source', () => { - const packageRoot = path.join(tmpDir, 'package'); - const scriptPath = path.join( - packageRoot, - 'dist/apple/runner/AgentDeviceRunner/RecordingScripts/recording-overlay.swift', - ); - fs.mkdirSync(path.dirname(scriptPath), { recursive: true }); - fs.writeFileSync(scriptPath, 'print("overlay")\n'); - - const candidates = buildRecordingScriptPathCandidates( - 'recording-overlay.swift', - path.join(packageRoot, 'dist/src'), - packageRoot, - tmpDir, - ); - const firstExisting = candidates.find((candidate) => fs.existsSync(candidate)); - - expect(firstExisting).toBe(scriptPath); -}); diff --git a/packages/capture-kit/src/recording/overlay.ts b/packages/capture-kit/src/recording/overlay.ts index 6dc03ccf82..9e4397072f 100644 --- a/packages/capture-kit/src/recording/overlay.ts +++ b/packages/capture-kit/src/recording/overlay.ts @@ -1,60 +1,19 @@ import fs from 'node:fs'; import path from 'node:path'; -import { fileURLToPath } from 'node:url'; import { runCmd } from '@agent-device/host-kit/command'; import { AppError } from '@agent-device/kernel/errors'; -import { buildSwiftToolEnv, compileSwiftSourceFile } from './swift-cache.ts'; import { findProjectRoot } from '@agent-device/host-kit/version'; +import { + buildSwiftToolEnv, + compileSwiftSourceFile, + resolveRecordingScriptPath, +} from './swift-cache.ts'; import { waitForPlayableVideo, waitForStableFile } from './video.ts'; import { DEFAULT_RECORDING_EXPORT_QUALITY, type RecordingExportQuality, } from '@agent-device/contracts/recording'; -export function buildRecordingScriptPathCandidates( - scriptName: string, - moduleDir: string, - projectRoot: string, - cwd: string, -): string[] { - const sourceScriptPath = `apple/runner/AgentDeviceRunner/RecordingScripts/${scriptName}`; - const packagedScriptPath = `dist/${sourceScriptPath}`; - return [ - path.resolve(moduleDir, scriptName), - path.resolve(projectRoot, sourceScriptPath), - path.resolve(moduleDir, `../${sourceScriptPath}`), - path.resolve(moduleDir, `../../${sourceScriptPath}`), - path.resolve(moduleDir, `../../../${sourceScriptPath}`), - path.resolve(projectRoot, packagedScriptPath), - path.resolve(cwd, sourceScriptPath), - ]; -} - -function resolveRecordingScriptPath(scriptName: string): string { - const moduleDir = path.dirname(fileURLToPath(import.meta.url)); - const scriptCandidates = buildRecordingScriptPathCandidates( - scriptName, - moduleDir, - findProjectRoot(), - process.cwd(), - ); - - for (const candidate of scriptCandidates) { - if (fs.existsSync(candidate)) { - return candidate; - } - } - - throw new AppError('COMMAND_FAILED', `Missing recording helper script: ${scriptName}`, { - hint: 'Ensure apple/runner/AgentDeviceRunner/RecordingScripts is present in this checkout or bundled under dist/apple/runner in the package.', - scriptName, - searchedPaths: scriptCandidates, - }); -} - -let overlayScriptPath: string | undefined; -let exportSupportScriptPath: string | undefined; - export function getRecordingOverlaySupportWarning( hostPlatform: NodeJS.Platform = process.platform, ): string | undefined { @@ -64,13 +23,19 @@ export function getRecordingOverlaySupportWarning( return 'touch overlay burn-in is only available on macOS hosts; returning raw video plus gesture telemetry'; } +let overlayScriptPath: string | undefined; +let exportSupportScriptPath: string | undefined; + function getOverlayScriptPath(): string { - overlayScriptPath ??= resolveRecordingScriptPath('recording-overlay.swift'); + overlayScriptPath ??= resolveRecordingScriptPath('recording-overlay.swift', findProjectRoot()); return overlayScriptPath; } function getExportSupportScriptPath(): string { - exportSupportScriptPath ??= resolveRecordingScriptPath('RecordingExportSupport.swift'); + exportSupportScriptPath ??= resolveRecordingScriptPath( + 'RecordingExportSupport.swift', + findProjectRoot(), + ); return exportSupportScriptPath; } diff --git a/packages/capture-kit/src/recording/swift-cache.test.ts b/packages/capture-kit/src/recording/swift-cache.test.ts index 7ec3ca69a8..c008547f1b 100644 --- a/packages/capture-kit/src/recording/swift-cache.test.ts +++ b/packages/capture-kit/src/recording/swift-cache.test.ts @@ -14,7 +14,11 @@ vi.mock(import('@agent-device/host-kit/command'), async (importOriginal) => ({ })); import { runCmd } from '@agent-device/host-kit/command'; -import { compileSwiftSourceFile, compileSwiftSourceText } from './swift-cache.ts'; +import { + buildRecordingScriptPathCandidates, + compileSwiftSourceFile, + compileSwiftSourceText, +} from './swift-cache.ts'; const mockRunCmd = vi.mocked(runCmd); @@ -246,3 +250,23 @@ async function expectConcurrentCacheReuse(compile: () => Promise): Promi expect(fs.readFileSync(firstExecutable, 'utf8')).toBe('compiled once'); expect(mockRunCmd).toHaveBeenCalledTimes(1); } + +test('recording script candidates include packaged dist apple-runner source', () => { + const packageRoot = path.join(tmpDir, 'package'); + const scriptPath = path.join( + packageRoot, + 'dist/apple/runner/AgentDeviceRunner/RecordingScripts/recording-overlay.swift', + ); + fs.mkdirSync(path.dirname(scriptPath), { recursive: true }); + fs.writeFileSync(scriptPath, 'print("overlay")\n'); + + const candidates = buildRecordingScriptPathCandidates( + 'recording-overlay.swift', + path.join(packageRoot, 'dist/src'), + packageRoot, + tmpDir, + ); + const firstExisting = candidates.find((candidate) => fs.existsSync(candidate)); + + expect(firstExisting).toBe(scriptPath); +}); diff --git a/packages/capture-kit/src/recording/swift-cache.ts b/packages/capture-kit/src/recording/swift-cache.ts index 366a2db6b4..36471a8c3f 100644 --- a/packages/capture-kit/src/recording/swift-cache.ts +++ b/packages/capture-kit/src/recording/swift-cache.ts @@ -2,6 +2,7 @@ import { createHash } from 'node:crypto'; import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; +import { fileURLToPath } from 'node:url'; import { trimEdgeDashes } from '@agent-device/kernel/collections'; import { AppError } from '@agent-device/kernel/errors'; import { runCmd } from '@agent-device/host-kit/command'; @@ -14,6 +15,59 @@ import { const SWIFT_CACHE_VERSION = '2'; const LOCK_RETRY_DELAY_MS = 25; +const RECORDING_SCRIPT_SUBDIRECTORY = 'apple/runner/AgentDeviceRunner/RecordingScripts'; + +/** + * Where a recording helper script can live, in the order a checkout, a source build, and a + * published package each answer from. + */ +export function buildRecordingScriptPathCandidates( + scriptName: string, + moduleDir: string, + projectRoot: string, + cwd: string, +): string[] { + const sourceScriptPath = `${RECORDING_SCRIPT_SUBDIRECTORY}/${scriptName}`; + const packagedScriptPath = `dist/${sourceScriptPath}`; + return [ + path.resolve(moduleDir, scriptName), + path.resolve(projectRoot, sourceScriptPath), + path.resolve(moduleDir, `../${sourceScriptPath}`), + path.resolve(moduleDir, `../../${sourceScriptPath}`), + path.resolve(moduleDir, `../../../${sourceScriptPath}`), + path.resolve(projectRoot, packagedScriptPath), + path.resolve(cwd, sourceScriptPath), + ]; +} + +/** + * Resolves one recording helper script: checkout, built package, or working-directory fallback. + * The caller names the project root it trusts, so this module stays free of host metadata lookup. + */ +export function resolveRecordingScriptPath( + scriptName: string, + projectRoot: string, + moduleDir = path.dirname(fileURLToPath(import.meta.url)), + cwd = process.cwd(), +): string { + const scriptCandidates = buildRecordingScriptPathCandidates( + scriptName, + moduleDir, + projectRoot, + cwd, + ); + for (const candidate of scriptCandidates) { + if (fs.existsSync(candidate)) { + return candidate; + } + } + + throw new AppError('COMMAND_FAILED', `Missing recording helper script: ${scriptName}`, { + hint: `Ensure ${RECORDING_SCRIPT_SUBDIRECTORY} is present in this checkout or bundled under dist/${RECORDING_SCRIPT_SUBDIRECTORY} in the package.`, + scriptName, + searchedPaths: scriptCandidates, + }); +} export function buildSwiftToolEnv(env: NodeJS.ProcessEnv = process.env): NodeJS.ProcessEnv { const root = getSwiftCacheRoot(); From 0f825dd913670086d491e83a0011ea372298b70a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 22 Sep 2026 13:08:18 +0200 Subject: [PATCH 009/132] feat(recording): decode contact-sheet frames from an exported clip (#2744) --- .../RecordingExportSupport.swift | 3 + .../RecordingScripts/recording-frames.swift | 203 ++++++++++ .../capture-kit/src/png-pixels.fixtures.ts | 21 + .../__tests__/recording-scripts.test.ts | 31 ++ .../recording/contact-sheet-frames.test.ts | 363 ++++++++++++++++++ .../src/recording/contact-sheet-frames.ts | 278 ++++++++++++++ .../src/recording/contact-sheet-report.ts | 11 + .../src/recording/contact-sheet.fixtures.ts | 94 +++++ 8 files changed, 1004 insertions(+) create mode 100644 apple/runner/AgentDeviceRunner/RecordingScripts/recording-frames.swift create mode 100644 packages/capture-kit/src/png-pixels.fixtures.ts create mode 100644 packages/capture-kit/src/recording/contact-sheet-frames.test.ts create mode 100644 packages/capture-kit/src/recording/contact-sheet-frames.ts create mode 100644 packages/capture-kit/src/recording/contact-sheet-report.ts create mode 100644 packages/capture-kit/src/recording/contact-sheet.fixtures.ts diff --git a/apple/runner/AgentDeviceRunner/RecordingScripts/RecordingExportSupport.swift b/apple/runner/AgentDeviceRunner/RecordingScripts/RecordingExportSupport.swift index 5837d32ff4..8892f41584 100644 --- a/apple/runner/AgentDeviceRunner/RecordingScripts/RecordingExportSupport.swift +++ b/apple/runner/AgentDeviceRunner/RecordingScripts/RecordingExportSupport.swift @@ -9,11 +9,14 @@ enum RecordingScriptError: Error, CustomStringConvertible { case invalidArgs(String) case missingVideoTrack case exportFailed(String) + case frameWriteFailed(String) var description: String { switch self { case .invalidArgs(let message): return message + case .frameWriteFailed(let message): + return message case .missingVideoTrack: return "Input video does not contain a video track." case .exportFailed(let message): diff --git a/apple/runner/AgentDeviceRunner/RecordingScripts/recording-frames.swift b/apple/runner/AgentDeviceRunner/RecordingScripts/recording-frames.swift new file mode 100644 index 0000000000..2fc72fbcf2 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/RecordingScripts/recording-frames.swift @@ -0,0 +1,203 @@ +import AVFoundation +import Foundation +import ImageIO +import UniformTypeIdentifiers + +/// Extracts the frames at the requested presentation times from a recording and writes each one as +/// a PNG, then prints a JSON manifest naming the time each returned image actually carries. +/// +/// The caller owns the sampling grid: it passes an explicit, bounded list of times, so this script +/// never walks a whole clip. `--times` is the whole job description, which keeps the amount of +/// decoding, the bytes on disk, and the wall clock all bounded by the request rather than by the +/// recording's length. +/// +/// Reported times are the generator's *actual* return times, not the ones that were asked for. A +/// decoder answers a request with the frame it holds at that moment; the manifest says which frame +/// that was, so nothing downstream can print a requested time as though it had been decoded. + +struct FrameManifest: Encodable { + struct Frame: Encodable { + let index: Int + let requestedTimeMs: Double + let actualTimeMs: Double + let width: Int + let height: Int + let path: String + } + + struct Skipped: Encodable { + let index: Int + let requestedTimeMs: Double + let reason: String + } + + let inputPath: String + let durationMs: Double + let frames: [Frame] + let skipped: [Skipped] +} + +/// Entry point: `@main` because multi-file swiftc compilation reserves top-level statements for +/// `main.swift`, which cannot be shared per-script. +@main +enum RecordingFrames { + static func main() { + do { + try run() + } catch { + fputs("recording-frames: \(error)\n", stderr) + exit(1) + } + } +} + +func run() throws { + let arguments = Array(CommandLine.arguments.dropFirst()) + let inputURL = URL(fileURLWithPath: try requiredOption(arguments, "--input")) + let outputDirectoryURL = URL(fileURLWithPath: try requiredOption(arguments, "--output-dir"), isDirectory: true) + let requestedTimesMs = try parseTimes(try requiredOption(arguments, "--times")) + let maxWidth = try parseMaxWidth(optionValue(arguments, "--max-width")) + + guard !requestedTimesMs.isEmpty else { + throw RecordingScriptError.invalidArgs("--times must name at least one presentation time") + } + try FileManager.default.createDirectory(at: outputDirectoryURL, withIntermediateDirectories: true) + + let asset = AVURLAsset(url: inputURL) + _ = try sourceVideoTrack(of: asset) + let durationMs = max(0, asset.duration.seconds * 1000) + + let generator = AVAssetImageGenerator(asset: asset) + generator.appliesPreferredTrackTransform = true + generator.maximumSize = CGSize(width: maxWidth, height: maxWidth) + generator.requestedTimeToleranceBefore = .zero + generator.requestedTimeToleranceAfter = .zero + + var frames: [FrameManifest.Frame] = [] + var skipped: [FrameManifest.Skipped] = [] + + for (index, requestedTimeMs) in requestedTimesMs.enumerated() { + let requestedTime = CMTime(seconds: requestedTimeMs / 1000, preferredTimescale: 600) + var actualTime = CMTime.invalid + let cgImage: CGImage + do { + cgImage = try generator.copyCGImage(at: requestedTime, actualTime: &actualTime) + } catch { + skipped.append( + FrameManifest.Skipped( + index: index, + requestedTimeMs: requestedTimeMs, + reason: "\(error)" + ) + ) + continue + } + + // A frame the decoder cannot place on the timeline is not a frame at 0 ms. Writing it would + // label a mid-clip moment as the opening one, so the sample goes unanswered instead and the + // manifest says so. + guard CMTIME_IS_VALID(actualTime), actualTime.timescale != 0 else { + skipped.append( + FrameManifest.Skipped( + index: index, + requestedTimeMs: requestedTimeMs, + reason: "Decoder returned no presentation time for this frame" + ) + ) + continue + } + + let outputURL = outputDirectoryURL.appendingPathComponent(frameFileName(index: index)) + do { + try writePNG(cgImage: cgImage, to: outputURL) + } catch { + // A frame the decoder produced but storage refused is a failed run, not a moment the + // recording has nothing to show: reporting it as skipped would let a partial sheet claim + // to cover the clip. + try? FileManager.default.removeItem(at: outputURL) + throw RecordingScriptError.frameWriteFailed( + "Could not write frame \(index) at \(requestedTimeMs)ms to \(outputURL.path): \(error)" + ) + } + + frames.append( + FrameManifest.Frame( + index: index, + requestedTimeMs: requestedTimeMs, + actualTimeMs: timeIntervalMs(actualTime), + width: cgImage.width, + height: cgImage.height, + path: outputURL.path + ) + ) + } + + let manifest = FrameManifest( + inputPath: inputURL.path, + durationMs: durationMs, + frames: frames, + skipped: skipped + ) + let encoder = JSONEncoder() + encoder.outputFormatting = [.sortedKeys] + FileHandle.standardOutput.write(try encoder.encode(manifest)) +} + +func requiredOption(_ arguments: [String], _ flag: String) throws -> String { + guard let index = arguments.firstIndex(of: flag) else { + throw RecordingScriptError.invalidArgs("Missing \(flag)") + } + return try recordingOptionValue(arguments, index + 1, flag) +} + +func optionValue(_ arguments: [String], _ flag: String) -> String? { + guard let index = arguments.firstIndex(of: flag), index + 1 < arguments.count else { + return nil + } + return arguments[index + 1] +} + +func parseTimes(_ value: String) throws -> [Double] { + let parts = value.split(separator: ",", omittingEmptySubsequences: true) + guard !parts.isEmpty else { + throw RecordingScriptError.invalidArgs("--times must be a comma-separated list of milliseconds") + } + return try parts.map { part in + guard let parsed = Double(part), parsed.isFinite, parsed >= 0 else { + throw RecordingScriptError.invalidArgs("Invalid sample time: \(part)") + } + return parsed + } +} + +func parseMaxWidth(_ value: String?) throws -> Int { + guard let value else { return 360 } + guard let parsed = Int(value), parsed > 0 else { + throw RecordingScriptError.invalidArgs("Invalid --max-width: \(value)") + } + return parsed +} + +func timeIntervalMs(_ time: CMTime) -> Double { + guard CMTIME_IS_VALID(time), time.timescale != 0 else { return 0 } + return CMTimeGetSeconds(time) * 1000 +} + +func frameFileName(index: Int) -> String { + String(format: "frame-%04d.png", index) +} + +func writePNG(cgImage: CGImage, to url: URL) throws { + guard let destination = CGImageDestinationCreateWithURL( + url as CFURL, + UTType.png.identifier as CFString, + 1, + nil + ) else { + throw RecordingScriptError.exportFailed("Failed to open PNG destination for \(url.lastPathComponent)") + } + CGImageDestinationAddImage(destination, cgImage, nil) + guard CGImageDestinationFinalize(destination) else { + throw RecordingScriptError.exportFailed("Failed to encode PNG for \(url.lastPathComponent)") + } +} diff --git a/packages/capture-kit/src/png-pixels.fixtures.ts b/packages/capture-kit/src/png-pixels.fixtures.ts new file mode 100644 index 0000000000..5e8807d5bc --- /dev/null +++ b/packages/capture-kit/src/png-pixels.fixtures.ts @@ -0,0 +1,21 @@ +import { PNG } from './png.ts'; + +/** + * Decoded-frame builders for tests that reason about pixels: a solid fill is enough to name what a + * test claims a frame contains without shipping a PNG fixture through the repo. + */ + +export type Rgba = readonly [number, number, number, number]; + +export const BLACK: Rgba = [0, 0, 0, 255]; + +export function solidPng(width: number, height: number, color: Rgba = BLACK): PNG { + const png = new PNG({ width, height }); + for (let offset = 0; offset < png.data.length; offset += 4) { + png.data[offset] = color[0]; + png.data[offset + 1] = color[1]; + png.data[offset + 2] = color[2]; + png.data[offset + 3] = color[3]; + } + return png; +} diff --git a/packages/capture-kit/src/recording/__tests__/recording-scripts.test.ts b/packages/capture-kit/src/recording/__tests__/recording-scripts.test.ts index b7620408f7..466627d398 100644 --- a/packages/capture-kit/src/recording/__tests__/recording-scripts.test.ts +++ b/packages/capture-kit/src/recording/__tests__/recording-scripts.test.ts @@ -3,6 +3,9 @@ import assert from 'node:assert/strict'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { runCmd } from '@agent-device/host-kit/command'; +import { AppError } from '@agent-device/kernel/errors'; +import { CONTACT_SHEET_UNSUPPORTED_HOST_REASON } from '../contact-sheet-report.ts'; +import { assertContactSheetHostSupport } from '../contact-sheet-frames.ts'; import { getRecordingOverlaySupportWarning } from '../overlay.ts'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); @@ -71,6 +74,20 @@ test( SWIFT_TYPECHECK_TIMEOUT_MS, ); +test( + 'recording frames Swift script typechecks', + async (t) => { + if (process.platform !== 'darwin') { + t.skip('Swift recording scripts are only validated on macOS'); + } + + await assertSwiftScriptTypechecks(path.join(recordingScriptsDir, 'recording-frames.swift'), [ + path.join(recordingScriptsDir, 'RecordingExportSupport.swift'), + ]); + }, + SWIFT_TYPECHECK_TIMEOUT_MS, +); + test('recording overlays are explicitly unsupported on non-macOS hosts', () => { assert.equal( getRecordingOverlaySupportWarning('linux'), @@ -78,3 +95,17 @@ test('recording overlays are explicitly unsupported on non-macOS hosts', () => { ); assert.equal(getRecordingOverlaySupportWarning('darwin'), undefined); }); + +test('contact sheets are explicitly unsupported on non-macOS hosts', () => { + assert.throws( + () => assertContactSheetHostSupport('linux'), + (error: unknown) => { + return ( + error instanceof AppError && + error.code === 'UNSUPPORTED_OPERATION' && + error.details?.reason === CONTACT_SHEET_UNSUPPORTED_HOST_REASON + ); + }, + ); + assert.doesNotThrow(() => assertContactSheetHostSupport('darwin')); +}); diff --git a/packages/capture-kit/src/recording/contact-sheet-frames.test.ts b/packages/capture-kit/src/recording/contact-sheet-frames.test.ts new file mode 100644 index 0000000000..b8266c16ab --- /dev/null +++ b/packages/capture-kit/src/recording/contact-sheet-frames.test.ts @@ -0,0 +1,363 @@ +import { beforeEach, describe, expect, test, vi } from 'vitest'; +import path from 'node:path'; +import { + CONTACT_SHEET_EXTRACTION_REASON, + CONTACT_SHEET_NO_FRAMES_REASON, + CONTACT_SHEET_UNSUPPORTED_HOST_REASON, +} from './contact-sheet-report.ts'; +import { AppError, isRequestCanceledError } from '@agent-device/kernel/errors'; +import { mkdtempForTestSync } from '../tmp-dir.fixtures.ts'; +import { solidPng } from '../png-pixels.fixtures.ts'; +import { writeDecodedFrames, type StubFrame } from './contact-sheet.fixtures.ts'; +import { extractRecordingFrames } from './contact-sheet-frames.ts'; + +vi.mock(import('@agent-device/host-kit/command'), async (importOriginal) => ({ + ...(await importOriginal()), + runCmd: vi.fn(), +})); + +vi.mock(import('./swift-cache.ts'), async (importOriginal) => ({ + ...(await importOriginal()), + compileSwiftSourceFile: vi.fn(async () => '/cached/bin/recording-frames'), +})); + +import { runCmd } from '@agent-device/host-kit/command'; + +const mockRunCmd = vi.mocked(runCmd); +const scratchDir = mkdtempForTestSync('agent-device-contact-sheet-frames-'); +const CELL_WIDTH = 360; +const VIDEO = '/tmp/recording.mp4'; + +function reasonOf(action: () => Promise): Promise | unknown { + return action().then( + () => 'no error thrown', + (error: unknown) => (error instanceof AppError ? error.details?.reason : error), + ); +} + +/** Decoding is Apple tooling, so every case here states the host it is pretending to be. */ +function extract( + input: Omit[0], 'hostPlatform'>, +): Promise>> { + return extractRecordingFrames({ ...input, hostPlatform: 'darwin' }); +} + +function withTimes(args: readonly string[], times: string): string[] { + const rewritten = [...args]; + rewritten[rewritten.indexOf('--times') + 1] = times; + return rewritten; +} + +function answerWith(framesByRequestedTimeMs: ReadonlyMap) { + mockRunCmd.mockImplementation(async (_cmd, args) => + writeDecodedFrames({ args: args as string[], framesByRequestedTimeMs }), + ); +} + +describe('extractRecordingFrames', () => { + beforeEach(() => { + mockRunCmd.mockReset(); + }); + + test('refuses to decode at all on a host with no frame decoder', async () => { + expect( + await reasonOf(() => + extractRecordingFrames({ + videoPath: VIDEO, + scratchDir, + maxWidth: CELL_WIDTH, + timesMs: [0], + hostPlatform: 'linux', + }), + ), + ).toBe(CONTACT_SHEET_UNSUPPORTED_HOST_REASON); + // Refused before the compiler or the decoder ran, so a Linux host never pays for a spawn. + expect(mockRunCmd).not.toHaveBeenCalled(); + }); + + test('asks the helper for exactly the grid it was given', async () => { + answerWith( + new Map([ + [0, { png: solidPng(4, 4), actualTimeMs: 0 }], + [250, { png: solidPng(4, 4), actualTimeMs: 250 }], + ]), + ); + + await extract({ + videoPath: VIDEO, + scratchDir, + maxWidth: CELL_WIDTH, + timesMs: [0, 250], + }); + + const [, args] = mockRunCmd.mock.calls[0]!; + expect(args).toEqual([ + '--input', + VIDEO, + '--output-dir', + scratchDir, + '--times', + '0,250', + '--max-width', + '360', + ]); + }); + + test('reports the times the decoder actually returned', async () => { + answerWith( + new Map([ + // A decoder answers a request with the frame it holds, and out of request order is legal: + // the sheet must still read left to right in time. + [0, { png: solidPng(4, 4), actualTimeMs: 500 }], + [250, { png: solidPng(4, 4), actualTimeMs: 0 }], + [500, { png: solidPng(4, 4), actualTimeMs: 250 }], + ]), + ); + + const extracted = await extract({ + videoPath: VIDEO, + scratchDir, + maxWidth: CELL_WIDTH, + timesMs: [0, 250, 500], + }); + + expect(extracted.frames.map((frame) => frame.actualTimeMs)).toEqual([0, 250, 500]); + expect(extracted.frames.map((frame) => frame.requestedTimeMs)).toEqual([250, 500, 0]); + expect(extracted.skippedSampleCount).toBe(0); + }); + + test('counts the sample times the decoder declined', async () => { + answerWith(new Map([[0, { png: solidPng(4, 4), actualTimeMs: 0 }]])); + + const extracted = await extract({ + videoPath: VIDEO, + scratchDir, + maxWidth: CELL_WIDTH, + timesMs: [0, 250, 500], + }); + + expect(extracted.frames).toHaveLength(1); + expect(extracted.skippedSampleCount).toBe(2); + }); + + test('drops a frame the manifest names but the helper never wrote', async () => { + const written = new Map([ + [0, { png: solidPng(4, 4), actualTimeMs: 0 }], + [250, { png: solidPng(4, 4), actualTimeMs: 250 }], + ]); + mockRunCmd.mockImplementation(async (_cmd, args) => { + const result = writeDecodedFrames({ + args: args as string[], + framesByRequestedTimeMs: written, + }); + const manifest = JSON.parse(result.stdout) as { frames: { path: string }[] }; + manifest.frames[0]!.path = path.join(scratchDir, 'frame-9999.png'); + return { ...result, stdout: JSON.stringify(manifest) }; + }); + + const extracted = await extract({ + videoPath: VIDEO, + scratchDir, + maxWidth: CELL_WIDTH, + timesMs: [0, 250], + }); + + expect(extracted.frames.map((frame) => frame.actualTimeMs)).toEqual([250]); + // A frame the manifest promised and never delivered shrinks coverage, and the sheet has to say + // so rather than let one fewer cell look like a quieter recording. + expect(extracted.skippedSampleCount).toBe(1); + }); + + test('refuses a manifest that answers more samples than were asked for', async () => { + const answers = new Map([ + [0, { png: solidPng(4, 4), actualTimeMs: 0 }], + [250, { png: solidPng(4, 4), actualTimeMs: 250 }], + ]); + mockRunCmd.mockImplementation(async (_cmd, args) => + writeDecodedFrames({ + // The decoder was asked for one sample and hands back two: whatever those bytes are, they + // are not the grid this command budgeted to decode. + args: withTimes(args as string[], '0,250'), + framesByRequestedTimeMs: answers, + }), + ); + + expect( + await reasonOf(() => + extract({ + videoPath: VIDEO, + scratchDir, + maxWidth: CELL_WIDTH, + timesMs: [0], + }), + ), + ).toBe(CONTACT_SHEET_EXTRACTION_REASON); + }); + + test('refuses a frame entry that cannot say which frame it is', async () => { + mockRunCmd.mockResolvedValue({ + stdout: JSON.stringify({ + frames: [{ index: 0, path: 'frame-0.png', width: 4, height: 4 }], + skipped: [], + }), + stderr: '', + exitCode: 0, + }); + + expect( + await reasonOf(() => + extract({ + videoPath: VIDEO, + scratchDir, + maxWidth: CELL_WIDTH, + timesMs: [0], + }), + ), + ).toBe(CONTACT_SHEET_EXTRACTION_REASON); + }); + + test('counts samples the manifest neither answered nor declared skipped', async () => { + answerWith(new Map([[0, { png: solidPng(4, 4), actualTimeMs: 0 }]])); + + const extracted = await extract({ + videoPath: VIDEO, + scratchDir, + maxWidth: CELL_WIDTH, + timesMs: [0, 250, 500], + }); + + // A decoder that stays silent about the two samples it dropped still covered one of three, and + // a sheet may not report that as full coverage. + expect(extracted.frames).toHaveLength(1); + expect(extracted.skippedSampleCount).toBe(2); + }); + + test('keeps the extraction reason when the decoder cannot be built', async () => { + const { compileSwiftSourceFile } = await import('./swift-cache.ts'); + vi.mocked(compileSwiftSourceFile).mockRejectedValueOnce(new Error('swiftc died')); + + expect( + await reasonOf(() => + extract({ + videoPath: VIDEO, + scratchDir, + maxWidth: CELL_WIDTH, + timesMs: [0], + }), + ), + ).toBe(CONTACT_SHEET_EXTRACTION_REASON); + }); + + test('keeps the extraction reason when the decoder never started', async () => { + mockRunCmd.mockRejectedValue(new Error('spawn /cached/bin/recording-frames ENOENT')); + + expect( + await reasonOf(() => + extract({ + videoPath: VIDEO, + scratchDir, + maxWidth: CELL_WIDTH, + timesMs: [0], + }), + ), + ).toBe(CONTACT_SHEET_EXTRACTION_REASON); + }); + + test('hands the decoder the cancellation of the request it serves', async () => { + answerWith(new Map([[0, { png: solidPng(4, 4), actualTimeMs: 0 }]])); + const controller = new AbortController(); + + await extract({ + videoPath: VIDEO, + scratchDir, + maxWidth: CELL_WIDTH, + timesMs: [0], + signal: controller.signal, + }); + + expect(mockRunCmd.mock.calls[0]![2]).toMatchObject({ + allowFailure: true, + signal: controller.signal, + }); + }); + + test('answers a cancelled request as a cancellation, not an extraction failure', async () => { + const controller = new AbortController(); + controller.abort(); + + const error = await extract({ + videoPath: VIDEO, + scratchDir, + maxWidth: CELL_WIDTH, + timesMs: [0], + signal: controller.signal, + }).catch((error: unknown) => error); + + expect(isRequestCanceledError(error)).toBe(true); + expect(mockRunCmd).not.toHaveBeenCalled(); + }); + + test('reports a failed extraction run rather than an empty sheet', async () => { + answerWith(new Map()); + mockRunCmd.mockImplementation(async (_cmd, args) => + writeDecodedFrames({ + args: args as string[], + framesByRequestedTimeMs: new Map(), + failWithExitCode: 1, + }), + ); + + await expect( + extract({ videoPath: VIDEO, scratchDir, maxWidth: CELL_WIDTH, timesMs: [0] }), + ).rejects.toThrow(/could not open asset/); + expect( + await reasonOf(() => + extract({ + videoPath: VIDEO, + scratchDir, + maxWidth: CELL_WIDTH, + timesMs: [0], + }), + ), + ).toBe(CONTACT_SHEET_EXTRACTION_REASON); + }); + + test('reports an unreadable manifest as an extraction failure', async () => { + mockRunCmd.mockResolvedValue({ stdout: 'not json', stderr: '', exitCode: 0 }); + + expect( + await reasonOf(() => + extract({ + videoPath: VIDEO, + scratchDir, + maxWidth: CELL_WIDTH, + timesMs: [0], + }), + ), + ).toBe(CONTACT_SHEET_EXTRACTION_REASON); + }); + + test('reports a clip that returned nothing usable', async () => { + answerWith(new Map()); + + expect( + await reasonOf(() => + extract({ + videoPath: VIDEO, + scratchDir, + maxWidth: CELL_WIDTH, + timesMs: [0], + }), + ), + ).toBe(CONTACT_SHEET_NO_FRAMES_REASON); + }); + + test('refuses to spawn a decoder for an empty grid', async () => { + expect( + await reasonOf(() => + extract({ videoPath: VIDEO, scratchDir, maxWidth: CELL_WIDTH, timesMs: [] }), + ), + ).toBe(CONTACT_SHEET_NO_FRAMES_REASON); + expect(mockRunCmd).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/capture-kit/src/recording/contact-sheet-frames.ts b/packages/capture-kit/src/recording/contact-sheet-frames.ts new file mode 100644 index 0000000000..fd6f92de77 --- /dev/null +++ b/packages/capture-kit/src/recording/contact-sheet-frames.ts @@ -0,0 +1,278 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { + CONTACT_SHEET_EXTRACTION_REASON, + CONTACT_SHEET_NO_FRAMES_REASON, + CONTACT_SHEET_UNSUPPORTED_HOST_REASON, +} from './contact-sheet-report.ts'; +import { + AppError, + createRequestCanceledError, + errorMessage, + isRequestCanceledError, +} from '@agent-device/kernel/errors'; +import { runCmd } from '@agent-device/host-kit/command'; +import { findProjectRoot } from '@agent-device/host-kit/version'; +import { + buildSwiftToolEnv, + compileSwiftSourceFile, + resolveRecordingScriptPath, +} from './swift-cache.ts'; + +const FRAMES_SCRIPT = 'recording-frames.swift'; +const SHARED_SUPPORT_SCRIPT = 'RecordingExportSupport.swift'; +const EXTRACTION_TIMEOUT_MS = 90_000; +const COMPILATION_TIMEOUT_MS = 120_000; + +/** + * Frame decoding is Apple AVFoundation tooling, so a non-macOS host cannot decode a single frame at + * all. A caller that derives something from a sheet as a convenience catches this and keeps what it + * has; a caller that was asked for frames reports it. + */ +export function assertContactSheetHostSupport( + hostPlatform: NodeJS.Platform = process.platform, +): void { + if (hostPlatform === 'darwin') return; + throw new AppError( + 'UNSUPPORTED_OPERATION', + 'Contact sheets can only be built on macOS hosts, which is where the frame decoder runs', + { + reason: CONTACT_SHEET_UNSUPPORTED_HOST_REASON, + hostPlatform, + hint: 'Run this command on the macOS host that recorded the clip, or read the video artifact itself.', + }, + ); +} + +export type ExtractedRecordingFrame = Readonly<{ + /** The time that was asked for, in milliseconds from the clip start. */ + requestedTimeMs: number; + /** The time the decoder says the returned frame carries. */ + actualTimeMs: number; + path: string; +}>; + +export type ExtractedRecordingFrames = Readonly<{ + frames: readonly ExtractedRecordingFrame[]; + /** Requested sample times the decoder declined to answer. */ + skippedSampleCount: number; +}>; + +/** + * Decodes the frames at `timesMs` out of a recording into `scratchDir`, which the caller owns. + * + * Decoding is Apple AVFoundation tooling compiled through the same cached Swift seam the touch + * overlay burn-in uses, so a sheet costs one cached compile and one bounded decode pass rather than + * a new runtime dependency. + */ +export async function extractRecordingFrames( + input: Readonly<{ + videoPath: string; + scratchDir: string; + timesMs: readonly number[]; + /** Widest cell the caller will draw, so no frame is decoded wider than it can be shown. */ + maxWidth: number; + hostPlatform?: NodeJS.Platform; + signal?: AbortSignal; + }>, +): Promise { + // Decoding is the Apple tooling, so the host question has to be answered before this module + // spawns anything; a caller that only asks for frames gets the same reason as one that asked + // for a sheet. + assertContactSheetHostSupport(input.hostPlatform ?? process.platform); + if (input.timesMs.length === 0) { + throw new AppError('COMMAND_FAILED', 'Contact sheet sampling requested no frames', { + reason: CONTACT_SHEET_NO_FRAMES_REASON, + videoPath: input.videoPath, + }); + } + throwIfAborted(input.signal); + + const executablePath = await compileDecoder(input); + throwIfAborted(input.signal); + + let result; + try { + result = await runCmd( + executablePath, + [ + '--input', + input.videoPath, + '--output-dir', + input.scratchDir, + '--times', + input.timesMs.join(','), + '--max-width', + String(input.maxWidth), + ], + { + timeoutMs: EXTRACTION_TIMEOUT_MS, + env: buildSwiftToolEnv(), + allowFailure: true, + ...(input.signal === undefined ? {} : { signal: input.signal }), + }, + ); + } catch (error) { + throwExtractionFailure(error, input.videoPath, 'Failed to run the frame decoder'); + } + if (result.exitCode !== 0) { + throw new AppError( + 'COMMAND_FAILED', + `Failed to extract frames from the recording: ${lastNonEmptyLine(result.stderr) || `exit ${result.exitCode}`}`, + { + reason: CONTACT_SHEET_EXTRACTION_REASON, + videoPath: input.videoPath, + exitCode: result.exitCode, + }, + ); + } + + const extracted = parseExtractionManifest( + result.stdout, + input.videoPath, + input.scratchDir, + input.timesMs.length, + ); + if (extracted.frames.length === 0) { + throw new AppError( + 'COMMAND_FAILED', + 'The recording returned no decodable frames for a contact sheet', + { + reason: CONTACT_SHEET_NO_FRAMES_REASON, + videoPath: input.videoPath, + skippedSampleCount: extracted.skippedSampleCount, + }, + ); + } + return extracted; +} + +type FrameManifest = Readonly<{ + frames: readonly ExtractedRecordingFrame[]; + skippedSampleCount: number; +}>; + +function parseExtractionManifest( + stdout: string, + videoPath: string, + scratchDir: string, + requestedSampleCount: number, +): FrameManifest { + let decoded: unknown; + try { + decoded = JSON.parse(stdout); + } catch (error) { + throw extractionUnreadable(videoPath, `invalid manifest: ${String(error)}`); + } + + const frames = readRecord(decoded).frames; + if (!Array.isArray(frames)) { + throw extractionUnreadable(videoPath, 'manifest has no frames array'); + } + const skipped = readRecord(decoded).skipped; + + if (frames.length > requestedSampleCount) { + throw extractionUnreadable( + videoPath, + `returned ${frames.length} frames for ${requestedSampleCount} requested samples`, + ); + } + + const readable = frames.flatMap((entry) => { + const frame = readRecord(entry); + const name = stringField(frame.path); + const actualTimeMs = numberField(frame.actualTimeMs); + if (name === undefined || actualTimeMs === undefined || actualTimeMs < 0) { + // An entry that cannot say which frame it is cannot be shown or counted; the manifest itself + // is broken, which is a different claim from a sample time the decoder declined. + throw extractionUnreadable(videoPath, `frame entry is missing a path or presentation time`); + } + const framePath = path.resolve(scratchDir, path.basename(name)); + // Bytes that never arrived leave the sample unanswered, which the coverage shortfall below + // counts; a manifest that promised a frame and delivered nothing cannot be shown as a cell. + if (!fs.existsSync(framePath)) return []; + return [ + { + requestedTimeMs: numberField(frame.requestedTimeMs) ?? actualTimeMs, + actualTimeMs, + path: framePath, + }, + ]; + }); + + return { + // A decoder asked out of order answers out of order; the sheet reads left to right in time. + frames: readable.sort((left, right) => left.actualTimeMs - right.actualTimeMs), + // Coverage is what was asked for minus what arrived. Counting the shortfall rather than + // trusting the manifest's own skipped list keeps a decoder that answers nothing silently + // impossible: the sheet has to say it saw less than it asked for. + skippedSampleCount: Math.max( + Array.isArray(skipped) ? skipped.length : 0, + requestedSampleCount - readable.length, + ), + }; +} + +function extractionError(message: string, videoPath: string, cause?: unknown): AppError { + return new AppError( + 'COMMAND_FAILED', + message, + { reason: CONTACT_SHEET_EXTRACTION_REASON, videoPath }, + cause, + ); +} + +function extractionUnreadable(videoPath: string, detail: string): AppError { + return extractionError(`Frame extraction returned an unreadable manifest: ${detail}`, videoPath); +} + +/** + * A helper that could not start, was killed at its deadline, or never compiled is an extraction + * failure too; letting a raw process error escape would strand it outside the typed taxonomy. A + * canceled request and a failure that already names its reason are both answers the caller asked + * for, so they travel unchanged. + */ +function throwExtractionFailure(error: unknown, videoPath: string, stage: string): never { + if (isRequestCanceledError(error)) throw error; + if (error instanceof AppError && error.code === 'COMMAND_FAILED' && error.details?.reason) { + throw error; + } + throw extractionError(`${stage} for ${videoPath}: ${errorMessage(error)}`, videoPath, error); +} + +function readRecord(value: unknown): Record { + return typeof value === 'object' && value !== null ? (value as Record) : {}; +} + +function stringField(value: unknown): string | undefined { + return typeof value === 'string' && value !== '' ? value : undefined; +} + +function numberField(value: unknown): number | undefined { + return typeof value === 'number' && Number.isFinite(value) ? value : undefined; +} + +function lastNonEmptyLine(text: string): string { + return (text.trim().split('\n').pop() ?? '').trim(); +} + +/** + * Compiles the decoder helper, or fails as an extraction failure: a cold Swift cache that cannot + * build the helper is exactly as unable to produce a sheet as a helper that exits nonzero. + */ +async function compileDecoder(input: { videoPath: string; signal?: AbortSignal }): Promise { + try { + return await compileSwiftSourceFile({ + sourcePath: resolveRecordingScriptPath(FRAMES_SCRIPT, findProjectRoot()), + extraSourcePaths: [resolveRecordingScriptPath(SHARED_SUPPORT_SCRIPT, findProjectRoot())], + cacheName: 'recording-frames', + timeoutMs: COMPILATION_TIMEOUT_MS, + }); + } catch (error) { + throwExtractionFailure(error, input.videoPath, 'Could not build the frame decoder'); + } +} + +function throwIfAborted(signal: AbortSignal | undefined): void { + if (signal?.aborted) throw createRequestCanceledError(); +} diff --git a/packages/capture-kit/src/recording/contact-sheet-report.ts b/packages/capture-kit/src/recording/contact-sheet-report.ts new file mode 100644 index 0000000000..d0e5b6952c --- /dev/null +++ b/packages/capture-kit/src/recording/contact-sheet-report.ts @@ -0,0 +1,11 @@ +/** + * Why a contact sheet was not drawn. Each reason is a typed constant the extraction path fails with, + * so a caller branches on the reason instead of reading an error message. + */ + +/** The host cannot extract frames at all: frame decoding is Apple AVFoundation tooling. */ +export const CONTACT_SHEET_UNSUPPORTED_HOST_REASON = 'contact_sheet_unsupported_host'; +/** Frame extraction ran and failed, rather than returning fewer frames. */ +export const CONTACT_SHEET_EXTRACTION_REASON = 'contact_sheet_frame_extraction_failed'; +/** Extraction returned nothing usable, so there is no sheet to draw. */ +export const CONTACT_SHEET_NO_FRAMES_REASON = 'contact_sheet_no_frames'; diff --git a/packages/capture-kit/src/recording/contact-sheet.fixtures.ts b/packages/capture-kit/src/recording/contact-sheet.fixtures.ts new file mode 100644 index 0000000000..e808b01b66 --- /dev/null +++ b/packages/capture-kit/src/recording/contact-sheet.fixtures.ts @@ -0,0 +1,94 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { encodePngPixels } from '../png-encode.ts'; +import type { PNG } from '../png.ts'; + +/** + * The frame helper's side of a conversation, replayed against a scratch directory. Frames are + * written as the PNG bytes a real decoder hands back, so the pipeline decodes them the same way it + * decodes a simulator's recording. + */ + +/** A frame the fake decoder answers with: the pixels plus the time it claims to carry. */ +export type StubFrame = Readonly<{ png: PNG; actualTimeMs: number }>; + +export type StubDecoderResult = Readonly<{ + stdout: string; + stderr: string; + exitCode: number; +}>; + +/** + * Answers the frame helper's argv by writing PNG files where it was told to write them and + * returning the manifest the real helper prints. A requested time with no stub frame is reported + * as skipped, exactly as a decoder that declines a request is. + */ +export function writeDecodedFrames( + input: Readonly<{ + args: readonly string[]; + framesByRequestedTimeMs: ReadonlyMap; + failWithExitCode?: number; + manifest?: string; + }>, +): StubDecoderResult { + if (input.failWithExitCode !== undefined) { + return { + stdout: '', + stderr: 'recording-frames: could not open asset', + exitCode: input.failWithExitCode, + }; + } + if (input.manifest !== undefined) { + return { stdout: input.manifest, stderr: '', exitCode: 0 }; + } + + const outputDir = requireOption(input.args, '--output-dir'); + const requestedTimes = parseTimes(requireOption(input.args, '--times')); + const frames: unknown[] = []; + const skipped: unknown[] = []; + + requestedTimes.forEach((requestedTimeMs, index) => { + const frame = input.framesByRequestedTimeMs.get(requestedTimeMs); + if (!frame) { + skipped.push({ index, requestedTimeMs, reason: 'no frame at time' }); + return; + } + const filePath = path.join(outputDir, frameFileName(index)); + fs.writeFileSync( + filePath, + encodePngPixels(frame.png.data, frame.png.width, frame.png.height, 4), + ); + frames.push({ + index, + requestedTimeMs, + actualTimeMs: frame.actualTimeMs, + width: frame.png.width, + height: frame.png.height, + path: filePath, + }); + }); + + return { + stdout: JSON.stringify({ inputPath: 'fixture.mp4', durationMs: 0, frames, skipped }), + stderr: '', + exitCode: 0, + }; +} + +function frameFileName(index: number): string { + return `frame-${String(index).padStart(4, '0')}.png`; +} + +function requireOption(args: readonly string[], flag: string): string { + const index = args.indexOf(flag); + const value = index === -1 ? undefined : args[index + 1]; + if (!value) throw new Error(`fixture decoder was called without ${flag}`); + return value; +} + +function parseTimes(value: string): number[] { + return value + .split(',') + .filter((part) => part !== '') + .map(Number); +} From 6e765b6b9adb8e51a1149774f6fc56bee49a0e9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 22 Sep 2026 13:08:18 +0200 Subject: [PATCH 010/132] feat(recording): choose the frames a contact sheet shows (#2745) --- .../src/png-changed-pixel-ratio.test.ts | 47 ++++++++ .../src/png-changed-pixel-ratio.ts | 47 ++++++++ .../capture-kit/src/png-pixels.fixtures.ts | 38 +++++-- .../src/recording/contact-sheet-plan.test.ts | 69 ++++++++++++ .../src/recording/contact-sheet-plan.ts | 60 ++++++++++ .../src/recording/contact-sheet-report.ts | 2 + .../recording/contact-sheet-selection.test.ts | 98 +++++++++++++++++ .../src/recording/contact-sheet-selection.ts | 103 ++++++++++++++++++ 8 files changed, 456 insertions(+), 8 deletions(-) create mode 100644 packages/capture-kit/src/png-changed-pixel-ratio.test.ts create mode 100644 packages/capture-kit/src/png-changed-pixel-ratio.ts create mode 100644 packages/capture-kit/src/recording/contact-sheet-plan.test.ts create mode 100644 packages/capture-kit/src/recording/contact-sheet-plan.ts create mode 100644 packages/capture-kit/src/recording/contact-sheet-selection.test.ts create mode 100644 packages/capture-kit/src/recording/contact-sheet-selection.ts diff --git a/packages/capture-kit/src/png-changed-pixel-ratio.test.ts b/packages/capture-kit/src/png-changed-pixel-ratio.test.ts new file mode 100644 index 0000000000..24eb9b979e --- /dev/null +++ b/packages/capture-kit/src/png-changed-pixel-ratio.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, test } from 'vitest'; +import { computePngChangedPixelRatio } from './png-changed-pixel-ratio.ts'; +import { BLACK, paintPng, solidPng, WHITE } from './png-pixels.fixtures.ts'; + +function pixels(png: { width: number; height: number; data: Buffer }) { + return { width: png.width, height: png.height, data: png.data }; +} + +describe('computePngChangedPixelRatio', () => { + test('reports nothing moved for identical frames', () => { + expect( + computePngChangedPixelRatio(pixels(solidPng(4, 4, BLACK)), pixels(solidPng(4, 4, BLACK))), + ).toEqual({ status: 'compared', changedPixelRatio: 0 }); + }); + + test('counts a pixel whose blue channel alone moved', () => { + const blue = paintPng( + solidPng(4, 4, BLACK), + { x: 0, y: 0, width: 1, height: 1 }, + [0, 0, 9, 255], + ); + expect(computePngChangedPixelRatio(pixels(solidPng(4, 4, BLACK)), pixels(blue))).toEqual({ + status: 'compared', + changedPixelRatio: 1 / 16, + }); + }); + + test('ignores alpha, which never reaches the recorded screen', () => { + const transparent = solidPng(4, 4, BLACK); + transparent.data[3] = 0; + expect(computePngChangedPixelRatio(pixels(solidPng(4, 4, BLACK)), pixels(transparent))).toEqual( + { status: 'compared', changedPixelRatio: 0 }, + ); + }); + + test('counts every pixel when the whole frame moved', () => { + expect( + computePngChangedPixelRatio(pixels(solidPng(4, 4, BLACK)), pixels(solidPng(4, 4, WHITE))), + ).toEqual({ status: 'compared', changedPixelRatio: 1 }); + }); + + test('refuses a ratio across frames of different shapes', () => { + expect( + computePngChangedPixelRatio(pixels(solidPng(4, 4, BLACK)), pixels(solidPng(4, 8, BLACK))), + ).toEqual({ status: 'dimension_mismatch' }); + }); +}); diff --git a/packages/capture-kit/src/png-changed-pixel-ratio.ts b/packages/capture-kit/src/png-changed-pixel-ratio.ts new file mode 100644 index 0000000000..f7b0fa0ebd --- /dev/null +++ b/packages/capture-kit/src/png-changed-pixel-ratio.ts @@ -0,0 +1,47 @@ +import type { PngRgbImage } from './png-rgb-difference.ts'; + +export type { PngRgbImage }; + +export type PngChangedPixelRatioResult = + | { readonly status: 'compared'; readonly changedPixelRatio: number } + | { readonly status: 'dimension_mismatch' }; + +/** + * The share of pixels whose color moved at all between two decoded PNGs. + * + * A pixel counts as changed when any of its RGB channels differs, ignoring alpha, so a + * translucent-layer fade that leaves the composite untouched is not reported. This is a + * coverage metric, not a magnitude one: `computePngRgbDifference` answers how far the colors + * moved, and this answers how much of the frame moved. A recording that only shifts a 1 px + * progress line scores a ratio near zero either way. + * + * Dimensions must match. Callers comparing frames of different sizes are asking a different + * question, and returning a made-up ratio would hide it. + */ +export function computePngChangedPixelRatio( + first: PngRgbImage, + second: PngRgbImage, +): PngChangedPixelRatioResult { + if (first.width !== second.width || first.height !== second.height) { + return { status: 'dimension_mismatch' }; + } + + const totalPixels = first.width * first.height; + if (totalPixels === 0) return { status: 'compared', changedPixelRatio: 0 }; + if (first.data.length !== second.data.length) { + return { status: 'dimension_mismatch' }; + } + + let changedPixels = 0; + for (let offset = 0; offset + 3 < first.data.length; offset += 4) { + if ( + first.data[offset] !== second.data[offset] || + first.data[offset + 1] !== second.data[offset + 1] || + first.data[offset + 2] !== second.data[offset + 2] + ) { + changedPixels += 1; + } + } + + return { status: 'compared', changedPixelRatio: changedPixels / totalPixels }; +} diff --git a/packages/capture-kit/src/png-pixels.fixtures.ts b/packages/capture-kit/src/png-pixels.fixtures.ts index 5e8807d5bc..4833b55ad5 100644 --- a/packages/capture-kit/src/png-pixels.fixtures.ts +++ b/packages/capture-kit/src/png-pixels.fixtures.ts @@ -1,21 +1,43 @@ import { PNG } from './png.ts'; /** - * Decoded-frame builders for tests that reason about pixels: a solid fill is enough to name what a - * test claims a frame contains without shipping a PNG fixture through the repo. + * Decoded-frame builders for tests that reason about pixels: a solid fill and one painted rectangle + * name exactly what a test claims changed between two frames. */ export type Rgba = readonly [number, number, number, number]; +export type Rectangle = Readonly<{ x: number; y: number; width: number; height: number }>; export const BLACK: Rgba = [0, 0, 0, 255]; +export const WHITE: Rgba = [255, 255, 255, 255]; +export const RED: Rgba = [255, 0, 0, 255]; export function solidPng(width: number, height: number, color: Rgba = BLACK): PNG { - const png = new PNG({ width, height }); - for (let offset = 0; offset < png.data.length; offset += 4) { - png.data[offset] = color[0]; - png.data[offset + 1] = color[1]; - png.data[offset + 2] = color[2]; - png.data[offset + 3] = color[3]; + return fillPng(new PNG({ width, height }), () => true, color); +} + +/** Paints one rectangle of `color` onto a copy of `source`, leaving the source untouched. */ +export function paintPng(source: PNG, rectangle: Rectangle, color: Rgba): PNG { + const copy = solidPng(source.width, source.height); + source.data.copy(copy.data); + const inside = (column: number, row: number) => + column >= rectangle.x && + column < rectangle.x + rectangle.width && + row >= rectangle.y && + row < rectangle.y + rectangle.height; + return fillPng(copy, inside, color); +} + +function fillPng(png: PNG, paint: (column: number, row: number) => boolean, color: Rgba): PNG { + for (let row = 0; row < png.height; row += 1) { + for (let column = 0; column < png.width; column += 1) { + if (!paint(column, row)) continue; + const offset = (row * png.width + column) * 4; + png.data[offset] = color[0]; + png.data[offset + 1] = color[1]; + png.data[offset + 2] = color[2]; + png.data[offset + 3] = color[3]; + } } return png; } diff --git a/packages/capture-kit/src/recording/contact-sheet-plan.test.ts b/packages/capture-kit/src/recording/contact-sheet-plan.test.ts new file mode 100644 index 0000000000..c3c0ed5bc9 --- /dev/null +++ b/packages/capture-kit/src/recording/contact-sheet-plan.test.ts @@ -0,0 +1,69 @@ +import { describe, expect, test } from 'vitest'; +import { CONTACT_SHEET_DURATION_REASON } from './contact-sheet-report.ts'; +import { AppError } from '@agent-device/kernel/errors'; +import { + CONTACT_SHEET_SAMPLE_INTERVAL_MS, + MAX_CONTACT_SHEET_SAMPLED_FRAMES, + planContactSheetSampleTimes, +} from './contact-sheet-plan.ts'; + +const VIDEO = '/tmp/recording.mp4'; + +function reasonOf(action: () => unknown): unknown { + try { + action(); + } catch (error) { + return error instanceof AppError ? error.details?.reason : `not an AppError: ${String(error)}`; + } + return 'no error thrown'; +} + +describe('planContactSheetSampleTimes', () => { + test('names both endpoints of a clip no longer than one sample step', () => { + expect(planContactSheetSampleTimes(0, VIDEO)).toEqual({ durationMs: 0, timesMs: [0] }); + // One step is not one frame: a clip can change between its first and last presentation sample, + // and the sheet promises its ending whatever the clip's length. + expect(planContactSheetSampleTimes(CONTACT_SHEET_SAMPLE_INTERVAL_MS, VIDEO)).toEqual({ + durationMs: CONTACT_SHEET_SAMPLE_INTERVAL_MS, + timesMs: [0, CONTACT_SHEET_SAMPLE_INTERVAL_MS], + }); + expect(planContactSheetSampleTimes(100, VIDEO)).toEqual({ durationMs: 100, timesMs: [0, 100] }); + }); + + test('samples every step while the clip is shorter than the cap', () => { + expect(planContactSheetSampleTimes(1_000, VIDEO).timesMs).toEqual([0, 250, 500, 750, 1000]); + }); + + test('stretches the same grid over a long clip instead of adding samples', () => { + const hour = planContactSheetSampleTimes(3_600_000, VIDEO); + + expect(hour.timesMs).toHaveLength(MAX_CONTACT_SHEET_SAMPLED_FRAMES); + expect(hour.timesMs[0]).toBe(0); + expect(hour.timesMs.at(-1)).toBe(3_600_000); + expect(new Set(hour.timesMs).size).toBe(MAX_CONTACT_SHEET_SAMPLED_FRAMES); + hour.timesMs.forEach((time, index) => { + if (index > 0) expect(time).toBeGreaterThan(hour.timesMs[index - 1]!); + }); + }); + + test('keeps both endpoints, which is what a coverage claim rests on', () => { + for (const durationMs of [5_000, 41_000, 900_000, 7_200_000]) { + const plan = planContactSheetSampleTimes(durationMs, VIDEO); + expect(plan.durationMs).toBe(durationMs); + expect(plan.timesMs[0]).toBe(0); + expect(plan.timesMs.at(-1)).toBe(durationMs); + } + }); + + test('refuses to plan over a timeline the container cannot name', () => { + expect(reasonOf(() => planContactSheetSampleTimes(undefined, VIDEO))).toBe( + CONTACT_SHEET_DURATION_REASON, + ); + expect(reasonOf(() => planContactSheetSampleTimes(Number.NaN, VIDEO))).toBe( + CONTACT_SHEET_DURATION_REASON, + ); + expect(reasonOf(() => planContactSheetSampleTimes(-1, VIDEO))).toBe( + CONTACT_SHEET_DURATION_REASON, + ); + }); +}); diff --git a/packages/capture-kit/src/recording/contact-sheet-plan.ts b/packages/capture-kit/src/recording/contact-sheet-plan.ts new file mode 100644 index 0000000000..5a1c79db37 --- /dev/null +++ b/packages/capture-kit/src/recording/contact-sheet-plan.ts @@ -0,0 +1,60 @@ +import { CONTACT_SHEET_DURATION_REASON } from './contact-sheet-report.ts'; +import { AppError } from '@agent-device/kernel/errors'; + +/** Spacing between requested sample times, in milliseconds. */ +export const CONTACT_SHEET_SAMPLE_INTERVAL_MS = 250; +/** Sample times one sheet asks the decoder for, however long the clip runs. */ +export const MAX_CONTACT_SHEET_SAMPLED_FRAMES = 48; + +/** + * The times to sample from a clip of `durationMs`, spread evenly across the whole timeline. + * + * Coverage is the point: the grid always names the start and the end, and it never grows with the + * recording. A 5s take samples 21 times; a 2h take samples the same 48 times across its length, so + * a long recording costs the same decode budget as a short one and still shows both endpoints. + * + * A grid is a promise with a hole in it. A flash that opens and closes entirely between two sample + * times is not in the returned frames and no threshold can recover it; that is why the sheet + * reports how many times it sampled rather than claiming to have reviewed the footage. + * + * A clip whose timeline cannot be read is refused rather than guessed at. Sampling an unknown + * length would mean either walking the whole file or drawing a grid over a duration the container + * never claimed, and a sheet built that way could not say what it covered. + */ +export type ContactSheetSamplePlan = Readonly<{ + /** Timeline the grid was planned over, in milliseconds. */ + durationMs: number; + timesMs: readonly number[]; +}>; + +export function planContactSheetSampleTimes( + durationMs: number | undefined, + videoPath: string, +): ContactSheetSamplePlan { + if (durationMs === undefined || !Number.isFinite(durationMs) || durationMs < 0) { + throw new AppError( + 'COMMAND_FAILED', + `Cannot plan a contact sheet: the video timeline of ${videoPath} could not be read`, + { + reason: CONTACT_SHEET_DURATION_REASON, + videoPath, + hint: 'Retry once the recording finished finalizing; a clip whose container is still being written has no readable duration.', + }, + ); + } + if (durationMs <= CONTACT_SHEET_SAMPLE_INTERVAL_MS) { + // A clip shorter than one sampling interval still ends somewhere, and the sheet always promises + // its last frame, so the closing sample is asked for even when it is the only other one. + return { durationMs, timesMs: durationMs > 0 ? [0, Math.round(durationMs)] : [0] }; + } + + const count = Math.min( + MAX_CONTACT_SHEET_SAMPLED_FRAMES, + Math.floor(durationMs / CONTACT_SHEET_SAMPLE_INTERVAL_MS) + 1, + ); + const step = durationMs / (count - 1); + return { + durationMs, + timesMs: Array.from({ length: count }, (_, index) => Math.round(index * step)), + }; +} diff --git a/packages/capture-kit/src/recording/contact-sheet-report.ts b/packages/capture-kit/src/recording/contact-sheet-report.ts index d0e5b6952c..fb872f89ff 100644 --- a/packages/capture-kit/src/recording/contact-sheet-report.ts +++ b/packages/capture-kit/src/recording/contact-sheet-report.ts @@ -5,6 +5,8 @@ /** The host cannot extract frames at all: frame decoding is Apple AVFoundation tooling. */ export const CONTACT_SHEET_UNSUPPORTED_HOST_REASON = 'contact_sheet_unsupported_host'; +/** The MP4 timeline could not be read, so no bounded sample grid can be planned. */ +export const CONTACT_SHEET_DURATION_REASON = 'contact_sheet_duration_unknown'; /** Frame extraction ran and failed, rather than returning fewer frames. */ export const CONTACT_SHEET_EXTRACTION_REASON = 'contact_sheet_frame_extraction_failed'; /** Extraction returned nothing usable, so there is no sheet to draw. */ diff --git a/packages/capture-kit/src/recording/contact-sheet-selection.test.ts b/packages/capture-kit/src/recording/contact-sheet-selection.test.ts new file mode 100644 index 0000000000..69791ef8b9 --- /dev/null +++ b/packages/capture-kit/src/recording/contact-sheet-selection.test.ts @@ -0,0 +1,98 @@ +import { describe, expect, test } from 'vitest'; +import { paintPng, RED, solidPng } from '../png-pixels.fixtures.ts'; +import { + MAX_CONTACT_SHEET_CELLS, + selectContactSheetCells, + type ContactSheetSample, +} from './contact-sheet-selection.ts'; + +const FRAME_SIZE = 10; + +function sample(timeMs: number, changedPixels: number, offset = 0): ContactSheetSample { + return { + timeMs, + image: paintPng( + solidPng(FRAME_SIZE, FRAME_SIZE), + { x: offset, y: 0, width: changedPixels, height: 1 }, + RED, + ), + }; +} + +function distinctSample(index: number): ContactSheetSample { + // Frames of a different shape always count as fully changed, which keeps this fixture honest + // without asking it to invent pixels a video decoder would have had to produce. + return { timeMs: index * 100, image: solidPng(FRAME_SIZE + index, FRAME_SIZE) }; +} + +describe('selectContactSheetCells', () => { + test('returns nothing for no samples', () => { + expect(selectContactSheetCells([])).toEqual({ cells: [], keptCellCount: 0, thinned: false }); + }); + + test('keeps the only frame it was given', () => { + const selection = selectContactSheetCells([sample(0, 0)]); + + expect(selection.cells).toHaveLength(1); + expect(selection.cells[0]).toMatchObject({ timeMs: 0, changedPixelRatio: 1 }); + expect(selection.thinned).toBe(false); + }); + + test('ends with the final frame even when nothing visibly moved', () => { + const selection = selectContactSheetCells([sample(0, 0), sample(250, 0), sample(500, 0)], 0.04); + + expect(selection.cells.map((cell) => cell.timeMs)).toEqual([0, 500]); + expect(selection.cells.at(-1)?.changedPixelRatio).toBe(0); + }); + + test('measures against the last kept cell so small changes add up', () => { + const selection = selectContactSheetCells( + [ + sample(0, 0), + // 1% of the frame: too small on its own, and it does not become the baseline. + sample(250, 1), + // 10% of the frame, but only 9% against the previous sample. + sample(500, FRAME_SIZE), + ], + 0.1, + ); + + expect(selection.cells.map((cell) => cell.timeMs)).toEqual([0, 500]); + expect(selection.cells[1]?.changedPixelRatio).toBeCloseTo(0.1, 10); + }); + + test('keeps a frame whose shape changed', () => { + const selection = selectContactSheetCells( + [sample(0, 0), { timeMs: 250, image: solidPng(20, FRAME_SIZE) }], + 0.5, + ); + + expect(selection.cells.map((cell) => cell.timeMs)).toEqual([0, 250]); + expect(selection.cells[1]?.changedPixelRatio).toBe(1); + }); + + test('thins evenly across the kept sequence and keeps both ends', () => { + const samples = Array.from({ length: 30 }, (_, index) => distinctSample(index)); + + const selection = selectContactSheetCells(samples, 0.04, 4); + + expect(selection.keptCellCount).toBe(30); + expect(selection.thinned).toBe(true); + expect(selection.cells).toHaveLength(4); + expect(selection.cells[0]?.timeMs).toBe(0); + expect(selection.cells.at(-1)?.timeMs).toBe(2_900); + const times = selection.cells.map((cell) => cell.timeMs); + expect(new Set(times).size).toBe(times.length); + }); + + test('prints every kept cell while the count stays under the sheet cap', () => { + const samples = Array.from({ length: MAX_CONTACT_SHEET_CELLS }, (_, index) => + distinctSample(index), + ); + + const selection = selectContactSheetCells(samples); + + expect(selection.thinned).toBe(false); + expect(selection.cells).toHaveLength(MAX_CONTACT_SHEET_CELLS); + }); +}); diff --git a/packages/capture-kit/src/recording/contact-sheet-selection.ts b/packages/capture-kit/src/recording/contact-sheet-selection.ts new file mode 100644 index 0000000000..18ae835bec --- /dev/null +++ b/packages/capture-kit/src/recording/contact-sheet-selection.ts @@ -0,0 +1,103 @@ +import { computePngChangedPixelRatio, type PngRgbImage } from '../png-changed-pixel-ratio.ts'; + +/** + * Which sampled frames earn a cell of their own. + * + * A cell is kept when it moves enough of the frame relative to the cell kept *before* it, not + * relative to its immediate predecessor. Comparing with the previous kept cell is what lets small + * changes add up: a list that scrolls one row per frame keeps nothing against its own predecessor +" * and everything against the last frame that was shown. + * + * Calibrated on iOS simulator recordings at this sheet's 360px comparison width: an idle status-bar + * clock tick measures up to 0.009, while the smallest change worth a cell — a pane arriving over + * unchanged chrome — measured 0.030. Any value in that gap separates the two, and this one sits + * nearer the noise so a quiet change costs an extra cell instead of going unshown. + */ +const CONTACT_SHEET_CHANGED_PIXEL_THRESHOLD = 0.02; + +/** Cells one sheet prints. The grid thins evenly across time to hold this, keeping both ends. */ +export const MAX_CONTACT_SHEET_CELLS = 24; + +export type ContactSheetSample = Readonly<{ + /** Presentation time the decoder reported for this frame, in milliseconds from the clip start. */ + timeMs: number; + image: PngRgbImage; +}>; + +export type ContactSheetCell = Readonly<{ + timeMs: number; + changedPixelRatio: number; + image: PngRgbImage; +}>; + +export type ContactSheetSelection = Readonly<{ + cells: readonly ContactSheetCell[]; + /** Cells the rule kept before the grid was thinned to `MAX_CONTACT_SHEET_CELLS`. */ + keptCellCount: number; + /** Whether thinning dropped kept cells, which the sheet discloses rather than hides. */ + thinned: boolean; +}>; + +export function selectContactSheetCells( + samples: readonly ContactSheetSample[], + threshold: number = CONTACT_SHEET_CHANGED_PIXEL_THRESHOLD, + maxCells: number = MAX_CONTACT_SHEET_CELLS, +): ContactSheetSelection { + const kept: ContactSheetCell[] = []; + let baseline: PngRgbImage | undefined; + + for (const sample of samples) { + const changedPixelRatio = baseline + ? changedRatioAgainst(baseline, sample.image) + : // The first frame has nothing to differ from; it establishes the sheet's starting state. + 1; + if (changedPixelRatio < threshold) continue; + kept.push({ timeMs: sample.timeMs, changedPixelRatio, image: sample.image }); + baseline = sample.image; + } + + keepFinalState(kept, samples); + + const cells = kept.length > maxCells ? thinEvenly(kept, maxCells) : kept; + return { + cells, + keptCellCount: kept.length, + thinned: cells.length < kept.length, + }; +} + +/** + * Ends the sheet with the last frame the decoder returned. + * + * The rule alone would drop a clip that settles quietly: a screen that fades rather than moves + * keeps only its opening cell, and the caller is left with no picture of where the recording + * ended — which is the state a caller most often wants checked. The endpoint is appended with the + * ratio it actually measured, so a cell that is there for coverage rather than change still says + * so. + */ +function keepFinalState(kept: ContactSheetCell[], samples: readonly ContactSheetSample[]): void { + const final = samples.at(-1); + if (!final) return; + const previous = kept.at(-1); + if (!previous || previous.timeMs === final.timeMs) return; + kept.push({ + timeMs: final.timeMs, + changedPixelRatio: changedRatioAgainst(previous.image, final.image), + image: final.image, + }); +} + +function changedRatioAgainst(baseline: PngRgbImage, candidate: PngRgbImage): number { + const result = computePngChangedPixelRatio(baseline, candidate); + // A frame that changed shape reshaped the whole picture, which is the largest change there is. + return result.status === 'compared' ? result.changedPixelRatio : 1; +} + +function thinEvenly( + kept: readonly ContactSheetCell[], + maxCells: number, +): readonly ContactSheetCell[] { + if (maxCells <= 1) return [kept.at(-1)!]; + const stride = (kept.length - 1) / (maxCells - 1); + return Array.from({ length: maxCells }, (_, index) => kept[Math.round(index * stride)]!); +} From d8d3b35640ebb4651033de8ebc5078a3a0e59c51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 22 Sep 2026 13:08:19 +0200 Subject: [PATCH 011/132] feat(capture-kit): paint the contact-sheet grid onto one PNG (#2746) --- .../capture-kit/src/png-pixels.fixtures.ts | 10 +- packages/capture-kit/src/png-resize.ts | 2 +- .../src/recording/contact-sheet-plan.ts | 2 + .../recording/contact-sheet-render.test.ts | 88 ++++++++ .../src/recording/contact-sheet-render.ts | 205 ++++++++++++++++++ .../src/recording/contact-sheet-report.ts | 2 + .../src/screenshot-overlay-draw.test.ts | 64 ++++++ .../src/screenshot-overlay-draw.ts | 157 ++++++++------ 8 files changed, 461 insertions(+), 69 deletions(-) create mode 100644 packages/capture-kit/src/recording/contact-sheet-render.test.ts create mode 100644 packages/capture-kit/src/recording/contact-sheet-render.ts create mode 100644 packages/capture-kit/src/screenshot-overlay-draw.test.ts diff --git a/packages/capture-kit/src/png-pixels.fixtures.ts b/packages/capture-kit/src/png-pixels.fixtures.ts index 4833b55ad5..ef525331f0 100644 --- a/packages/capture-kit/src/png-pixels.fixtures.ts +++ b/packages/capture-kit/src/png-pixels.fixtures.ts @@ -1,11 +1,12 @@ import { PNG } from './png.ts'; +import { setPngPixel, type PngGlyphColor } from './screenshot-overlay-draw.ts'; /** * Decoded-frame builders for tests that reason about pixels: a solid fill and one painted rectangle * name exactly what a test claims changed between two frames. */ -export type Rgba = readonly [number, number, number, number]; +export type Rgba = PngGlyphColor; export type Rectangle = Readonly<{ x: number; y: number; width: number; height: number }>; export const BLACK: Rgba = [0, 0, 0, 255]; @@ -31,12 +32,7 @@ export function paintPng(source: PNG, rectangle: Rectangle, color: Rgba): PNG { function fillPng(png: PNG, paint: (column: number, row: number) => boolean, color: Rgba): PNG { for (let row = 0; row < png.height; row += 1) { for (let column = 0; column < png.width; column += 1) { - if (!paint(column, row)) continue; - const offset = (row * png.width + column) * 4; - png.data[offset] = color[0]; - png.data[offset + 1] = color[1]; - png.data[offset + 2] = color[2]; - png.data[offset + 3] = color[3]; + if (paint(column, row)) setPngPixel(png, column, row, color); } } return png; diff --git a/packages/capture-kit/src/png-resize.ts b/packages/capture-kit/src/png-resize.ts index 7bbab74696..f759956514 100644 --- a/packages/capture-kit/src/png-resize.ts +++ b/packages/capture-kit/src/png-resize.ts @@ -39,7 +39,7 @@ export async function resizePngFile( await fs.writeFile(filePath, await encodePngAsync(resizePngBox(source, width, height))); } -function resizePngBox(source: PNG, width: number, height: number): PNG { +export function resizePngBox(source: PNG, width: number, height: number): PNG { const output = new PNG({ width, height }); for (let y = 0; y < height; y += 1) { const sourceTop = (y * source.height) / height; diff --git a/packages/capture-kit/src/recording/contact-sheet-plan.ts b/packages/capture-kit/src/recording/contact-sheet-plan.ts index 5a1c79db37..049a7c21f4 100644 --- a/packages/capture-kit/src/recording/contact-sheet-plan.ts +++ b/packages/capture-kit/src/recording/contact-sheet-plan.ts @@ -5,6 +5,8 @@ import { AppError } from '@agent-device/kernel/errors'; export const CONTACT_SHEET_SAMPLE_INTERVAL_MS = 250; /** Sample times one sheet asks the decoder for, however long the clip runs. */ export const MAX_CONTACT_SHEET_SAMPLED_FRAMES = 48; +/** Width the decoder is asked to return frames at, which is also the widest cell the sheet draws. */ +export const CONTACT_SHEET_FRAME_WIDTH = 360; /** * The times to sample from a clip of `durationMs`, spread evenly across the whole timeline. diff --git a/packages/capture-kit/src/recording/contact-sheet-render.test.ts b/packages/capture-kit/src/recording/contact-sheet-render.test.ts new file mode 100644 index 0000000000..a7086e4945 --- /dev/null +++ b/packages/capture-kit/src/recording/contact-sheet-render.test.ts @@ -0,0 +1,88 @@ +import { describe, expect, test } from 'vitest'; +import { AppError } from '@agent-device/kernel/errors'; +import { BLACK, solidPng, WHITE } from '../png-pixels.fixtures.ts'; +import { decodePng } from '../png.ts'; +import { CONTACT_SHEET_FRAME_WIDTH } from './contact-sheet-plan.ts'; +import { + MIN_CONTACT_SHEET_CELL_WIDTH, + formatContactSheetTimestamp, + renderContactSheet, +} from './contact-sheet-render.ts'; +import type { ContactSheetCell } from './contact-sheet-selection.ts'; + +function cell(timeMs: number, width = 20, height = 10): ContactSheetCell { + return { timeMs, changedPixelRatio: 1, image: solidPng(width, height, BLACK) }; +} + +function countPixels(png: ReturnType, color: readonly number[]): number { + let count = 0; + for (let offset = 0; offset < png.data.length; offset += 4) { + if ( + png.data[offset] === color[0] && + png.data[offset + 1] === color[1] && + png.data[offset + 2] === color[2] + ) { + count += 1; + } + } + return count; +} + +describe('formatContactSheetTimestamp', () => { + test('prints elapsed time the glyph table can spell', () => { + expect(formatContactSheetTimestamp(0)).toBe('00:00:00.000'); + expect(formatContactSheetTimestamp(1_234)).toBe('00:00:01.234'); + expect(formatContactSheetTimestamp(3_723_456)).toBe('01:02:03.456'); + expect(formatContactSheetTimestamp(3_600_000)).toBe('01:00:00.000'); + }); +}); + +describe('renderContactSheet', () => { + test('encodes one grid whose bytes carry the reported size', () => { + const sheet = renderContactSheet({ + cells: [cell(0), cell(250)], + maxPixels: 20_000_000, + }); + + const decoded = decodePng(sheet.bytes, 'contact sheet'); + expect([decoded.width, decoded.height]).toEqual([sheet.width, sheet.height]); + // Two cells in one row of four columns: padding, two cells, and the gap between them. + expect(sheet.width).toBe(CONTACT_SHEET_FRAME_WIDTH * 2 + 8 * 3); + }); + + test('burns a timestamp label above every cell', () => { + const sheet = renderContactSheet({ + cells: [cell(0), cell(250), cell(500)], + maxPixels: 20_000_000, + }); + + const decoded = decodePng(sheet.bytes, 'contact sheet'); + expect(countPixels(decoded, WHITE)).toBeGreaterThan(0); + }); + + test('shrinks cells to stay inside the caller’s pixel budget', () => { + const sheet = renderContactSheet({ + cells: Array.from({ length: 12 }, (_, index) => cell(index * 250, 80, 40)), + maxPixels: 250_000, + }); + + expect(sheet.width * sheet.height).toBeLessThanOrEqual(250_000); + expect(sheet.cellWidth).toBeLessThan(CONTACT_SHEET_FRAME_WIDTH); + expect(sheet.cellWidth).toBeGreaterThanOrEqual(MIN_CONTACT_SHEET_CELL_WIDTH); + }); + + test('refuses rather than printing a sheet nobody can read', () => { + expect(() => + renderContactSheet({ cells: [cell(0, 40, 400), cell(250, 40, 400)], maxPixels: 4_000 }), + ).toThrow(/maxImagePixels/); + }); + + test('refuses a sheet with no cells instead of writing an empty page', () => { + try { + renderContactSheet({ cells: [], maxPixels: 20_000_000 }); + throw new Error('expected renderContactSheet to refuse'); + } catch (error) { + expect(error).toBeInstanceOf(AppError); + } + }); +}); diff --git a/packages/capture-kit/src/recording/contact-sheet-render.ts b/packages/capture-kit/src/recording/contact-sheet-render.ts new file mode 100644 index 0000000000..d6c0827c03 --- /dev/null +++ b/packages/capture-kit/src/recording/contact-sheet-render.ts @@ -0,0 +1,205 @@ +import { CONTACT_SHEET_PIXEL_BUDGET_REASON } from './contact-sheet-report.ts'; +import { AppError } from '@agent-device/kernel/errors'; +import { encodePngPixels } from '../png-encode.ts'; +import { PNG } from '../png.ts'; +import { resizePngBox } from '../png-resize.ts'; +import { + drawPngGlyphText, + measurePngGlyphTextHeight, + setPngPixel, + type PngGlyphColor, +} from '../screenshot-overlay-draw.ts'; +import { CONTACT_SHEET_FRAME_WIDTH } from './contact-sheet-plan.ts'; +import type { ContactSheetCell } from './contact-sheet-selection.ts'; + +/** Cells per row. Four keeps a portrait phone frame readable without a canvas wider than a screen. */ +const CONTACT_SHEET_COLUMNS = 4; +/** Narrowest a cell may shrink to before the sheet refuses rather than print unreadable frames. */ +export const MIN_CONTACT_SHEET_CELL_WIDTH = 160; +/** + * Cell widths tried, widest first. A cell never grows past the width frames were decoded at, and it + * shrinks in readable steps before the sheet refuses. + */ +const LAYOUT_CELL_WIDTHS = [ + CONTACT_SHEET_FRAME_WIDTH, + 320, + 280, + 240, + 200, + MIN_CONTACT_SHEET_CELL_WIDTH, +] as const; + +const SHEET_PADDING = 8; +const CELL_GAP = 8; +const LABEL_INSET = 4; +const LABEL_SCALE = 2; +const LABEL_COLOR = [255, 255, 255, 255] as const; +const SHEET_BACKGROUND = [17, 24, 39, 255] as const; +const LABEL_HEIGHT = measurePngGlyphTextHeight(LABEL_SCALE) + LABEL_INSET * 2; + +export type ContactSheetRenderInput = Readonly<{ + cells: readonly ContactSheetCell[]; + maxPixels: number; +}>; + +export type ContactSheetRenderResult = Readonly<{ + bytes: Buffer; + width: number; + height: number; + cellWidth: number; +}>; + +/** Lays the kept cells out in one row-major grid and encodes it as PNG. */ +export function renderContactSheet(input: ContactSheetRenderInput): ContactSheetRenderResult { + const { cells } = input; + if (cells.length === 0) { + throw new AppError('COMMAND_FAILED', 'Contact sheet has no cells to render'); + } + + const aspect = cells[0]!.image.height / Math.max(1, cells[0]!.image.width); + const layout = fitLayout(aspect, cells.length, input); + const canvas = createCanvas(layout.width, layout.height, SHEET_BACKGROUND); + + cells.forEach((cell, index) => { + const column = index % layout.columns; + const row = Math.floor(index / layout.columns); + drawCell( + canvas, + SHEET_PADDING + column * (layout.cellWidth + CELL_GAP), + SHEET_PADDING + row * (LABEL_HEIGHT + layout.cellHeight + CELL_GAP), + cell, + layout.cellWidth, + layout.cellHeight, + ); + }); + + return { + bytes: encodePngPixels(canvas.data, layout.width, layout.height, 4), + width: layout.width, + height: layout.height, + cellWidth: layout.cellWidth, + }; +} + +function drawCell( + canvas: PNG, + x: number, + y: number, + cell: ContactSheetCell, + cellWidth: number, + cellHeight: number, +): void { + drawPngGlyphText(canvas, { + x: x + LABEL_INSET, + y: y + LABEL_INSET, + text: formatContactSheetTimestamp(cell.timeMs), + color: LABEL_COLOR, + scale: LABEL_SCALE, + }); + blit(canvas, resizePngBox(toPng(cell.image), cellWidth, cellHeight), x, y + LABEL_HEIGHT); +} + +type ContactSheetLayout = Readonly<{ + width: number; + height: number; + columns: number; + cellWidth: number; + cellHeight: number; +}>; + +/** + * Sizes the grid so it fits the caller's pixel budget. + * + * A cell shrinks before a sheet refuses: an over-budget clip is ordinary (a tall iPad recording + * with many changes), and re-laying the same cells out at a smaller cell is always available. + */ +function fitLayout( + aspect: number, + cellCount: number, + input: ContactSheetRenderInput, +): ContactSheetLayout { + const fitted = fitToBudget(aspect, cellCount, input.maxPixels); + if (fitted) return fitted; + const smallest = layoutAt(aspect, cellCount, MIN_CONTACT_SHEET_CELL_WIDTH); + throw new AppError( + 'COMMAND_FAILED', + `Contact sheet would be ${smallest.width * smallest.height} pixels, above the configured maxImagePixels limit of ${input.maxPixels}`, + { + reason: CONTACT_SHEET_PIXEL_BUDGET_REASON, + cellCount: input.cells.length, + maxPixels: input.maxPixels, + hint: 'Shorten the recording or raise the command policy maxImagePixels limit.', + }, + ); +} + +/** + * Walks the cell-width ladder until the grid fits, because the label strip does not shrink with the + * cell and a single proportional guess can land over budget. The ladder is finite and strictly + * descending, so the narrowest readable cell is always the last thing tried. + */ +function fitToBudget( + aspect: number, + cellCount: number, + budget: number, +): ContactSheetLayout | undefined { + for (const cellWidth of LAYOUT_CELL_WIDTHS) { + const layout = layoutAt(aspect, cellCount, cellWidth); + if (layout.width * layout.height <= budget) return layout; + } + return undefined; +} + +function layoutAt(aspect: number, cellCount: number, cellWidth: number): ContactSheetLayout { + const columns = Math.min(CONTACT_SHEET_COLUMNS, cellCount); + const rows = Math.ceil(cellCount / columns); + const cellHeight = Math.max(1, Math.round(cellWidth * aspect)); + return { + columns, + cellWidth, + cellHeight, + width: SHEET_PADDING * 2 + columns * cellWidth + (columns - 1) * CELL_GAP, + height: SHEET_PADDING * 2 + rows * (LABEL_HEIGHT + cellHeight) + (rows - 1) * CELL_GAP, + }; +} + +function createCanvas(width: number, height: number, color: PngGlyphColor): PNG { + const canvas = new PNG({ width, height }); + for (let row = 0; row < height; row += 1) { + for (let column = 0; column < width; column += 1) { + setPngPixel(canvas, column, row, color); + } + } + return canvas; +} + +function blit(canvas: PNG, source: PNG, x: number, y: number): void { + for (let row = 0; row < source.height; row += 1) { + for (let column = 0; column < source.width; column += 1) { + const sourceOffset = (row * source.width + column) * 4; + setPngPixel(canvas, x + column, y + row, [ + source.data[sourceOffset]!, + source.data[sourceOffset + 1]!, + source.data[sourceOffset + 2]!, + source.data[sourceOffset + 3]!, + ]); + } + } +} + +function toPng(image: ContactSheetCell['image']): PNG { + const png = new PNG({ width: image.width, height: image.height }); + Buffer.from(image.data.buffer, image.data.byteOffset, image.data.byteLength).copy(png.data); + return png; +} + +/** Elapsed time from the clip start as `HH:MM:SS.mmm`, the shape the glyph table spells. */ +export function formatContactSheetTimestamp(timeMs: number): string { + const wholeSeconds = Math.floor(timeMs / 1000); + const hours = Math.floor(wholeSeconds / 3600); + const minutes = Math.floor((wholeSeconds % 3600) / 60); + const seconds = wholeSeconds % 60; + const milliseconds = Math.floor(timeMs % 1000); + const clock = [hours, minutes, seconds].map((part) => String(part).padStart(2, '0')).join(':'); + return `${clock}.${String(milliseconds).padStart(3, '0')}`; +} diff --git a/packages/capture-kit/src/recording/contact-sheet-report.ts b/packages/capture-kit/src/recording/contact-sheet-report.ts index fb872f89ff..7c519fdc98 100644 --- a/packages/capture-kit/src/recording/contact-sheet-report.ts +++ b/packages/capture-kit/src/recording/contact-sheet-report.ts @@ -11,3 +11,5 @@ export const CONTACT_SHEET_DURATION_REASON = 'contact_sheet_duration_unknown'; export const CONTACT_SHEET_EXTRACTION_REASON = 'contact_sheet_frame_extraction_failed'; /** Extraction returned nothing usable, so there is no sheet to draw. */ export const CONTACT_SHEET_NO_FRAMES_REASON = 'contact_sheet_no_frames'; +/** The requested sheet would exceed the caller's image pixel budget even at the smallest cell. */ +export const CONTACT_SHEET_PIXEL_BUDGET_REASON = 'contact_sheet_pixel_budget_exceeded'; diff --git a/packages/capture-kit/src/screenshot-overlay-draw.test.ts b/packages/capture-kit/src/screenshot-overlay-draw.test.ts new file mode 100644 index 0000000000..5ed0932700 --- /dev/null +++ b/packages/capture-kit/src/screenshot-overlay-draw.test.ts @@ -0,0 +1,64 @@ +import { describe, expect, test } from 'vitest'; +import { BLACK, solidPng, WHITE } from './png-pixels.fixtures.ts'; +import { PNG } from './png.ts'; +import { drawPngGlyphText, measurePngGlyphTextHeight } from './screenshot-overlay-draw.ts'; + +/** One glyph column count plus the gap after it; the distance the painter moves per character. */ +const GLYPH_PITCH = 6; + +function countPixels(png: PNG, color: readonly number[]): number { + let count = 0; + for (let offset = 0; offset < png.data.length; offset += 4) { + if ( + png.data[offset] === color[0] && + png.data[offset + 1] === color[1] && + png.data[offset + 2] === color[2] && + png.data[offset + 3] === color[3] + ) { + count += 1; + } + } + return count; +} + +describe('png glyph text', () => { + test('paints the timestamp characters an elapsed-time label is built from', () => { + for (const text of ['0', '9', ':', '.', '00:00:01.234']) { + const png = solidPng(140, 12, BLACK); + drawPngGlyphText(png, { x: 2, y: 2, text, color: WHITE }); + expect(countPixels(png, WHITE), text).toBeGreaterThan(0); + } + }); + + test('scales a glyph to whole blocks so a label stays crisp', () => { + const single = solidPng(40, 12, BLACK); + drawPngGlyphText(single, { x: 2, y: 2, text: '1', color: WHITE }); + const doubled = solidPng(40, 20, BLACK); + drawPngGlyphText(doubled, { x: 2, y: 2, text: '1', color: WHITE, scale: 2 }); + + expect(countPixels(doubled, WHITE)).toBe(countPixels(single, WHITE) * 4); + expect(measurePngGlyphTextHeight()).toBe(7); + expect(measurePngGlyphTextHeight(2)).toBe(14); + }); + + test('advances past a character the table does not cover', () => { + const blank = solidPng(40, 12, BLACK); + drawPngGlyphText(blank, { x: 2, y: 2, text: 'z', color: WHITE }); + expect(countPixels(blank, WHITE)).toBe(0); + + const shifted = solidPng(40, 12, BLACK); + drawPngGlyphText(shifted, { x: 2, y: 2, text: 'z1', color: WHITE }); + const aligned = solidPng(40, 12, BLACK); + drawPngGlyphText(aligned, { x: 2 + GLYPH_PITCH, y: 2, text: '1', color: WHITE }); + + expect(shifted.data.equals(aligned.data)).toBe(true); + }); + + test('clips at the image edge instead of writing past it', () => { + const png = solidPng(4, 4, BLACK); + expect(() => + drawPngGlyphText(png, { x: -3, y: -3, text: '00:00', color: WHITE, scale: 2 }), + ).not.toThrow(); + expect(png.data.length).toBe(4 * 4 * 4); + }); +}); diff --git a/packages/capture-kit/src/screenshot-overlay-draw.ts b/packages/capture-kit/src/screenshot-overlay-draw.ts index ef6a7ba920..eeacc94675 100644 --- a/packages/capture-kit/src/screenshot-overlay-draw.ts +++ b/packages/capture-kit/src/screenshot-overlay-draw.ts @@ -3,20 +3,28 @@ import type { PNG } from './png.ts'; import { clamp } from './screenshot-overlay-rects.ts'; /** - * Rasterizing one overlay ref onto a decoded PNG: border, badge, and the bitmap glyphs the badge - * needs. Which node earns a ref, and where its rect lands, is `screenshot-overlay.ts`'s question — - * this module only paints what it is handed. + * Rasterizing onto a decoded PNG: border, badge, and the elapsed-time labels other annotators paint. + * Which node earns a ref, and where its rect lands, is `screenshot-overlay.ts`'s question — this + * module only paints what it is handed. */ -const BORDER_COLOR = [255, 59, 48, 255] as const; -const BADGE_COLOR = [255, 214, 10, 255] as const; -const TEXT_COLOR = [0, 0, 0, 255] as const; -const FONT_WIDTH = 5; -const FONT_HEIGHT = 7; -const FONT_SPACING = 1; +const BORDER_COLOR: PngGlyphColor = [255, 59, 48, 255]; +const BADGE_COLOR: PngGlyphColor = [255, 214, 10, 255]; +const TEXT_COLOR: PngGlyphColor = [0, 0, 0, 255]; +const GLYPH_WIDTH = 5; +const GLYPH_HEIGHT = 7; +const GLYPH_SPACING = 1; +const GLYPH_PITCH = GLYPH_WIDTH + GLYPH_SPACING; const BADGE_PADDING_X = 3; const BADGE_PADDING_Y = 2; const BADGE_MARGIN = 2; const BORDER_THICKNESS = 2; + +/** + * The bitmap glyph table every capture annotator paints with. It covers the characters those labels + * are built from — the `e` ref prefix, digits, and the separator and decimal point of an elapsed + * time — and nothing else, so an unsupported character paints as blank rather than inventing a + * shape nobody reviewed. + */ const FONT: Record = { e: ['01110', '10000', '11110', '10000', '10000', '10001', '01110'], '0': ['01110', '10001', '10011', '10101', '11001', '10001', '01110'], @@ -29,20 +37,79 @@ const FONT: Record = { '7': ['11111', '00001', '00010', '00100', '01000', '01000', '01000'], '8': ['01110', '10001', '10001', '01110', '10001', '10001', '01110'], '9': ['01110', '10001', '10001', '01111', '00001', '00001', '01110'], -} as const; -// Badges currently render only `eN` refs, so the bitmap font intentionally covers `e` and digits. + ':': ['00000', '00100', '00100', '00000', '00100', '00100', '00000'], + '.': ['00000', '00000', '00000', '00000', '00000', '00110', '00110'], +}; + +/** One RGBA pixel, in the order PNG rows store them. */ +export type PngGlyphColor = readonly [number, number, number, number]; export function drawOverlayRef(png: PNG, overlayRef: ScreenshotOverlayRef): void { drawRectBorder(png, overlayRef.overlayRect, BORDER_COLOR, BORDER_THICKNESS); drawBadge(png, overlayRef.overlayRect, overlayRef.ref); } -function drawRectBorder( +/** Width the painted text occupies at scale 1, with no trailing inter-character gap. */ +function measurePngGlyphTextWidth(text: string): number { + if (text === '') return 0; + return text.length * GLYPH_PITCH - GLYPH_SPACING; +} + +export function measurePngGlyphTextHeight(scale = 1): number { + return GLYPH_HEIGHT * scale; +} + +/** + * Paints `text` at `x, y`, each glyph block `scale` pixels wide and tall, clipping at the image + * edge. A character the table does not cover still advances the cursor, so a partially supported + * label keeps its layout. + */ +export function drawPngGlyphText( + png: PNG, + input: Readonly<{ + x: number; + y: number; + text: string; + color: PngGlyphColor; + scale?: number; + }>, +): void { + const scale = input.scale ?? 1; + let cursorX = input.x; + for (const character of input.text.toLowerCase()) { + const glyph = FONT[character]; + if (glyph) drawGlyph(png, glyph, cursorX, input.y, scale, input.color); + cursorX += GLYPH_PITCH * scale; + } +} + +function drawGlyph( png: PNG, - rect: Rect, - color: readonly [number, number, number, number], - thickness: number, + glyph: readonly string[], + x: number, + y: number, + scale: number, + color: PngGlyphColor, ): void { + for (let row = 0; row < glyph.length; row += 1) { + for (let column = 0; column < glyph[row]!.length; column += 1) { + if (glyph[row]![column] !== '1') continue; + fillRect(png, x + column * scale, y + row * scale, scale, scale, color); + } + } +} + +/** Writes one pixel, dropping anything outside the image. */ +export function setPngPixel(png: PNG, x: number, y: number, color: PngGlyphColor): void { + if (x < 0 || y < 0 || x >= png.width || y >= png.height) return; + const index = (png.width * y + x) * 4; + png.data[index] = color[0]; + png.data[index + 1] = color[1]; + png.data[index + 2] = color[2]; + png.data[index + 3] = color[3]; +} + +function drawRectBorder(png: PNG, rect: Rect, color: PngGlyphColor, thickness: number): void { for (let offset = 0; offset < thickness; offset += 1) { drawHorizontalLine(png, rect.x, rect.x + rect.width - 1, rect.y + offset, color); drawHorizontalLine( @@ -64,9 +131,8 @@ function drawRectBorder( } function drawBadge(png: PNG, rect: Rect, text: string): void { - const badgeWidth = - BADGE_PADDING_X * 2 + text.length * FONT_WIDTH + Math.max(0, text.length - 1) * FONT_SPACING; - const badgeHeight = BADGE_PADDING_Y * 2 + FONT_HEIGHT; + const badgeWidth = BADGE_PADDING_X * 2 + measurePngGlyphTextWidth(text); + const badgeHeight = BADGE_PADDING_Y * 2 + GLYPH_HEIGHT; const x = clamp(rect.x, 0, Math.max(0, png.width - badgeWidth)); const preferredY = rect.y - badgeHeight - BADGE_MARGIN; const y = @@ -74,29 +140,12 @@ function drawBadge(png: PNG, rect: Rect, text: string): void { ? preferredY : clamp(rect.y + BADGE_MARGIN, 0, Math.max(0, png.height - badgeHeight)); fillRect(png, x, y, badgeWidth, badgeHeight, BADGE_COLOR); - drawText(png, x + BADGE_PADDING_X, y + BADGE_PADDING_Y, text, TEXT_COLOR); -} - -function drawText( - png: PNG, - x: number, - y: number, - text: string, - color: readonly [number, number, number, number], -): void { - let cursorX = x; - for (const character of text.toLowerCase()) { - const glyph = FONT[character]; - if (glyph) { - for (let row = 0; row < glyph.length; row += 1) { - for (let column = 0; column < glyph[row]!.length; column += 1) { - if (glyph[row]![column] !== '1') continue; - setPixel(png, cursorX + column, y + row, color); - } - } - } - cursorX += FONT_WIDTH + FONT_SPACING; - } + drawPngGlyphText(png, { + x: x + BADGE_PADDING_X, + y: y + BADGE_PADDING_Y, + text, + color: TEXT_COLOR, + }); } function fillRect( @@ -105,11 +154,11 @@ function fillRect( y: number, width: number, height: number, - color: readonly [number, number, number, number], + color: PngGlyphColor, ): void { for (let row = 0; row < height; row += 1) { for (let column = 0; column < width; column += 1) { - setPixel(png, x + column, y + row, color); + setPngPixel(png, x + column, y + row, color); } } } @@ -119,10 +168,10 @@ function drawHorizontalLine( startX: number, endX: number, y: number, - color: readonly [number, number, number, number], + color: PngGlyphColor, ): void { for (let x = startX; x <= endX; x += 1) { - setPixel(png, x, y, color); + setPngPixel(png, x, y, color); } } @@ -131,23 +180,9 @@ function drawVerticalLine( x: number, startY: number, endY: number, - color: readonly [number, number, number, number], + color: PngGlyphColor, ): void { for (let y = startY; y <= endY; y += 1) { - setPixel(png, x, y, color); + setPngPixel(png, x, y, color); } } - -function setPixel( - png: PNG, - x: number, - y: number, - color: readonly [number, number, number, number], -): void { - if (x < 0 || y < 0 || x >= png.width || y >= png.height) return; - const index = (png.width * y + x) * 4; - png.data[index] = color[0]; - png.data[index + 1] = color[1]; - png.data[index + 2] = color[2]; - png.data[index + 3] = color[3]; -} From bb4ccffbacc85d952c762be5fe0a4bdcc0d89148 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 22 Sep 2026 13:08:19 +0200 Subject: [PATCH 012/132] feat(recording): build a contact sheet from a finished recording (#2752) --- .../src/recording/artifact-paths.test.ts | 23 ++ .../src/recording/artifact-paths.ts | 11 + .../src/recording/contact-sheet-report.ts | 41 ++- .../src/recording/contact-sheet-selection.ts | 2 +- .../src/recording/contact-sheet.test.ts | 335 ++++++++++++++++++ .../src/recording/contact-sheet.ts | 244 +++++++++++++ packages/capture-kit/src/recording/video.ts | 13 +- .../host-kit/src/internal/atomic-file.test.ts | 45 +++ packages/host-kit/src/internal/atomic-file.ts | 18 +- 9 files changed, 717 insertions(+), 15 deletions(-) create mode 100644 packages/capture-kit/src/recording/artifact-paths.test.ts create mode 100644 packages/capture-kit/src/recording/contact-sheet.test.ts create mode 100644 packages/capture-kit/src/recording/contact-sheet.ts create mode 100644 packages/host-kit/src/internal/atomic-file.test.ts diff --git a/packages/capture-kit/src/recording/artifact-paths.test.ts b/packages/capture-kit/src/recording/artifact-paths.test.ts new file mode 100644 index 0000000000..cd85d4649d --- /dev/null +++ b/packages/capture-kit/src/recording/artifact-paths.test.ts @@ -0,0 +1,23 @@ +import { describe, expect, test } from 'vitest'; +import { + collectedRecordingPath, + nativeRecordingPath, + recordingContactSheetPath, +} from './artifact-paths.ts'; + +describe('recording artifact sibling paths', () => { + test('keeps the recorder file and the collected copy beside the export', () => { + expect(nativeRecordingPath('/tmp/recording.mp4')).toBe('/tmp/recording.native.mp4'); + expect(collectedRecordingPath('/tmp/recording.mp4')).toBe('/tmp/recording.collected.mp4'); + }); + + test('names the contact sheet after the report it is, not the container it came from', () => { + expect(recordingContactSheetPath('/tmp/recording.mp4')).toBe( + '/tmp/recording.contact-sheet.png', + ); + expect(recordingContactSheetPath('/tmp/rec')).toBe('/tmp/rec.contact-sheet.png'); + expect(recordingContactSheetPath('/tmp/rec.TAKE-2.MP4')).toBe( + '/tmp/rec.TAKE-2.contact-sheet.png', + ); + }); +}); diff --git a/packages/capture-kit/src/recording/artifact-paths.ts b/packages/capture-kit/src/recording/artifact-paths.ts index 80cacd48d7..294d35bdf8 100644 --- a/packages/capture-kit/src/recording/artifact-paths.ts +++ b/packages/capture-kit/src/recording/artifact-paths.ts @@ -19,6 +19,17 @@ export function collectedRecordingPath(exportPath: string): string { return siblingRecordingPath(exportPath, 'collected'); } +/** + * The report a caller asks a finished export to carry: one PNG holding the frames that changed, so + * an agent can read a recording without playing it. It is derived, so unlike the paths above it is + * never a recorder's and is always safe to rebuild from the export. + */ +export function recordingContactSheetPath(exportPath: string): string { + const extension = path.extname(exportPath); + const base = extension === '' ? exportPath : exportPath.slice(0, -extension.length); + return `${base}.contact-sheet.png`; +} + function siblingRecordingPath(exportPath: string, role: 'native' | 'collected'): string { const extension = path.extname(exportPath); const base = extension === '' ? exportPath : exportPath.slice(0, -extension.length); diff --git a/packages/capture-kit/src/recording/contact-sheet-report.ts b/packages/capture-kit/src/recording/contact-sheet-report.ts index 7c519fdc98..cfbd653551 100644 --- a/packages/capture-kit/src/recording/contact-sheet-report.ts +++ b/packages/capture-kit/src/recording/contact-sheet-report.ts @@ -1,6 +1,8 @@ /** - * Why a contact sheet was not drawn. Each reason is a typed constant the extraction path fails with, - * so a caller branches on the reason instead of reading an error message. + * What a contact sheet reports: the shape of the finished grid, and the typed reasons it refuses with + * when there is no grid to draw. Both stay here rather than in cross-layer contracts because nothing + * but this package's pipeline produces them and the surface that reads them already imports this + * package; a caller branches on a reason instead of reading an error message. */ /** The host cannot extract frames at all: frame decoding is Apple AVFoundation tooling. */ @@ -13,3 +15,38 @@ export const CONTACT_SHEET_EXTRACTION_REASON = 'contact_sheet_frame_extraction_f export const CONTACT_SHEET_NO_FRAMES_REASON = 'contact_sheet_no_frames'; /** The requested sheet would exceed the caller's image pixel budget even at the smallest cell. */ export const CONTACT_SHEET_PIXEL_BUDGET_REASON = 'contact_sheet_pixel_budget_exceeded'; +/** The file is present but is not a container this feature decodes, such as a WebM recording. */ +export const CONTACT_SHEET_CONTAINER_REASON = 'contact_sheet_container_unsupported'; +/** The sheet could not be written where it was asked to land. */ +export const CONTACT_SHEET_OUTPUT_WRITE_REASON = 'contact_sheet_output_write_failed'; +/** The output resolves to the recording itself, which a sheet must never replace. */ +export const CONTACT_SHEET_OUTPUT_COLLISION_REASON = 'contact_sheet_output_collides_with_input'; + +export type RecordingContactSheetCell = { + /** Presentation time of the decoded frame this cell shows, in milliseconds from the clip start. */ + readonly timeMs: number; + /** Share of pixels that moved since the previously kept cell, from 0 to 1. */ + readonly changedPixelRatio: number; +}; + +export type RecordingContactSheetResult = { + readonly path: string; + readonly videoPath: string; + /** Video timeline the grid was planned over, in milliseconds. */ + readonly durationMs: number; + readonly width: number; + readonly height: number; + /** + * Sample times the grid asked for. Sampling is a grid, not a scan: a transient that falls wholly + * between two sample times is absent from the sheet. + */ + readonly sampledFrameCount: number; + /** Frames the decoder actually returned. */ + readonly decodedFrameCount: number; + /** Requested sample times the decoder declined to answer. */ + readonly skippedSampleCount: number; + /** Changed-pixel share a frame had to exceed to earn its own cell. */ + readonly changedPixelThreshold: number; + readonly cells: readonly RecordingContactSheetCell[]; + readonly warning?: string; +}; diff --git a/packages/capture-kit/src/recording/contact-sheet-selection.ts b/packages/capture-kit/src/recording/contact-sheet-selection.ts index 18ae835bec..0a128f094e 100644 --- a/packages/capture-kit/src/recording/contact-sheet-selection.ts +++ b/packages/capture-kit/src/recording/contact-sheet-selection.ts @@ -13,7 +13,7 @@ import { computePngChangedPixelRatio, type PngRgbImage } from '../png-changed-pi * unchanged chrome — measured 0.030. Any value in that gap separates the two, and this one sits * nearer the noise so a quiet change costs an extra cell instead of going unshown. */ -const CONTACT_SHEET_CHANGED_PIXEL_THRESHOLD = 0.02; +export const CONTACT_SHEET_CHANGED_PIXEL_THRESHOLD = 0.02; /** Cells one sheet prints. The grid thins evenly across time to hold this, keeping both ends. */ export const MAX_CONTACT_SHEET_CELLS = 24; diff --git a/packages/capture-kit/src/recording/contact-sheet.test.ts b/packages/capture-kit/src/recording/contact-sheet.test.ts new file mode 100644 index 0000000000..464f65dd3b --- /dev/null +++ b/packages/capture-kit/src/recording/contact-sheet.test.ts @@ -0,0 +1,335 @@ +import { beforeEach, describe, expect, test, vi } from 'vitest'; +import fs from 'node:fs'; +import path from 'node:path'; +import { + CONTACT_SHEET_CONTAINER_REASON, + CONTACT_SHEET_DURATION_REASON, + CONTACT_SHEET_EXTRACTION_REASON, + CONTACT_SHEET_OUTPUT_COLLISION_REASON, + CONTACT_SHEET_PIXEL_BUDGET_REASON, + CONTACT_SHEET_UNSUPPORTED_HOST_REASON, +} from './contact-sheet-report.ts'; +import { AppError } from '@agent-device/kernel/errors'; +import { BLACK, RED, WHITE, paintPng, solidPng } from '../png-pixels.fixtures.ts'; +import { mkdtempForTestSync } from '../tmp-dir.fixtures.ts'; +import { decodePng } from '../png.ts'; +import { mp4Atom, mp4MovieHeader } from './mp4.fixtures.ts'; +import { writeDecodedFrames, type StubFrame } from './contact-sheet.fixtures.ts'; +import { buildRecordingContactSheet, recordingContactSheetPath } from './contact-sheet.ts'; + +vi.mock(import('@agent-device/host-kit/command'), async (importOriginal) => ({ + ...(await importOriginal()), + runCmd: vi.fn(), +})); + +vi.mock(import('./swift-cache.ts'), async (importOriginal) => ({ + ...(await importOriginal()), + compileSwiftSourceFile: vi.fn(async () => '/cached/bin/recording-frames'), +})); + +import { runCmd } from '@agent-device/host-kit/command'; +import { likelyPlayableWebmContainer } from '../__tests__/test-utils/video-fixtures.ts'; + +const mockRunCmd = vi.mocked(runCmd); +const directory = mkdtempForTestSync('agent-device-contact-sheet-pipeline-'); + +/** A recording the container sniff and the timeline reader both recognise, at a chosen length. */ +function recording(name: string, durationMs: number): string { + const filePath = path.join(directory, `${name}.mp4`); + fs.writeFileSync( + filePath, + Buffer.concat([ + mp4Atom('ftyp', Buffer.alloc(24)), + mp4Atom('mdat', Buffer.alloc(16)), + mp4Atom( + 'moov', + mp4Atom( + 'mvhd', + mp4MovieHeader({ version: 0, timescale: 1_000, duration: unknownable(durationMs) }), + ), + ), + ]), + ); + return filePath; +} + +function unknownable(durationMs: number): number { + return durationMs < 0 ? 0xffffffff : durationMs; +} + +function frame(timeMs: number, changed: boolean): StubFrame { + return { + png: changed + ? paintPng(solidPng(8, 8, BLACK), { x: 0, y: 0, width: 8, height: 8 }, RED) + : solidPng(8, 8, BLACK), + actualTimeMs: timeMs, + }; +} + +/** + * Answers the grid the planner will ask for. `presentMs` names the times the decoder answers; + * anything the grid asked for and is left out is reported skipped, as a real decoder declines. + */ +function answerForGrid( + input: Readonly<{ + timesMs: readonly number[]; + presentMs?: readonly number[]; + changedMs?: readonly number[]; + }>, +): void { + const present = new Set(input.presentMs ?? input.timesMs); + const changed = new Set(input.changedMs ?? []); + const byRequestedTime = new Map(); + for (const timeMs of input.timesMs) { + if (present.has(timeMs)) byRequestedTime.set(timeMs, frame(timeMs, changed.has(timeMs))); + } + mockRunCmd.mockImplementation(async (_cmd, args) => + writeDecodedFrames({ args: args as string[], framesByRequestedTimeMs: byRequestedTime }), + ); +} + +/** + * Every case runs on a macOS host unless it names another, because frame decoding is Apple tooling + * and the unit lane also runs on Linux. + */ +type BuildInput = Partial< + Omit[0], 'videoPath' | 'hostPlatform'> +> & { hostPlatform?: NodeJS.Platform }; + +function build(videoPath: string, input: BuildInput = {}) { + return buildRecordingContactSheet({ + ...input, + videoPath, + maxPixels: input.maxPixels ?? 20_000_000, + changedPixelThreshold: input.changedPixelThreshold ?? 0.1, + hostPlatform: input.hostPlatform ?? 'darwin', + }); +} + +async function reasonOf(action: () => Promise): Promise { + try { + await action(); + } catch (error) { + return error instanceof AppError ? error.details?.reason : error; + } + return 'no error thrown'; +} + +describe('buildRecordingContactSheet', () => { + beforeEach(() => { + mockRunCmd.mockReset(); + }); + + test('writes the sheet beside the recording and says what it covered', async () => { + const video = recording('cover', 1_000); + answerForGrid({ timesMs: [0, 250, 500, 750, 1000], changedMs: [500] }); + + const sheet = await build(video); + + expect(sheet.path).toBe(recordingContactSheetPath(video)); + expect(fs.existsSync(sheet.path)).toBe(true); + expect(sheet.videoPath).toBe(video); + expect(sheet.durationMs).toBe(1_000); + expect(sheet.sampledFrameCount).toBe(5); + expect(sheet.decodedFrameCount).toBe(5); + expect(sheet.skippedSampleCount).toBe(0); + // 750 returns to the opening screen, which is a change against the frame kept before it. + expect(sheet.cells.map((cell) => cell.timeMs)).toEqual([0, 500, 750, 1000]); + expect(sheet.warning).toBeUndefined(); + + const decoded = decodePng(fs.readFileSync(sheet.path), 'sheet'); + expect([decoded.width, decoded.height]).toEqual([sheet.width, sheet.height]); + expect( + [...fs.readdirSync(directory)].some( + (name) => name.endsWith('.writing') || name.endsWith('.tmp'), + ), + ).toBe(false); + }); + + test('honours an explicit output path', async () => { + const video = recording('explicit', 500); + const outputPath = path.join(directory, 'named', 'sheet.png'); + answerForGrid({ timesMs: [0, 250, 500], changedMs: [250] }); + + const sheet = await build(video, { outputPath }); + + expect(sheet.path).toBe(outputPath); + expect(fs.existsSync(outputPath)).toBe(true); + }); + + test('keeps one cell when the recorded screen never changed', async () => { + const video = recording('static', 500); + answerForGrid({ timesMs: [0, 250, 500] }); + + const sheet = await build(video); + + expect(sheet.cells).toHaveLength(2); + expect(sheet.cells.at(-1)).toMatchObject({ timeMs: 500, changedPixelRatio: 0 }); + }); + + test('discloses the sample times the decoder declined', async () => { + const video = recording('skipped', 1_000); + answerForGrid({ + timesMs: [0, 250, 500, 750, 1000], + presentMs: [0, 750, 1000], + changedMs: [750], + }); + + const sheet = await build(video); + + expect(sheet.sampledFrameCount).toBe(5); + expect(sheet.decodedFrameCount).toBe(3); + expect(sheet.skippedSampleCount).toBe(2); + expect(sheet.warning).toMatch(/2 sample times returned no frame/); + }); + + test('refuses a WebM recording rather than decoding nothing and calling it a sheet', async () => { + const webm = path.join(directory, 'web.webm'); + fs.writeFileSync(webm, likelyPlayableWebmContainer()); + + const error = await build(webm).catch((error: unknown) => error); + + expect(error).toBeInstanceOf(AppError); + expect((error as AppError).details?.reason).toBe(CONTACT_SHEET_CONTAINER_REASON); + expect((error as AppError).message).toMatch(/WebM/); + }); + + test('refuses a file that is not a readable MP4', async () => { + const notAVideo = path.join(directory, 'broken.mp4'); + fs.writeFileSync(notAVideo, 'not a video at all'); + + expect(await reasonOf(() => build(notAVideo))).toBe(CONTACT_SHEET_CONTAINER_REASON); + }); + + test('refuses a clip whose timeline the container cannot name', async () => { + expect(await reasonOf(() => build(recording('no-duration', -1)))).toBe( + CONTACT_SHEET_DURATION_REASON, + ); + }); + + test('surfaces a failed extraction and writes no sheet', async () => { + const video = recording('extraction-fails', 1_000); + mockRunCmd.mockImplementation(async (_cmd, args) => + writeDecodedFrames({ + args: args as string[], + framesByRequestedTimeMs: new Map(), + failWithExitCode: 1, + }), + ); + + expect(await reasonOf(() => build(video))).toBe(CONTACT_SHEET_EXTRACTION_REASON); + expect(fs.existsSync(recordingContactSheetPath(video))).toBe(false); + }); + + test('refuses to try on a host with no frame decoder', async () => { + const video = recording('unsupported-host', 1_000); + + expect(await reasonOf(() => build(video, { hostPlatform: 'linux' }))).toBe( + CONTACT_SHEET_UNSUPPORTED_HOST_REASON, + ); + expect(mockRunCmd).not.toHaveBeenCalled(); + }); + + test('draws the sheet on a decoded frame whatever the source pixels were', async () => { + const video = recording('pixels', 250); + const byRequestedTime = new Map([ + [0, { png: solidPng(8, 8, WHITE), actualTimeMs: 0 }], + ]); + mockRunCmd.mockImplementation(async (_cmd, args) => + writeDecodedFrames({ args: args as string[], framesByRequestedTimeMs: byRequestedTime }), + ); + + const sheet = await build(video); + const decoded = decodePng(fs.readFileSync(sheet.path), 'sheet'); + + let white = 0; + for (let offset = 0; offset < decoded.data.length; offset += 4) { + if (decoded.data[offset] === 255 && decoded.data[offset + 1] === 255) white += 1; + } + expect(white).toBeGreaterThan(0); + }); + + test('refuses to write the sheet where the recording it describes is', async () => { + const video = recording('collision', 1_000); + + expect(await reasonOf(() => build(video, { outputPath: video }))).toBe( + CONTACT_SHEET_OUTPUT_COLLISION_REASON, + ); + // Refused before the decoder ran, so a mistyped --out cannot cost a ruined recording. + expect(mockRunCmd).not.toHaveBeenCalled(); + expect(fs.readFileSync(video).length).toBeGreaterThan(0); + }); + + test('refuses an output path that reaches the recording through a second name', async () => { + const video = recording('aliased', 1_000); + const symlink = path.join(path.dirname(video), 'sheet-of-my-recording.mp4'); + const hardLink = path.join(path.dirname(video), 'recording-copy.mp4'); + fs.symlinkSync(video, symlink); + fs.linkSync(video, hardLink); + + for (const outputPath of [symlink, hardLink]) { + expect(await reasonOf(() => build(video, { outputPath }))).toBe( + CONTACT_SHEET_OUTPUT_COLLISION_REASON, + ); + } + // Neither alias was opened for writing, so the recording is still whole behind both names. + expect(mockRunCmd).not.toHaveBeenCalled(); + expect(fs.readFileSync(video).length).toBe(fs.statSync(hardLink).size); + }); + + test('refuses a decoded frame too large for the sheet it was meant to fill', async () => { + const video = recording('oversized', 250); + mockRunCmd.mockImplementation(async (_cmd, args) => + writeDecodedFrames({ + args: args as string[], + framesByRequestedTimeMs: new Map([ + [0, { png: solidPng(16, 16, BLACK), actualTimeMs: 0 }], + [250, { png: solidPng(16, 16, BLACK), actualTimeMs: 250 }], + ]), + }), + ); + + expect(await reasonOf(() => build(video, { maxPixels: 100 }))).toBe( + CONTACT_SHEET_EXTRACTION_REASON, + ); + }); + + test('refuses a wider frame than the cells the decoder was asked to draw', async () => { + const video = recording('wide', 250); + mockRunCmd.mockImplementation(async (_cmd, args) => + writeDecodedFrames({ + args: args as string[], + framesByRequestedTimeMs: new Map([ + [0, { png: solidPng(400, 40, BLACK), actualTimeMs: 0 }], + [250, { png: solidPng(400, 40, BLACK), actualTimeMs: 250 }], + ]), + }), + ); + + expect(await reasonOf(() => build(video))).toBe(CONTACT_SHEET_EXTRACTION_REASON); + }); + + test('refuses a sheet the pixel budget cannot hold and leaves nothing at the output', async () => { + const video = recording('budget', 1_000); + const out = path.join(directory, 'budget-sheet.png'); + answerForGrid({ timesMs: [0, 250, 500, 750, 1000], changedMs: [250, 500, 750, 1000] }); + + expect(await reasonOf(() => build(video, { outputPath: out, maxPixels: 200 }))).toBe( + CONTACT_SHEET_PIXEL_BUDGET_REASON, + ); + expect(fs.existsSync(out)).toBe(false); + expect(fs.readdirSync(directory).filter((name) => name.endsWith('.writing'))).toEqual([]); + }); + + test('replaces a sheet that is already at the output path', async () => { + const video = recording('replace', 1_000); + const out = path.join(directory, 'replace-sheet.png'); + fs.writeFileSync(out, 'a sheet from an earlier run'); + answerForGrid({ timesMs: [0, 250, 500, 750, 1000], changedMs: [500] }); + + const sheet = await build(video, { outputPath: out }); + + expect(sheet.path).toBe(out); + expect(fs.readFileSync(out).subarray(1, 4).toString()).toBe('PNG'); + }); +}); diff --git a/packages/capture-kit/src/recording/contact-sheet.ts b/packages/capture-kit/src/recording/contact-sheet.ts new file mode 100644 index 0000000000..8510a73401 --- /dev/null +++ b/packages/capture-kit/src/recording/contact-sheet.ts @@ -0,0 +1,244 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { + CONTACT_SHEET_CONTAINER_REASON, + CONTACT_SHEET_EXTRACTION_REASON, + CONTACT_SHEET_OUTPUT_COLLISION_REASON, + CONTACT_SHEET_OUTPUT_WRITE_REASON, + type RecordingContactSheetCell, + type RecordingContactSheetResult, +} from './contact-sheet-report.ts'; +import { publishFileSync } from '@agent-device/host-kit/file'; +import { AppError, createRequestCanceledError } from '@agent-device/kernel/errors'; +import { decodePngAsync } from '../png-worker-client.ts'; +import { readPngSize } from '../png-size.ts'; +import { recordingContactSheetPath } from './artifact-paths.ts'; +import { + assertContactSheetHostSupport, + extractRecordingFrames, + type ExtractedRecordingFrames, +} from './contact-sheet-frames.ts'; +import { CONTACT_SHEET_FRAME_WIDTH, planContactSheetSampleTimes } from './contact-sheet-plan.ts'; +import { renderContactSheet } from './contact-sheet-render.ts'; +import { + CONTACT_SHEET_CHANGED_PIXEL_THRESHOLD, + MAX_CONTACT_SHEET_CELLS, + selectContactSheetCells, + type ContactSheetSample, +} from './contact-sheet-selection.ts'; +import { readMp4DurationMs } from './mp4-duration.ts'; +import { readVideoContainerKind } from './video.ts'; + +export { recordingContactSheetPath }; + +/** + * Builds the one PNG that lets a caller read a recording without playing it. + * + * Frames come out of the finished export, never from a parallel capture: the sheet is drawn from + * the same bytes the caller was handed, so it cannot describe a screen the delivered video does not + * contain. The grid it samples is bounded by the sheet, not by the clip's length. + */ +export async function buildRecordingContactSheet( + input: Readonly<{ + videoPath: string; + outputPath?: string; + maxPixels: number; + changedPixelThreshold?: number; + hostPlatform?: NodeJS.Platform; + signal?: AbortSignal; + }>, +): Promise { + assertContactSheetHostSupport(input.hostPlatform ?? process.platform); + const videoPath = path.resolve(input.videoPath); + const outputPath = path.resolve(input.outputPath ?? recordingContactSheetPath(videoPath)); + assertOutputDoesNotReplaceRecording(videoPath, outputPath); + + const container = await readVideoContainerKind(videoPath); + if (container !== 'mp4') { + throw new AppError( + 'INVALID_ARGS', + container === 'webm' + ? `Contact sheets need an MP4 recording; ${videoPath} is WebM, whose frames this decoder does not read` + : `Contact sheets need an MP4 recording; ${videoPath} is not a readable MP4`, + { + reason: CONTACT_SHEET_CONTAINER_REASON, + videoPath, + container: container ?? 'unknown', + hint: 'Re-record on a backend that exports MP4, or pass the MP4 artifact the recording produced.', + }, + ); + } + + const grid = planContactSheetSampleTimes(readMp4DurationMs(videoPath), videoPath); + const scratchDir = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-contact-sheet-')); + try { + const extracted = await extractRecordingFrames({ + videoPath, + scratchDir, + timesMs: grid.timesMs, + maxWidth: CONTACT_SHEET_FRAME_WIDTH, + hostPlatform: input.hostPlatform ?? process.platform, + ...(input.signal === undefined ? {} : { signal: input.signal }), + }); + const samples = await decodeFrameSamples(extracted.frames, input.maxPixels); + throwIfCanceled(input.signal); + const selection = selectContactSheetCells( + samples, + input.changedPixelThreshold ?? CONTACT_SHEET_CHANGED_PIXEL_THRESHOLD, + ); + const sheet = renderContactSheet({ cells: selection.cells, maxPixels: input.maxPixels }); + writeContactSheetFile(outputPath, sheet.bytes); + + return { + path: outputPath, + videoPath, + durationMs: grid.durationMs, + width: sheet.width, + height: sheet.height, + sampledFrameCount: grid.timesMs.length, + decodedFrameCount: extracted.frames.length, + skippedSampleCount: extracted.skippedSampleCount, + changedPixelThreshold: input.changedPixelThreshold ?? CONTACT_SHEET_CHANGED_PIXEL_THRESHOLD, + cells: selection.cells.map(toCellSummary), + ...joinContactSheetWarnings(selection, extracted), + }; + } finally { + fs.rmSync(scratchDir, { recursive: true, force: true }); + } +} + +/** + * Decodes what the decoder handed over, refusing any frame that is bigger than the whole sheet it + * is meant to fill. The sheet's pixel budget is the caller's ceiling on this command's memory, and + * a decoder asked for 360px-wide cells that returns something else broke the contract. + */ +async function decodeFrameSamples( + frames: ExtractedRecordingFrames['frames'], + maxPixels: number, +): Promise { + const samples: ContactSheetSample[] = []; + for (const frame of frames) { + const size = await readPngSize(frame.path); + if (size.width > CONTACT_SHEET_FRAME_WIDTH) { + throw new AppError( + 'COMMAND_FAILED', + `Frame extraction returned a ${size.width}px-wide frame when the sheet asked for ${CONTACT_SHEET_FRAME_WIDTH}px cells`, + { + reason: CONTACT_SHEET_EXTRACTION_REASON, + framePath: frame.path, + frameWidth: size.width, + frameWidthLimit: CONTACT_SHEET_FRAME_WIDTH, + }, + ); + } + if (size.width * size.height > maxPixels) { + throw new AppError( + 'COMMAND_FAILED', + `Frame extraction returned a ${size.width}x${size.height} frame, above the ${maxPixels}-pixel budget for the whole sheet`, + { + reason: CONTACT_SHEET_EXTRACTION_REASON, + framePath: frame.path, + framePixels: size.width * size.height, + maxPixels, + }, + ); + } + const image = await decodePngAsync(fs.readFileSync(frame.path), 'recording frame'); + samples.push({ timeMs: frame.actualTimeMs, image }); + } + return samples; +} + +function throwIfCanceled(signal: AbortSignal | undefined): void { + if (signal?.aborted) throw createRequestCanceledError(); +} + +/** + * A sheet is derived from the recording, so it can never be the thing the recording becomes. The + * check runs before any decoding so a mistyped --out cannot cost a failed sheet on top of the + * recording it was pointing at. + * + * Resolved path strings cannot answer this on their own: a case-insensitive volume spells the same + * file two ways, and a hard link or symlink names it from a second path entirely. So an output that + * already exists is compared by the file it opens, and only an output that does not exist yet — + * where there is no file to open — falls back to comparing the names. + */ +function assertOutputDoesNotReplaceRecording(videoPath: string, outputPath: string): void { + const collides = existingFileIdentity(videoPath, outputPath) ?? outputPath === videoPath; + if (!collides) return; + throw new AppError( + 'INVALID_ARGS', + `A contact sheet cannot be written over the recording it describes: ${outputPath}`, + { + reason: CONTACT_SHEET_OUTPUT_COLLISION_REASON, + videoPath, + outputPath, + hint: 'Choose a different --out path, or omit it to write beside the recording.', + }, + ); +} + +/** Whether both paths open the same file, or `undefined` while one of them opens nothing. */ +function existingFileIdentity(left: string, right: string): boolean | undefined { + const leftStats = statExisting(left); + const rightStats = statExisting(right); + if (leftStats === undefined || rightStats === undefined) return undefined; + return leftStats.dev === rightStats.dev && leftStats.ino === rightStats.ino; +} + +function statExisting(target: string): fs.Stats | undefined { + try { + return fs.statSync(target); + } catch { + return undefined; + } +} + +function toCellSummary(cell: { + timeMs: number; + changedPixelRatio: number; +}): RecordingContactSheetCell { + return { timeMs: cell.timeMs, changedPixelRatio: cell.changedPixelRatio }; +} + +/** + * Discloses what the grid did not show rather than letting a full-looking sheet imply completeness. + */ +function joinContactSheetWarnings( + selection: ReturnType, + extracted: ExtractedRecordingFrames, +): { warning?: string } { + const warnings: string[] = []; + if (extracted.skippedSampleCount > 0) { + warnings.push( + `${extracted.skippedSampleCount} sample times returned no frame, so the sheet covers less of the recording than it asked for`, + ); + } + if (selection.thinned) { + warnings.push( + `${selection.keptCellCount} changes were found and ${MAX_CONTACT_SHEET_CELLS} cells are printed, so the sheet thins evenly across the recording instead of showing every change`, + ); + } + return warnings.length === 0 ? {} : { warning: warnings.join('; ') }; +} + +/** + * Publishes the sheet in one move, so a caller can never read a half-written PNG. + * + * The staging, the exclusive temp name, and the rename belong to the host's atomic publisher; this + * only makes sure a directory exists for the sheet it was asked to write and answers a failed write + * with the reason the command documents. + */ +function writeContactSheetFile(outputPath: string, bytes: Buffer): void { + try { + fs.mkdirSync(path.dirname(outputPath), { recursive: true }); + publishFileSync({ destination: outputPath, contents: bytes }); + } catch (error) { + throw new AppError( + 'COMMAND_FAILED', + `Failed to write the contact sheet to ${outputPath}: ${error instanceof Error ? error.message : String(error)}`, + { reason: CONTACT_SHEET_OUTPUT_WRITE_REASON, outputPath }, + ); + } +} diff --git a/packages/capture-kit/src/recording/video.ts b/packages/capture-kit/src/recording/video.ts index 16e9f13e79..adf1ca25b6 100644 --- a/packages/capture-kit/src/recording/video.ts +++ b/packages/capture-kit/src/recording/video.ts @@ -68,7 +68,7 @@ export async function waitForStableFile( } export async function isPlayableVideo(filePath: string): Promise { - const container = await likelyPlayableVideoContainer(filePath); + const container = await readVideoContainerKind(filePath); if (!container) return false; // AVFoundation is the MP4 semantic validator. It does not reliably load WebM on supported // macOS hosts, so WebM completion is established by its EBML document type + Segment marker. @@ -146,10 +146,17 @@ function isSwiftVideoValidatorUnavailable(stderr: string, stdout: string): boole * spawns nothing, which is what lets a stop check a copy before it trusts it (ADR 0024 2.3). */ export async function hasVideoContainer(filePath: string): Promise { - return (await likelyPlayableVideoContainer(filePath)) !== undefined; + return (await readVideoContainerKind(filePath)) !== undefined; } -async function likelyPlayableVideoContainer(filePath: string): Promise<'mp4' | 'webm' | undefined> { +/** + * Which video container a file actually is, sniffed from its bytes rather than its name. Callers + * that can only decode one container ask this before they spawn a decoder, because a `.mp4` that + * is not an MP4 and a WebM that is named well both answer differently. + */ +export async function readVideoContainerKind( + filePath: string, +): Promise<'mp4' | 'webm' | undefined> { try { const stats = fs.statSync(filePath); if (!stats.isFile() || stats.size <= 0) { diff --git a/packages/host-kit/src/internal/atomic-file.test.ts b/packages/host-kit/src/internal/atomic-file.test.ts new file mode 100644 index 0000000000..a35f010209 --- /dev/null +++ b/packages/host-kit/src/internal/atomic-file.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, test } from 'vitest'; +import fs from 'node:fs'; +import path from 'node:path'; +import { mkdtempForTestSync } from './tmp-dir.fixtures.ts'; +import { publishFileSync } from './atomic-file.ts'; + +const directory = mkdtempForTestSync('host-kit-atomic-file-'); + +function destination(name: string): string { + return path.join(directory, name); +} + +describe('publishFileSync', () => { + test('writes text as UTF-8', () => { + const target = destination('note.json'); + publishFileSync({ destination: target, contents: '{"ok":true,"mark":"é"}' }); + + expect(fs.readFileSync(target, 'utf8')).toBe('{"ok":true,"mark":"é"}'); + }); + + test('writes bytes exactly as the caller built them', () => { + // PNG magic plus bytes a UTF-8 round trip would not hand back unchanged. + const bytes = Uint8Array.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0xc3, 0x28]); + const target = destination('sheet.png'); + publishFileSync({ destination: target, contents: bytes }); + + expect([...fs.readFileSync(target)]).toEqual([...bytes]); + }); + + test('replaces what was already published at the destination', () => { + const target = destination('replaced.txt'); + publishFileSync({ destination: target, contents: 'first' }); + publishFileSync({ destination: target, contents: Uint8Array.from([2, 3, 4]) }); + + expect([...fs.readFileSync(target)]).toEqual([2, 3, 4]); + }); + + test('leaves no staging file beside what it published', () => { + const target = destination('clean.txt'); + publishFileSync({ destination: target, contents: 'body' }); + + const siblings = fs.readdirSync(directory).filter((name) => name.startsWith('clean.txt.')); + expect(siblings).toEqual([]); + }); +}); diff --git a/packages/host-kit/src/internal/atomic-file.ts b/packages/host-kit/src/internal/atomic-file.ts index 1d91a615e8..3df81e87ec 100644 --- a/packages/host-kit/src/internal/atomic-file.ts +++ b/packages/host-kit/src/internal/atomic-file.ts @@ -8,25 +8,25 @@ export type AtomicPublishMode = 'replace' | 'link-exclusive'; /** * Publishes a complete file from a same-directory temporary sibling. * + * Contents are either UTF-8 text or bytes: an encoding is only ever passed when the caller handed + * over text, so a PNG or a video chunk reaches the disk exactly as the caller built it. + * * The temporary file is always removed after the publish attempt. Cleanup is * best effort and never replaces the write or publish error; when a request * diagnostics scope exists, a cleanup failure is retained as secondary evidence. */ export function publishFileSync(options: { destination: string; - contents: string; + contents: string | Uint8Array; mode?: number; publish?: AtomicPublishMode; }): void { withAtomicPublishTempPathSync(options.destination, (temporaryPath) => { - if (options.mode === undefined) { - fs.writeFileSync(temporaryPath, options.contents, 'utf8'); - } else { - fs.writeFileSync(temporaryPath, options.contents, { - encoding: 'utf8', - mode: options.mode, - }); - } + const writeOptions = { + encoding: typeof options.contents === 'string' ? ('utf8' as const) : undefined, + ...(options.mode === undefined ? {} : { mode: options.mode }), + }; + fs.writeFileSync(temporaryPath, options.contents, writeOptions); if (options.publish === 'link-exclusive') { fs.linkSync(temporaryPath, options.destination); } else { From 9d6291f0f821acb67ce1a3f633563f333e817670 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 22 Sep 2026 13:08:20 +0200 Subject: [PATCH 013/132] feat(cli): read a recording with record contact-sheet (#2747) * feat(cli): read a recording with record contact-sheet * chore(gates): publish the contact-sheet capture-kit facade --- .fallowrc.json | 3 +- docs/agents/cli-flags.md | 6 + packages/capture-kit/package.json | 4 + .../src/recording/contact-sheet-report.ts | 2 + .../src/recording/contact-sheet.ts | 9 + .../command-registry/src/command-schema.ts | 11 + packages/kernel/src/contracts.ts | 1 + scripts/__tests__/eager-closure-budgets.ts | 9 +- scripts/layering/package-boundaries.test.ts | 1 + src/cli/commands/__tests__/recording.test.ts | 48 ++++ src/cli/commands/recording.ts | 64 +++++ src/cli/commands/router.ts | 1 + .../parser/__tests__/args-validation.test.ts | 59 +++++ src/cli/parser/args.ts | 44 ++++ src/cli/resolve-cli-options.test.ts | 40 +++ src/commands/recording/index.test.ts | 12 + src/commands/recording/index.ts | 33 ++- .../recording/runtime/contact-sheet.test.ts | 214 ++++++++++++++++ .../recording/runtime/contact-sheet.ts | 132 ++++++++++ src/commands/recording/runtime/index.ts | 10 + src/commands/schema/cli-help.ts | 1 + .../recording-contact-sheet.test.ts | 238 ++++++++++++++++++ test/wire-compat/ledger.json | 6 +- website/docs/docs/commands.md | 3 + 24 files changed, 940 insertions(+), 11 deletions(-) create mode 100644 src/cli/commands/__tests__/recording.test.ts create mode 100644 src/cli/commands/recording.ts create mode 100644 src/cli/resolve-cli-options.test.ts create mode 100644 src/commands/recording/runtime/contact-sheet.test.ts create mode 100644 src/commands/recording/runtime/contact-sheet.ts create mode 100644 test/integration/recording-contact-sheet.test.ts diff --git a/.fallowrc.json b/.fallowrc.json index 84cd610a37..ebe7cf1436 100644 --- a/.fallowrc.json +++ b/.fallowrc.json @@ -217,7 +217,7 @@ }, { "comment": "Dedicated CLI command handlers are reached only through the dynamic `import()` table `dedicatedCliCommandHandlerLoaders` in src/cli/commands/router.ts, which --production analysis cannot follow to a consumer. Same shape as the daemon route-handler entry above; that table is what enumerates this list, so add/remove here whenever a loader is added/removed.", - "file": "src/cli/commands/{auth,connection,daemon,device,proxy,replay,screenshot,takeover}.ts", + "file": "src/cli/commands/{auth,connection,daemon,device,proxy,recording,replay,screenshot,takeover}.ts", "exports": [ "authCommand", "connectCommand", @@ -226,6 +226,7 @@ "daemonCommand", "deviceCommand", "proxyCommand", + "recordingCommand", "replayCommand", "screenshotCommand", "diffCommand", diff --git a/docs/agents/cli-flags.md b/docs/agents/cli-flags.md index 56b0c3662c..0291f5531c 100644 --- a/docs/agents/cli-flags.md +++ b/docs/agents/cli-flags.md @@ -51,6 +51,12 @@ steps 1-3, plus step 9. everything in `allowedFlags`. Keep a cross-cutting opt-in out of every synopsis with `usageHidden: true` on its flag definition. `src/commands/schema/usage.test.ts` fails a tail that names an option the command does not accept, or one the hand-written grammar already wrote. +- A command whose first positional picks an action declares `flagsByAction` on its CLI schema, mapping + each action to the options it reads, and derives `allowedFlags` from that table. The parser then + refuses a typed option its action never reads, off the keys the caller provided: config, env, and + remote-config defaults fill `flags` without becoming a request, so `AGENT_DEVICE_FPS=30` never breaks + `record stop`. Keep an option every action reads out of the table — a row list is the only set the + parser refuses, and refusing `--json` would refuse the command. - Command-specific usage/flag metadata lives with the command family metadata that owns the command. - Parser/help *rendering* stays in `src/cli/parser/`; command schema metadata is derived from command metadata, family declarations, and the schema-only merge path in diff --git a/packages/capture-kit/package.json b/packages/capture-kit/package.json index 139662c3d2..f22fb90cc9 100644 --- a/packages/capture-kit/package.json +++ b/packages/capture-kit/package.json @@ -166,6 +166,10 @@ "types": "./src/recording/artifact.fixtures.ts", "default": "./src/recording/artifact.fixtures.ts" }, + "./recording-contact-sheet": { + "types": "./src/recording/contact-sheet.ts", + "default": "./src/recording/contact-sheet.ts" + }, "./recording-facts": { "types": "./src/recording/recording-facts.ts", "default": "./src/recording/recording-facts.ts" diff --git a/packages/capture-kit/src/recording/contact-sheet-report.ts b/packages/capture-kit/src/recording/contact-sheet-report.ts index cfbd653551..6647c10240 100644 --- a/packages/capture-kit/src/recording/contact-sheet-report.ts +++ b/packages/capture-kit/src/recording/contact-sheet-report.ts @@ -21,6 +21,8 @@ export const CONTACT_SHEET_CONTAINER_REASON = 'contact_sheet_container_unsupport export const CONTACT_SHEET_OUTPUT_WRITE_REASON = 'contact_sheet_output_write_failed'; /** The output resolves to the recording itself, which a sheet must never replace. */ export const CONTACT_SHEET_OUTPUT_COLLISION_REASON = 'contact_sheet_output_collides_with_input'; +/** The change threshold is not a finite share between 0 and 1. */ +export const CONTACT_SHEET_THRESHOLD_REASON = 'contact_sheet_threshold_invalid'; export type RecordingContactSheetCell = { /** Presentation time of the decoded frame this cell shows, in milliseconds from the clip start. */ diff --git a/packages/capture-kit/src/recording/contact-sheet.ts b/packages/capture-kit/src/recording/contact-sheet.ts index 8510a73401..131f9859f2 100644 --- a/packages/capture-kit/src/recording/contact-sheet.ts +++ b/packages/capture-kit/src/recording/contact-sheet.ts @@ -32,6 +32,15 @@ import { readVideoContainerKind } from './video.ts'; export { recordingContactSheetPath }; +/** + * Publishes what a caller has to name to read the answer: the report the pipeline returns, and the one + * refusal reason a caller can act on by retrying with another threshold. Every other reason already + * travels on the error the pipeline threw, so it stays in the module that raises it until some surface + * asks for it by name. + */ +export { CONTACT_SHEET_THRESHOLD_REASON } from './contact-sheet-report.ts'; +export type { RecordingContactSheetResult } from './contact-sheet-report.ts'; + /** * Builds the one PNG that lets a caller read a recording without playing it. * diff --git a/packages/command-registry/src/command-schema.ts b/packages/command-registry/src/command-schema.ts index c4e03c1fd4..50ab6793eb 100644 --- a/packages/command-registry/src/command-schema.ts +++ b/packages/command-registry/src/command-schema.ts @@ -10,6 +10,17 @@ export type CommandSchema = { positionalArgs?: readonly string[]; allowsExtraPositionals?: boolean; allowedFlags?: readonly FlagKey[]; + /** + * For a command whose first positional selects an action: the options that action reads. + * + * `allowedFlags` is command-wide, because the parser does not know actions, so every option the + * command declares reaches every one of its actions. An option the action never reads is then + * dropped without a word: `record start --out take.mp4` looks like it recorded there, and + * `record contact-sheet clip.mp4 --fps 30` looks like it sampled frames. Declaring the split lets + * the parser refuse a typed option its action cannot read — on the keys the caller typed, never on + * the config, env, or remote-config defaults that also fill the flag bag. + */ + flagsByAction?: Readonly>; supportedFlags?: readonly FlagKey[]; /** * Replaces the generated synopsis grammar in `--help`, for shapes the generator cannot express. diff --git a/packages/kernel/src/contracts.ts b/packages/kernel/src/contracts.ts index 3e3fedc3dc..1ef172702a 100644 --- a/packages/kernel/src/contracts.ts +++ b/packages/kernel/src/contracts.ts @@ -125,6 +125,7 @@ export type DaemonArtifactKnownType = | 'screenshot-diff' | 'screen-recording' | 'screen-recording-chunk' + | 'screen-recording-contact-sheet' | 'screen-recording-telemetry' | 'trace-log' | 'test-artifacts'; diff --git a/scripts/__tests__/eager-closure-budgets.ts b/scripts/__tests__/eager-closure-budgets.ts index 337a4b6e5a..c59dd07e78 100644 --- a/scripts/__tests__/eager-closure-budgets.ts +++ b/scripts/__tests__/eager-closure-budgets.ts @@ -127,7 +127,14 @@ export const NEW_ENTRY_CEILINGS: Readonly> = Objec */ export const APPROVED_OVER_CEILING: Readonly< Record -> = Object.freeze({}); +> = Object.freeze({ + 'packages/capture-kit/src/recording/contact-sheet.ts': { + issue: '#2747', + reason: + 'Reading a recording through one PNG needs the sample grid, the frame decoder seam, and the PNG codec behind a single surface. Every module it evaluates is capture-kit mechanics, so the property the ceiling guards — no façade evaluating a concrete platform implementation — holds; only its weight is above the p75 of façades that answer a narrower question.', + owner: 'capture-kit', + }, +}); /** The category is a function of the path, never a hand-written column. */ export function entryCategoryOf(entryFile: string): EntryCategory { diff --git a/scripts/layering/package-boundaries.test.ts b/scripts/layering/package-boundaries.test.ts index dd3402c096..d852129882 100644 --- a/scripts/layering/package-boundaries.test.ts +++ b/scripts/layering/package-boundaries.test.ts @@ -423,6 +423,7 @@ test('the real tree parses, declares, and passes R11', () => { '@agent-device/capture-kit/quality-warnings', '@agent-device/capture-kit/react-native-overlay', '@agent-device/capture-kit/recording-artifact-fixtures', + '@agent-device/capture-kit/recording-contact-sheet', '@agent-device/capture-kit/recording-facts', '@agent-device/capture-kit/recording-mp4-duration', '@agent-device/capture-kit/recording-mp4-fixtures', diff --git a/src/cli/commands/__tests__/recording.test.ts b/src/cli/commands/__tests__/recording.test.ts new file mode 100644 index 0000000000..72963b7106 --- /dev/null +++ b/src/cli/commands/__tests__/recording.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, test, vi } from 'vitest'; +import { AppError } from '@agent-device/kernel/errors'; +import type { ClientCommandParams } from '../router-types.ts'; +import { recordingCommand } from '../recording.ts'; + +function params(positionals: string[]): ClientCommandParams { + return { + positionals, + flags: {} as ClientCommandParams['flags'], + client: {} as ClientCommandParams['client'], + }; +} + +async function failure(action: () => Promise): Promise { + try { + await action(); + } catch (error) { + return error; + } + return 'resolved without an error'; +} + +describe('record contact-sheet CLI action', () => { + test('declines every action the generic route owns', async () => { + const handled = await recordingCommand({ + ...params(['start', '/tmp/recording.mp4']), + client: { recording: { record: vi.fn() } } as unknown as ClientCommandParams['client'], + }); + + expect(handled).toBe(false); + }); + + test('refuses a contact sheet with no recording to read', async () => { + const error = await failure(() => recordingCommand(params(['contact-sheet']))); + + expect(error).toBeInstanceOf(AppError); + expect((error as AppError).code).toBe('INVALID_ARGS'); + expect((error as AppError).message).toMatch(/requires a recording path/); + }); + + test('refuses more than one recording path', async () => { + const error = await failure(() => + recordingCommand(params(['contact-sheet', '/tmp/a.mp4', '/tmp/b.mp4'])), + ); + + expect((error as AppError).message).toMatch(/one recording path/); + }); +}); diff --git a/src/cli/commands/recording.ts b/src/cli/commands/recording.ts new file mode 100644 index 0000000000..286abfb649 --- /dev/null +++ b/src/cli/commands/recording.ts @@ -0,0 +1,64 @@ +import { AppError } from '@agent-device/kernel/errors'; +import type { RecordingContactSheetCommandResult } from '../../commands/recording/runtime/contact-sheet.ts'; +import { resolveUserPath } from '@agent-device/host-kit/file'; +import { writeCommandOutput } from './shared.ts'; +import type { ClientCommandHandler } from './router-types.ts'; + +/** The `record` action that reads a finished file locally instead of asking a device for anything. */ +const CONTACT_SHEET_ACTION = 'contact-sheet'; + +export const recordingCommand: ClientCommandHandler = async ({ positionals, flags }) => { + if (positionals[0] !== CONTACT_SHEET_ACTION) return false; + if (positionals.length > 2) { + throw new AppError( + 'INVALID_ARGS', + 'record contact-sheet accepts one recording path: record contact-sheet [--out ]', + ); + } + + const recordingPath = resolveUserPath(readRequiredPositional(positionals[1])); + const outputPath = typeof flags.out === 'string' ? resolveUserPath(flags.out) : undefined; + + // Lazy: createAgentDevice pulls in the client-side command runtime, which only this action needs. + const [{ createAgentDevice, localCommandPolicy }, { createLocalArtifactAdapter }] = + await Promise.all([import('../../runtime.ts'), import('../../io.ts')]); + const runtime = createAgentDevice({ + backend: { platform: 'ios' }, + artifacts: createLocalArtifactAdapter(), + sessions: { + get: (name) => ({ name }), + set: () => {}, + }, + policy: localCommandPolicy(), + }); + + const result = await runtime.recording.contactSheet({ + video: { kind: 'path', path: recordingPath }, + ...(outputPath ? { out: { kind: 'path', path: outputPath } } : {}), + }); + + await writeCommandOutput(flags, result, () => formatContactSheetSummary(result)); + return true; +}; + +function readRequiredPositional(value: string | undefined): string { + if (value) return value; + throw new AppError( + 'INVALID_ARGS', + 'record contact-sheet requires a recording path: record contact-sheet ', + ); +} + +/** + * Leads with the path so a caller can capture it the way it captures `record stop`, then says how + * much of the recording the sheet actually covers — a full grid is not the same claim as a review. + */ +function formatContactSheetSummary(result: RecordingContactSheetCommandResult): string { + const seconds = (result.durationMs / 1000).toFixed(result.durationMs % 1000 === 0 ? 0 : 1); + const lines = [ + result.path, + `${result.cells.length} cells over ${seconds}s ` + + `(${result.sampledFrameCount} sample times, ${result.decodedFrameCount} frames decoded)`, + ]; + return result.warning ? [...lines, result.warning].join('\n') : lines.join('\n'); +} diff --git a/src/cli/commands/router.ts b/src/cli/commands/router.ts index 0d717d52cf..38a7a6afd1 100644 --- a/src/cli/commands/router.ts +++ b/src/cli/commands/router.ts @@ -20,6 +20,7 @@ const dedicatedCliCommandHandlerLoaders = { replay: async () => (await import('./replay.ts')).replayCommand, screenshot: async () => (await import('./screenshot.ts')).screenshotCommand, diff: async () => (await import('./screenshot.ts')).diffCommand, + record: async () => (await import('./recording.ts')).recordingCommand, } satisfies ClientCommandHandlerMap; export async function tryRunClientBackedCommand(params: { diff --git a/src/cli/parser/__tests__/args-validation.test.ts b/src/cli/parser/__tests__/args-validation.test.ts index 2f729d841d..c08b39976a 100644 --- a/src/cli/parser/__tests__/args-validation.test.ts +++ b/src/cli/parser/__tests__/args-validation.test.ts @@ -32,6 +32,65 @@ test('parseArgs rejects invalid record --fps range', () => { ); }); +test('parseArgs refuses a record option the chosen action never reads', () => { + const cases: Array<[string[], string]> = [ + [ + ['record', 'start', './capture.mp4', '--out', './elsewhere.mp4'], + 'record start does not read --out', + ], + [['record', 'stop', '--fps', '30'], 'record stop does not read --fps'], + [['record', 'stop', '--hide-touches'], 'record stop does not read --hide-touches'], + [ + ['record', 'contact-sheet', './clip.mp4', '--fps', '30'], + 'record contact-sheet does not read --fps', + ], + ]; + + for (const [argv, message] of cases) { + assert.throws( + () => parseArgs(argv, { strictFlags: true }), + (error: unknown) => + error instanceof AppError && + error.code === 'INVALID_ARGS' && + error.message.startsWith(message), + `expected ${argv.join(' ')} to be refused`, + ); + } +}); + +test('parseArgs accepts the options each record action reads', () => { + const cases: string[][] = [ + ['record', 'start', './capture.mp4', '--fps', '30', '--quality', 'high', '--hide-touches'], + ['record', 'stop'], + ['record', 'contact-sheet', './clip.mp4', '--out', './sheet.png'], + ]; + + for (const argv of cases) { + assert.doesNotThrow(() => parseArgs(argv, { strictFlags: true }), `expected ${argv.join(' ')}`); + } +}); + +test('parseArgs leaves an option every action reads to the command itself', () => { + // --json and --no-record are read whatever action follows the command name, so an action table that + // does not list them must not turn them into unread options. + for (const argv of [ + ['record', 'stop', '--json'], + ['record', 'contact-sheet', './clip.mp4', '--no-record'], + ]) { + assert.doesNotThrow(() => parseArgs(argv, { strictFlags: true }), `expected ${argv.join(' ')}`); + } +}); + +test('parseArgs leaves a command without an action table to its own validation', () => { + // `perf` declares no action option table at all, and `record nonsense` names an action the record + // table never lists. Neither may be refused as an unread option here: the first is validated by its + // own command, and an action the table does not list belongs to the reader that owns the action list. + assert.doesNotThrow(() => parseArgs(['perf', 'start', './perf.json'], { strictFlags: true })); + assert.doesNotThrow(() => + parseArgs(['record', 'nonsense', '--fps', '30'], { strictFlags: true }), + ); +}); + test('parseArgs rejects invalid swipe pattern', () => { assert.throws( () => parseArgs(['swipe', '0', '0', '10', '10', '--pattern', 'diagonal']), diff --git a/src/cli/parser/args.ts b/src/cli/parser/args.ts index fae48dad9a..cf7cc19230 100644 --- a/src/cli/parser/args.ts +++ b/src/cli/parser/args.ts @@ -197,6 +197,18 @@ export function finalizeParsedArgs( delete (flags as Record)[entry.key]; } } + + const unread = findFlagsTheActionCannotRead(parsed); + if (unread.length > 0) { + const message = formatUnreadActionFlagMessage(parsed.command, parsed.positionals[0]!, unread); + if (strictFlags) { + throw new AppError('INVALID_ARGS', message); + } + warnings.push(message); + for (const entry of unread) { + delete (flags as Record)[entry.key]; + } + } for (const key of Object.keys(flags) as FlagKey[]) { if (flags[key] === undefined) continue; if (!isFlagSupportedForCommand(key, parsed.command)) { @@ -373,6 +385,38 @@ function normalizeParsedCommandAliases(parsed: ParsedArgs): ParsedArgs { return parsed; } +/** + * The typed options the selected action of a command cannot read. + * + * Only the options the table splits across actions are its business: a global flag such as `--json` + * or `--no-record` is read by every action, and refusing one would refuse the command itself. A + * command without the table, or an action it does not list, is left to the rest of the parser. + * + * Config, env, and remote-config defaults never appear in `providedFlags`, which is why + * `AGENT_DEVICE_FPS=30` does not fail `record stop`: a default the action ignores was never requested. + */ +function findFlagsTheActionCannotRead(parsed: RawParsedArgs): ParsedFlagRecord[] { + const flagsByAction = getCommandSchema(parsed.command)?.flagsByAction; + const action = parsed.positionals[0]; + if (flagsByAction === undefined || action === undefined) return []; + if (!Object.hasOwn(flagsByAction, action)) return []; + const reads = flagsByAction[action]; + if (reads === undefined) return []; + const actionScoped = new Set(Object.values(flagsByAction).flat()); + return parsed.providedFlags.filter( + (entry) => actionScoped.has(entry.key) && !reads.includes(entry.key), + ); +} + +function formatUnreadActionFlagMessage( + command: string | null, + action: string, + unread: ParsedFlagRecord[], +): string { + const tokens = unread.map((entry) => entry.token).join(', '); + return `${command} ${action} does not read ${tokens}. Run \`${command} ${action} --help\` for the options it reads.`; +} + function formatUnsupportedFlagMessage(command: string | null, unsupported: string[]): string { if (!command) { return unsupported.length === 1 diff --git a/src/cli/resolve-cli-options.test.ts b/src/cli/resolve-cli-options.test.ts new file mode 100644 index 0000000000..31e6d293b8 --- /dev/null +++ b/src/cli/resolve-cli-options.test.ts @@ -0,0 +1,40 @@ +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import { mkdtempForTestSync } from '../__tests__/test-utils/tmp-dir.ts'; +import { resolveCliOptions } from './resolve-cli-options.ts'; + +/** + * Points `~` at an empty directory so a developer's own ~/.agent-device/config.json cannot add + * defaults to what a test asserts, and keeps PATH so the rest of resolution behaves. + */ +function isolatedEnv(env: Record): Record { + const home = mkdtempForTestSync('agent-device-cli-env-'); + return { HOME: home, USERPROFILE: home, PATH: process.env.PATH ?? '', ...env }; +} + +test('a frame rate from the environment is not a flag the caller typed', () => { + // `record stop` reads no recording option. A default it ignores must not become a refusal, or + // AGENT_DEVICE_FPS in a CI shell would break every `record stop`. + const parsed = resolveCliOptions(['record', 'stop'], { + cwd: process.cwd(), + env: isolatedEnv({ AGENT_DEVICE_FPS: '30' }), + }); + + assert.equal(parsed.flags.fps, 30); + assert.deepEqual( + parsed.providedFlags.map((entry) => entry.key), + [], + ); +}); + +test('a frame rate the caller typed stays a typed flag', () => { + const parsed = resolveCliOptions(['record', 'start', './capture.mp4', '--fps', '30'], { + cwd: process.cwd(), + env: isolatedEnv({}), + }); + + assert.deepEqual( + parsed.providedFlags.map((entry) => entry.key), + ['fps'], + ); +}); diff --git a/src/commands/recording/index.test.ts b/src/commands/recording/index.test.ts index be24156946..55be43f6c6 100644 --- a/src/commands/recording/index.test.ts +++ b/src/commands/recording/index.test.ts @@ -1,5 +1,6 @@ import { describe, expect, test } from 'vitest'; import type { CliFlags } from '@agent-device/contracts/command'; +import { getCliCommandSchema } from '../schema/command-schema.ts'; import { recordCliReader, recordCommandFacet, @@ -75,3 +76,14 @@ describe('recording command interface', () => { }); }); }); + +describe('record CLI option declaration', () => { + test('hands the parser the table of which action reads which option', () => { + // The parser is what refuses an option its action cannot read, on the keys the caller typed. That + // holds only while this family's table reaches the schema the parser reads. + const reads = getCliCommandSchema('record').flagsByAction; + + expect(reads?.['stop']).toEqual([]); + expect(reads?.['contact-sheet']).toContain('out'); + }); +}); diff --git a/src/commands/recording/index.ts b/src/commands/recording/index.ts index 9168864416..62435afd63 100644 --- a/src/commands/recording/index.ts +++ b/src/commands/recording/index.ts @@ -9,6 +9,7 @@ import { } from '@agent-device/contracts/recording'; import { AppError } from '@agent-device/kernel/errors'; import type { CommandSchemaOverride } from '@agent-device/command-registry/command-schema'; +import type { FlagKey } from '@agent-device/command-registry/flag-types'; import { commonInputFromFlags, direct, optionalString } from '../cli-grammar/common.ts'; import type { CliReader, DaemonWriter } from '../cli-grammar/types.ts'; import { @@ -28,7 +29,7 @@ const TRACE_COMMAND_NAME = 'trace'; const RECORDING_ACTION_VALUES = ['start', 'stop'] as const; const recordCommandDescription = - 'Start or stop a screen recording for the active app session or, where supported, the selected device. Long Android recordings can return multiple video artifacts; HarmonyOS supports whole-screen recording on physical devices.'; + 'Start or stop a screen recording for the active app session or, where supported, the selected device, or build a contact-sheet PNG from a recording already exported. Long Android recordings can return multiple video artifacts; HarmonyOS supports whole-screen recording on physical devices.'; const traceCommandDescription = 'Start or stop trace-log capture and return the resulting artifact when capture ends. Use the same artifact path for the matching start and stop requests when an explicit path is required.'; @@ -58,13 +59,33 @@ export const traceCommandMetadata = defineFieldCommandMetadata( }, ); +/** + * Which `record` action reads which option. + * + * A new option has to name the action that reads it. That is what keeps one shared list from letting + * `record start --out take.mp4` look like it recorded to that path, and `record contact-sheet + * clip.mp4 --fps 30` look like frame sampling was configured. The parser is handed this table as + * `flagsByAction` and refuses an option the action cannot read. + */ +const RECORD_FLAGS_BY_ACTION: Readonly> = { + start: ['recordingScope', 'fps', 'quality', 'hideTouches'], + // `record stop` takes a session and a target, and reads no recording option. + stop: [], + 'contact-sheet': ['out'], +}; + +const RECORD_CLI_FLAGS: readonly FlagKey[] = [ + ...new Set(Object.values(RECORD_FLAGS_BY_ACTION).flat()), +]; + const recordCliSchema = { usageOverride: - 'record start [path] [--scope ] [--fps ] [--quality ] [--hide-touches] | record stop', + 'record start [path] [--scope ] [--fps ] [--quality ] [--hide-touches] | record stop | record contact-sheet [--out ]', usageFlags: [], - listUsageOverride: 'record start [path] | record stop', - positionalArgs: ['start|stop', 'path?'], - allowedFlags: ['recordingScope', 'fps', 'quality', 'hideTouches'], + listUsageOverride: 'record start [path] | record stop | record contact-sheet