Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -542,7 +542,7 @@ extension RunnerTests {
return sparseTruncatedSnapshotPayload(
message: recoveredSnapshotMessage(failure),
snapshotQuality: SnapshotQuality(
state: "sparse",
state: .sparse,
backend: SnapshotBackendKind.recursiveTree.rawValue,
reason: failure.message,
reasonCode: "ax-rejected",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,21 @@ import AgentDeviceSnapshotPresentation
// stamp the outcome with a structured quality verdict so the daemon renders state instead of
// re-deriving it from node shapes. Recovery ordering is data here, never a per-call-site branch.

/// The closed set of verdict states the host accepts. The wire strings are the shared table at
/// `contracts/fixtures/ios-snapshot-quality-states.json`, which `allCases` is pinned to; `reasonCode`
/// stays open because an unknown one costs only its wording, never the verdict.
enum SnapshotQualityState: String, Codable, CaseIterable {
/// First backend produced a usable tree.
case healthy
/// A later backend did.
case recovered
/// No backend produced a usable tree; the best attempt is returned as-is.
case sparse
}

/// Structured quality verdict shipped with every iOS snapshot payload.
struct SnapshotQuality: Codable {
/// healthy: first backend produced a usable tree. recovered: a later backend did.
/// sparse: no backend produced a usable tree; the best attempt is returned as-is.
let state: String
let state: SnapshotQualityState
/// Backend that produced the returned payload: tree | queries | private-ax.
let backend: String
/// Why recovery ran (first failure), why the payload is degraded, or why an internal backend
Expand Down Expand Up @@ -391,7 +401,7 @@ extension RunnerTests {
return stampedSnapshotPayload(
capture,
backend: kind,
state: recovered ? "recovered" : "healthy",
state: recovered ? .recovered : .healthy,
reason: recovered || firstFailure?.code == "requested-backend" ? firstFailure : nil
)
}
Expand All @@ -416,11 +426,11 @@ extension RunnerTests {
}

let fallbackPayload =
best.map { stampedSnapshotPayload($0.capture, backend: $0.kind, state: "sparse", reason: firstFailure) }
best.map { stampedSnapshotPayload($0.capture, backend: $0.kind, state: .sparse, reason: firstFailure) }
?? stampedSnapshotPayload(
SnapshotBackendCapture(payload: sparseTruncatedSnapshotPayload(), effectiveDepth: nil),
backend: effectivePlan.last ?? plan.last ?? .recursiveTree,
state: "sparse",
state: .sparse,
reason: firstFailure
)
return fallbackPayload
Expand Down Expand Up @@ -680,7 +690,7 @@ extension RunnerTests {
func stampedSnapshotPayload(
_ capture: SnapshotBackendCapture,
backend: SnapshotBackendKind,
state: String,
state: SnapshotQualityState,
reason: (reason: String, code: String)?
) -> DataPayload {
let health: RunnerAccessibilityHealth = reason?.code == "ax-rejected" ? .unavailable : .healthy
Expand All @@ -705,7 +715,7 @@ extension RunnerTests {
// "recovered") stays untruncated, so strict absence reads can trust it. Only a real cap
// (payload truncation, a depth-limited private AX capture) or a sparse terminal payload
// is truncated.
truncated: payload.truncated == true || state == "sparse" || capture.effectiveDepth != nil,
truncated: payload.truncated == true || state == .sparse || capture.effectiveDepth != nil,
qualityPayload: capture.qualityPayload.flatMap { quality in
guard let nodes = quality.nodes else { return nil }
return SnapshotQualityPayload(nodes: nodes, truncated: quality.truncated == true)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,7 @@ extension RunnerTests {
XCTAssertNil(box.error)
let quality = try XCTUnwrap(box.payload?.snapshotQuality)
XCTAssertEqual(quality.backend, SnapshotBackendKind.privateAX.rawValue)
XCTAssertEqual(quality.state, "recovered")
XCTAssertEqual(quality.state, .recovered)
XCTAssertTrue(
quality.reason?.contains("tree capture exceeded") == true,
"the tree XPC, not the viewport read, must be the abandoned block: \(quality.reason ?? "nil")"
Expand Down Expand Up @@ -345,7 +345,7 @@ extension RunnerTests {
SnapshotBackendKind.privateAX.rawValue,
"a sweep that ended on its slice deadline is a tier timeout, not an accepted capture"
)
XCTAssertEqual(quality?.state, "recovered")
XCTAssertEqual(quality?.state, .recovered)
XCTAssertGreaterThan(box.payload?.nodes?.count ?? 0, 1, "private AX answers with a real tree")
XCTAssertFalse(box.abandonedAtReturn, "the sweep must answer inside its own main-thread hop")
XCTAssertTrue(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,7 @@ extension RunnerTests {
let payload = stampedSnapshotPayload(
capture,
backend: .recursiveTree,
state: "healthy",
state: .healthy,
reason: nil
)

Expand All @@ -187,7 +187,7 @@ extension RunnerTests {
customActions: coverage
),
backend: .recursiveTree,
state: "healthy",
state: .healthy,
reason: nil
)
XCTAssertNil(silent.message)
Expand All @@ -200,7 +200,7 @@ extension RunnerTests {
effectiveDepth: 4
),
backend: .privateAX,
state: "recovered",
state: .recovered,
reason: (reason: "tree capture timed out", code: "budget")
)
XCTAssertEqual(underlying.message, "underlying")
Expand All @@ -224,24 +224,24 @@ extension RunnerTests {
// The CI signature behind `is absent ... capture was truncated`: a complete private AX
// tree selected while the XCTest channel is penalized is whole, and must say so.
let recovered = stampedSnapshotPayload(
complete, backend: .privateAX, state: "recovered", reason: deferred)
XCTAssertEqual(recovered.snapshotQuality?.state, "recovered")
complete, backend: .privateAX, state: .recovered, reason: deferred)
XCTAssertEqual(recovered.snapshotQuality?.state, .recovered)
XCTAssertEqual(recovered.truncated, false)

let depthLimited = stampedSnapshotPayload(
SnapshotBackendCapture(payload: complete.payload, effectiveDepth: 56),
backend: .privateAX, state: "recovered", reason: deferred)
backend: .privateAX, state: .recovered, reason: deferred)
XCTAssertEqual(depthLimited.truncated, true)

let cappedPayload = stampedSnapshotPayload(
SnapshotBackendCapture(
payload: DataPayload(nodes: complete.payload.nodes ?? [], truncated: true),
effectiveDepth: nil),
backend: .recursiveTree, state: "healthy", reason: nil)
backend: .recursiveTree, state: .healthy, reason: nil)
XCTAssertEqual(cappedPayload.truncated, true)

let sparse = stampedSnapshotPayload(
complete, backend: .querySweep, state: "sparse",
complete, backend: .querySweep, state: .sparse,
reason: ("snapshot returned no semantic controls or content", "sparse-tree"))
XCTAssertEqual(sparse.truncated, true)
}
Expand All @@ -260,7 +260,7 @@ extension RunnerTests {
let payload = stampedSnapshotPayload(
capture,
backend: .recursiveTree,
state: "healthy",
state: .healthy,
reason: nil
)

Expand Down Expand Up @@ -291,7 +291,7 @@ extension RunnerTests {
let payload = stampedSnapshotPayload(
capture,
backend: .recursiveTree,
state: "healthy",
state: .healthy,
reason: nil
)

Expand Down Expand Up @@ -507,7 +507,7 @@ extension RunnerTests {

let quality = try XCTUnwrap(capped.snapshotQuality)
XCTAssertEqual(quality.backend, SnapshotBackendKind.privateAX.rawValue)
XCTAssertNotEqual(quality.state, "sparse")
XCTAssertNotEqual(quality.state, .sparse)
let nodes = try XCTUnwrap(capped.nodes)
XCTAssertGreaterThan(nodes.count, 1)
XCTAssertEqual(nodes.map(\.depth).max(), 1)
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
import XCTest

#if AGENT_DEVICE_RUNNER_UNIT_TESTS
extension RunnerTests {
private struct StampedWireVerdict: Decodable {
struct Quality: Decodable {
let state: String
}
let snapshotQuality: Quality
}

private func loadSnapshotQualityStatesFixture() throws -> [String] {
let fixtureURL = URL(fileURLWithPath: #filePath)
.deletingLastPathComponent() // UnitTests
.deletingLastPathComponent() // AgentDeviceRunnerUITests
.deletingLastPathComponent() // AgentDeviceRunner
.deletingLastPathComponent() // runner
.deletingLastPathComponent() // apple
.deletingLastPathComponent() // repo root
.appendingPathComponent("contracts")
.appendingPathComponent("fixtures")
.appendingPathComponent("ios-snapshot-quality-states.json")
return try JSONDecoder().decode([String].self, from: Data(contentsOf: fixtureURL))
}

/// The one claim of this file: the runner's closed enum and the shared TypeScript table name the
/// same states. The kernel's `SNAPSHOT_QUALITY_STATES` is pinned to it too, so the two runtimes
/// cannot drift into a verdict the host drops along with its disclosure. Compared as a set: the
/// names are the contract, and a reordering of `allCases` cannot produce a wrong verdict.
func testSnapshotQualityStatesMatchSharedWireFixture() throws {
XCTAssertEqual(
Set(try loadSnapshotQualityStatesFixture()),
Set(SnapshotQualityState.allCases.map(\.rawValue)),
"update the fixture and the kernel tuple together with the enum"
)
}

/// What the daemon receives for each state, taken from the production stamping path rather than a
/// hand-built verdict: the wire string is the case's own raw value, so a change of
/// representation — an `Int` backing, a nested object — goes red here on the actual payload, and
/// a renamed raw value goes red in the fixture test above.
func testStampedVerdictEncodesTheCaseRawValue() throws {
let capture = SnapshotBackendCapture(
payload: DataPayload(nodes: [], truncated: false),
effectiveDepth: nil
)
for state in SnapshotQualityState.allCases {
let payload = stampedSnapshotPayload(
capture,
backend: .recursiveTree,
state: state,
reason: nil
)
let wire = try JSONDecoder().decode(
StampedWireVerdict.self,
from: JSONEncoder().encode(payload)
)
XCTAssertEqual(wire.snapshotQuality.state, state.rawValue)
}
}

/// Closed in both directions: a wire string nobody declared never becomes a verdict.
func testVerdictStateRejectsAnUndeclaredWireString() throws {
let json = Data(#"{"state":"degraded","backend":"tree"}"#.utf8)
XCTAssertThrowsError(try JSONDecoder().decode(SnapshotQuality.self, from: json))
}
}
#endif
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ extension RunnerTests {
XCTAssertEqual(payload.runnerFatalReason, Self.axSnapshotUnavailableReason)
// The planned terminal result carries the structured verdict like every other planned
// snapshot — downstream sparse handling keys off it, not off node shapes.
XCTAssertEqual(payload.snapshotQuality?.state, "sparse")
XCTAssertEqual(payload.snapshotQuality?.state, .sparse)
XCTAssertEqual(payload.snapshotQuality?.reasonCode, "ax-rejected")
XCTAssertEqual(payload.snapshotQuality?.reason, Self.axSnapshotFailureMessage)
XCTAssertNil(currentApp)
Expand Down
1 change: 1 addition & 0 deletions contracts/fixtures/ios-snapshot-quality-states.json
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
["healthy", "recovered", "sparse"]
58 changes: 58 additions & 0 deletions packages/capture-kit/src/snapshot-quality-verdict.test.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
import { test } from 'vitest';
import assert from 'node:assert/strict';

import { readSerializedSnapshotCaptureAnnotations } from '@agent-device/contracts/capture';
import { SNAPSHOT_QUALITY_STATES } from '@agent-device/kernel/snapshot';
import {
isSparseSnapshotQualityVerdict,
preferredSnapshotBackendForVerdict,
Expand Down Expand Up @@ -47,6 +49,22 @@ test('readSnapshotQualityVerdict rejects unknown state or backend as verdict-abs
assert.equal(readSnapshotQualityVerdict({ state: 'sparse', backend: 'mystery' }), undefined);
assert.equal(readSnapshotQualityVerdict({ backend: 'tree' }), undefined);
assert.equal(readSnapshotQualityVerdict(null), undefined);
// An inherited key is not a declared state: membership stays on the map's own keys.
assert.equal(readSnapshotQualityVerdict({ state: 'constructor', backend: 'tree' }), undefined);
});

test('readSnapshotQualityVerdict reads every declared wire state', () => {
for (const state of SNAPSHOT_QUALITY_STATES) {
assert.deepEqual(readSnapshotQualityVerdict({ state, backend: 'tree' }), {
state,
backend: 'tree',
reason: undefined,
reasonCode: undefined,
customActions: undefined,
effectiveDepth: undefined,
collapsedLeafIndexes: undefined,
});
}
});

test('readSnapshotQualityVerdict keeps the verdict but drops an unknown reasonCode', () => {
Expand Down Expand Up @@ -98,3 +116,43 @@ test('preferredSnapshotBackendForVerdict pins only private-ax captures', () => {
);
assert.equal(preferredSnapshotBackendForVerdict(undefined), undefined);
});

/**
* Two readings of one verdict exist on purpose: this module normalizes an untrusted runner payload,
* while contracts re-publishes what this repo published and normalizes nothing (the eager-closure
* gate forbids either reaching a shared module, and the duplication gate refuses a second
* normalization). They must still agree on which payloads are a verdict at all: a name one version
* cannot speak is verdict-absent on both sides of the daemon boundary.
*/
const VERDICT_PAYLOADS: unknown[] = [
{ state: 'sparse', backend: 'private-ax' },
{ state: 'healthy', backend: 'tree', reason: 'ok', reasonCode: 'requested-backend' },
{ state: 'recovered', backend: 'queries', reason: 42, effectiveDepth: '56' },
{ state: 'sparse', backend: 'tree', collapsedLeafIndexes: [3, 'four'] },
{ state: 'sparse', backend: 'tree', customActions: { read: 12 } },
{ state: 'sparse', backend: 'tree', customActions: { read: 12, candidates: 19 } },
{ state: 'sparse', backend: 'tree', timing: { acquisitionMs: 12.5 } },
{ state: 'sparse', backend: 'tree', timing: { acquisitionMs: 12.5, presentationMs: 34.75 } },
{ state: 'sparse', backend: 'tree', reasonCode: 'future-code' },
{ state: 'recovered', backend: 'android-helper', reasonCode: 'requested-backend' },
{ state: 'degraded', backend: 'tree' },
{ state: 'sparse', backend: 'uiautomator' },
{ state: 'constructor', backend: 'constructor' },
{ backend: 'tree' },
{ state: 'sparse' },
null,
'verdict',
];

test('the contracts re-read calls a verdict a verdict on every payload', () => {
for (const payload of VERDICT_PAYLOADS) {
const reRead = readSerializedSnapshotCaptureAnnotations({
snapshotQuality: payload,
}).snapshotQuality;
assert.equal(
reRead === undefined,
readSnapshotQualityVerdict(payload) === undefined,
JSON.stringify(payload),
);
}
});
Loading
Loading