From 4b0d14678d3a10bca023d4a2de275b8abf3b2bde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 24 Sep 2026 15:03:55 +0200 Subject: [PATCH 1/8] fix(ios-runner): bound the synthesized replacement pace so a value-owning field keeps it A replacement burst typed at XCTest's default 60 characters/second outruns an app that owns its field's value and re-applies it after the edit that produced it, which is how a controlled React Native `TextInput` behaves. That write erases whatever the burst typed while it was in flight and the field then sits stable short of the request: first characters and tail survive, a middle run is gone, which is the `fill id="field-email" ada@example` -> `aexample` CI signature. The commit wait can refuse such a value but cannot repair it, because a retype races the same write. 12 characters/second spaces characters ~83 ms apart. The new `--agent-device-text-entry-async-value-write` fixture owns its field's value and the iOS lane test pins the pace against it: raising the pace re-opens the race and turns that test red. A write-back still in flight 150 ms after an edit corrupts a burst at this pace too, so this narrows the window rather than closing it, and the measured wall-clock cost of the pace is recorded next to the commit ceiling it sits upstream of. Closes #2080 --- .../AgentDeviceRunner/AgentDeviceRunnerApp.m | 51 ++++++++++++++ .../RunnerSynthesizedTextEntry.h | 4 +- .../RunnerSynthesizedTextEntry.m | 20 +++++- .../RunnerTests+TextEntry.swift | 4 +- ...unnerTests+SynthesizedTextEntryTests.swift | 66 +++++++++++++++++++ .../RunnerTests+TextEntryPolicyTests.swift | 9 +-- 6 files changed, 146 insertions(+), 8 deletions(-) create mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m index b9cd0b9005..2406eb1e5f 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m @@ -65,6 +65,9 @@ @interface AgentDeviceRunnerViewController : UIViewController @property(nonatomic, strong) UILabel *alertActivationBusyAnswer; @property(nonatomic, assign) NSUInteger firstAlertActions; @property(nonatomic, assign) NSUInteger replacementAlertActions; +@property(nonatomic, strong) UILabel *textEntryWriteBackStatus; +@property(nonatomic, assign) NSUInteger textEntryWriteBackAttempts; +@property(nonatomic, assign) NSUInteger textEntryWriteBackApplies; @property(nonatomic, assign) BOOL alertFixtureStarted; @property(nonatomic, strong) NSTimer *alertActivationBusyBackstop; @property(nonatomic, strong) NSTimer *alertBannerRepost; @@ -175,6 +178,12 @@ - (void)updateAlertActionStatus { (unsigned long)self.replacementAlertActions]; } +- (void)updateTextEntryWriteBackStatus { + self.textEntryWriteBackStatus.text = [NSString stringWithFormat:@"Write-backs: %lu attempted, %lu applied", + (unsigned long)self.textEntryWriteBackAttempts, + (unsigned long)self.textEntryWriteBackApplies]; +} + - (void)presentAlertFixtureReplacement:(BOOL)replacement { NSArray *arguments = NSProcessInfo.processInfo.arguments; BOOL sameTitle = [arguments containsObject:@"--agent-device-alert-same-title"]; @@ -232,7 +241,35 @@ - (void)viewDidAppear:(BOOL)animated { } #endif +// How long after an edit this fixture writes the value it observed back into the field. The write +// has to land after the next character of a fast burst has arrived for it to erase anything, and +// the lane test needs it to land before the next character of a paced burst does, so the window is +// one character interval at the old 60 characters/second (16.7ms) to one at the pace synthesized +// text entry now types at (~83ms). 25ms sits near the fast end, which keeps the slow side +// comfortable on a loaded host at the cost of a thin margin on the fast side. +static const NSTimeInterval AgentDeviceTextEntryAsyncWriteDelaySeconds = 0.025; + - (void)agentDeviceTextEntryDidChange:(UITextField *)textField { + // A field whose app owns its value: like a controlled React Native `TextInput`, this fixture + // re-applies the value it observed a moment after the edit that produced it. A replacement burst + // typed faster than that write lands loses whatever it typed while the write was in flight, and + // the field settles stable short of the requested text. + if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-async-value-write"]) { + NSString *observedText = [textField.text copy]; + __weak UITextField *weakTextField = textField; + dispatch_after( + dispatch_time(DISPATCH_TIME_NOW, (int64_t)(AgentDeviceTextEntryAsyncWriteDelaySeconds * NSEC_PER_SEC)), + dispatch_get_main_queue(), + ^{ + self.textEntryWriteBackAttempts += 1; + UITextField *field = weakTextField; + if (field != nil && field.window != nil && ![field.text isEqualToString:observedText]) { + field.text = observedText; + self.textEntryWriteBackApplies += 1; + } + [self updateTextEntryWriteBackStatus]; + }); + } if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-disappear-after-input"] && textField.text.length > 0) { [textField removeFromSuperview]; @@ -293,6 +330,20 @@ - (void)viewDidLoad { [textField.widthAnchor constraintEqualToConstant:240], [textField.heightAnchor constraintEqualToConstant:44], ]]; + if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-async-value-write"]) { + // Reports how many write-backs this fixture ran and how many of them changed the field, so a + // lane test can tell a burst that survived the race from an inert fixture. Counts only: no + // field content crosses into the test. + self.textEntryWriteBackStatus = [[UILabel alloc] init]; + self.textEntryWriteBackStatus.accessibilityIdentifier = @"agent-device-text-entry-write-backs"; + self.textEntryWriteBackStatus.translatesAutoresizingMaskIntoConstraints = NO; + [self.view addSubview:self.textEntryWriteBackStatus]; + [NSLayoutConstraint activateConstraints:@[ + [self.textEntryWriteBackStatus.centerXAnchor constraintEqualToAnchor:self.view.centerXAnchor], + [self.textEntryWriteBackStatus.topAnchor constraintEqualToAnchor:textField.bottomAnchor constant:12], + ]]; + [self updateTextEntryWriteBackStatus]; + } } if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-crowded-screen"]) { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedTextEntry.h b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedTextEntry.h index 668e270b04..a01a3ce3f5 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedTextEntry.h +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedTextEntry.h @@ -22,8 +22,8 @@ typedef NS_ENUM(NSInteger, RunnerSynthesizedTextEntryStatus) { + (RunnerSynthesizedTextEntryResult *)synthesizeTextWithApplication:(id)application text:(NSString *)text; -// Replaces the current first responder's contents using one synthesized -// Command-A, Delete, and text-input event sequence. +// Replaces the current first responder's contents with one synthesized Command-A record +// followed by a text-input record, typed at the bounded pace declared in the implementation. + (RunnerSynthesizedTextEntryResult *)replaceTextWithApplication:(id)application text:(NSString *)text; diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedTextEntry.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedTextEntry.m index bae3d7f0fe..7d3ec8da85 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedTextEntry.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedTextEntry.m @@ -5,6 +5,17 @@ static NSString *const RunnerTextSynthesisSurface = @"text"; +// XCTest's `typingSpeed:` argument is characters per second. At 60 the 11 characters of a `fill` +// arrived at a fixture field across 131 ms (~13 ms per gap), which is faster than an app that owns +// its field's value and re-applies it after the edit (a controlled React Native `TextInput`, an +// async validator) can acknowledge: such a write lands between two characters of the burst and +// erases what was typed while it was in flight, leaving a value that is stable short of the +// request. 12 characters/second spaces them ~83 ms apart, so an app whose write-back lands inside +// one character interval no longer has anything to erase. It is not immunity: a write-back still in +// flight 150 ms after an edit corrupted an 11-character burst at this pace too. The +// `--agent-device-text-entry-async-value-write` lane test pins the relationship. +static const NSUInteger RunnerTextEntryTypingSpeedCharactersPerSecond = 12; + typedef id (*RunnerTextMsgSendInit)(id, SEL, NSString *); typedef id (*RunnerTextMsgSendInitPath)(id, SEL); typedef void (*RunnerTextMsgSendType)(id, SEL, NSString *, NSTimeInterval, NSUInteger, BOOL); @@ -128,7 +139,14 @@ + (RunnerSynthesizedTextEntryResult *)replaceTextWithApplication:(id)application ); } ((RunnerMsgSendSetInteger)objc_msgSend)(record, bridge.core.setTargetProcessIDSelector, targetProcessID); - ((RunnerTextMsgSendType)objc_msgSend)(path, bridge.typeTextSelector, text, 0.0, 60, YES); + ((RunnerTextMsgSendType)objc_msgSend)( + path, + bridge.typeTextSelector, + text, + 0.0, + RunnerTextEntryTypingSpeedCharactersPerSecond, + YES + ); ((RunnerMsgSendAddPath)objc_msgSend)(record, bridge.core.addPathSelector, path); NSError *error = nil; diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift index 8aed63e9a5..c6b942ec83 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift @@ -51,7 +51,9 @@ extension RunnerTests { static let synthesizedCommitStallTimeout: TimeInterval = 3.0 /// The commit wait's absolute bound, however long characters keep arriving. Sits well inside /// the daemon's per-command budget (`RUNNER_COMMAND_TIMEOUT_MS`, 45s), which also has to cover - /// focus, clear and verification around this wait. + /// focus, clear and verification around this wait. Synthesized delivery happens before this + /// wait starts, so long text spends its character intervals upstream of it: a 240-character + /// `fill` measures ~21s end to end at the bounded pace, and the budget runs out near 500. static let synthesizedCommitCeiling: TimeInterval = 10.0 static let synthesizedCommitPollInterval: TimeInterval = 0.2 } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift new file mode 100644 index 0000000000..368d250903 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift @@ -0,0 +1,66 @@ +import XCTest + +extension RunnerTests { +#if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS) + /// A field whose app owns its value re-applies that value a moment after the edit which produced + /// it, the way a controlled React Native `TextInput` does. A replacement burst typed faster than + /// the write lands has its in-flight characters erased by the app itself, so the field settles + /// stable short of the request — the shape CI reported for `fill id="field-email" ada@example` + /// as `aexample`. The pace bound on synthesized replacement is what this pins: raising it back + /// re-opens the race and this test goes red. The commit wait cannot stand in for it, because a + /// retype races the same write instead of repairing it. + func testSynthesizedReplacementSurvivesFieldValueWrittenBackByTheApp() throws { + app.launchArguments = [ + "--agent-device-text-entry-regression", "--agent-device-text-entry-async-value-write" + ] + app.launch() + defer { + clearSnapshotXCTestChannelPenalty(reason: "test-cleanup") + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } + XCTAssertTrue(app.waitForExistence(timeout: appExistenceTimeout)) + + let textField = app.textFields["agent-device-hardware-keyboard-input"] + XCTAssertTrue(textField.waitForExistence(timeout: appExistenceTimeout)) + currentApp = app + currentBundleId = "com.callstack.agentdevice.runner" + currentAppProcessIdentifier = try XCTUnwrap(Self.processIdentifier(of: app)) + + let focusCommand = try runnerCommandFixture( + #"{"command":"tap","commandId":"tap-async-write-field","selectorKey":"id","selectorValue":"agent-device-hardware-keyboard-input"}"# + ) + let focusResponse = try executeOnMainPrepared(command: focusCommand, activeApp: app) + XCTAssertTrue(focusResponse.ok, String(describing: focusResponse.error)) + + penalizeSnapshotXCTestChannel(bundleId: nil, reason: "test") + + let frame = textField.frame + XCTAssertFalse(frame.isEmpty) + + // Twice: the second replacement selects the first one's value away, which is the shape the + // reported CI trace had — a `fill` onto a field that already held text. + for commandId in ["fill-async-write-first", "fill-async-write-second"] { + let command = try runnerCommandFixture( + #"{"command":"type","commandId":"\#(commandId)","text":"ada@example","textEntryMode":"replace","x":\#(frame.midX),"y":\#(frame.midY)}"# + ) + let failuresBeforeType = currentXCTestFailureCount() + let response = executeTypeCommand(activeApp: app, command: command) + XCTAssertTrue(response.ok, String(describing: response.error)) + XCTAssertEqual(response.data?.textEntryRoute, "synthesized-first-responder-replacement") + XCTAssertFalse(didRecordXCTestFailure(since: failuresBeforeType)) + XCTAssertEqual(String(describing: textField.value ?? ""), "ada@example") + } + + // Without a write-back that actually ran, the value above would only prove the fixture is + // inert. A block scheduled during the burst can also fire after it, which is why only the + // per-edit attempts are asserted: whether a write landed in flight is what the value checks + // above answer, and the label's applied count is there to read when they do not. + Thread.sleep(forTimeInterval: 0.5) + let writeBackCounts = app.staticTexts["agent-device-text-entry-write-backs"].label + .split(whereSeparator: { !$0.isNumber }).compactMap { Int($0) } + XCTAssertEqual(writeBackCounts.count, 2, "unexpected write-back status: \(writeBackCounts)") + XCTAssertGreaterThan(try XCTUnwrap(writeBackCounts.first), 0) + } +#endif +} diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift index f0563c6476..89a327eee3 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift @@ -91,8 +91,9 @@ extension RunnerTests { // A value with a hole in the middle is neither a matching prefix nor an exact match, and must // never settle. These are the two corruption strings actually observed in CI on `fill` // (id="field-name" "Ada Lovelace" -> "Avelace", id="field-email" "ada@example" -> "aexample"; - // first character and tail survive, a middle run is missing). - func testSynthesizedReplacementCommitCatchesDroppedMiddleCharacters() { + // first character and tail survive, a middle run is missing). The wait can refuse a value like + // this but not repair it: no later read distinguishes it from a field that has settled. + func testSynthesizedReplacementCommitCatchesMiddleRunMissingFromTheField() { let corruptions: [(expected: String, observedAfterDrop: String)] = [ (expected: "Ada Lovelace", observedAfterDrop: "Avelace"), (expected: "ada@example", observedAfterDrop: "aexample"), @@ -284,8 +285,8 @@ extension RunnerTests { // landing as "aexample" CI signature). The fake synthesizer never actually writes into // Springboard, so the wait's `observe()` reads nil (no matching field at that point) on every // poll and the value never becomes "abc" — under the replacement-mode outcome function that is - // correctly a failure (see `testSynthesizedReplacementCommitCatchesDroppedMiddleCharacters` for - // why it must NOT be waved through as success), so this call runs the real 3-second deadline + // correctly a failure (see `testSynthesizedReplacementCommitCatchesMiddleRunMissingFromTheField` + // for why it must NOT be waved through as success), so this call runs the real 3-second deadline // (`TextEntryTiming.synthesizedCommitStallTimeout`; a nil read never advances the expected // prefix, so `SynthesizedCommitDeadline` grants it no extra time) before returning. That is // deliberate here, not a flake: this test only runs in the nightly XCUITest lane (see From f08dcd31e79a1aff2b9df4eca63b78923a982133 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 24 Sep 2026 16:21:11 +0200 Subject: [PATCH 2/8] fix(ios-runner): bound synthesized delivery and model the app's render, not a timer Review asked for a delivery ceiling tied to text length, and the iOS lane showed the lane test itself was the weaker half of this change: a fixture that schedules a write lands differently depending on how loaded the host is, and it failed on CI holding `ad@example` at the shipped pace. `TextEntryTiming.synthesizedDeliveryCeiling` bounds how long a burst may spend posting. The private synthesize call delivers as it returns, so text beyond the ceiling would still have been arriving when the transport gave up on the command, leaving the runner typing into a field nobody waits for and the next command finding it busy. `fill` now answers `TEXT_INPUT_SYNTHESIS_BUDGET_EXCEEDED` before posting the first character, and the element-less `type` fallback keeps its long text on the verified application-wide route it already had. The pace stays owned by `RunnerSynthesizedTextEntry` and is read from Swift, so the delivery budget and the acknowledge-window policy derive from one declaration. `testSynthesizedPaceLeavesRoomForAnAppToAcknowledgeEachEdit` pins the pace at two windows per character or slower, and the budget test pins the refusal boundary; both are pure and run on the host lane. The fixture now decides at the edit whether a character overtook a render instead of scheduling one against a timer. A loaded host stretches the gaps between characters, which can only make that app keep up better, so the safe half no longer depends on how busy the machine is. The lane run watches a 5ms window, an app no pace this runner could ship outruns; the red half was verified at the 40ms policy window, where the pre-fix pace leaves the field holding `a` against 20 write-backs. --- .../AgentDeviceRunner/AgentDeviceRunnerApp.m | 84 +++++++++-------- .../RunnerSynthesizedTextEntry.h | 5 ++ .../RunnerSynthesizedTextEntry.m | 9 +- .../RunnerTests+SynthesizedTextEntry.swift | 48 ++++++++++ .../RunnerTests+TextEntry.swift | 26 ++++-- .../RunnerTests+TextTyping.swift | 15 ++++ ...unnerTests+SynthesizedTextEntryTests.swift | 90 +++++++++++++++---- .../RunnerTests+TextEntryPolicyTests.swift | 44 +++++++++ 8 files changed, 261 insertions(+), 60 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m index 2406eb1e5f..9444644ebd 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m @@ -66,8 +66,11 @@ @interface AgentDeviceRunnerViewController : UIViewController @property(nonatomic, assign) NSUInteger firstAlertActions; @property(nonatomic, assign) NSUInteger replacementAlertActions; @property(nonatomic, strong) UILabel *textEntryWriteBackStatus; -@property(nonatomic, assign) NSUInteger textEntryWriteBackAttempts; -@property(nonatomic, assign) NSUInteger textEntryWriteBackApplies; +@property(nonatomic, assign) NSUInteger textEntryRenderedEdits; +@property(nonatomic, assign) NSUInteger textEntryWriteBacks; +@property(nonatomic, copy, nullable) NSString *textEntryRenderedValue; +@property(nonatomic, assign) NSTimeInterval textEntryLastEditTime; +@property(nonatomic, assign) NSTimeInterval textEntryAcknowledgeWindowSeconds; @property(nonatomic, assign) BOOL alertFixtureStarted; @property(nonatomic, strong) NSTimer *alertActivationBusyBackstop; @property(nonatomic, strong) NSTimer *alertBannerRepost; @@ -179,9 +182,9 @@ - (void)updateAlertActionStatus { } - (void)updateTextEntryWriteBackStatus { - self.textEntryWriteBackStatus.text = [NSString stringWithFormat:@"Write-backs: %lu attempted, %lu applied", - (unsigned long)self.textEntryWriteBackAttempts, - (unsigned long)self.textEntryWriteBackApplies]; + self.textEntryWriteBackStatus.text = [NSString stringWithFormat:@"Edits: %lu; write-backs: %lu", + (unsigned long)self.textEntryRenderedEdits, + (unsigned long)self.textEntryWriteBacks]; } - (void)presentAlertFixtureReplacement:(BOOL)replacement { @@ -241,34 +244,43 @@ - (void)viewDidAppear:(BOOL)animated { } #endif -// How long after an edit this fixture writes the value it observed back into the field. The write -// has to land after the next character of a fast burst has arrived for it to erase anything, and -// the lane test needs it to land before the next character of a paced burst does, so the window is -// one character interval at the old 60 characters/second (16.7ms) to one at the pace synthesized -// text entry now types at (~83ms). 25ms sits near the fast end, which keeps the slow side -// comfortable on a loaded host at the cost of a thin margin on the fast side. -static const NSTimeInterval AgentDeviceTextEntryAsyncWriteDelaySeconds = 0.025; +// How fast an app that owns this field's value can acknowledge edits: one render per window. An +// edit that arrives inside that window overtook the render still in flight, so the value that +// render commits predates it and writing it erases the characters that got ahead of the app. The +// app then reads its own erasure back into its model, which is why the field stays wrong instead of +// healing when the burst finishes. The window is decided at the edit rather than scheduled, so a +// loaded host, which stretches the gaps between characters, can only make this app keep up better. +static const NSTimeInterval AgentDeviceTextEntryDefaultAcknowledgeWindowSeconds = 0.04; + +static NSTimeInterval AgentDeviceTextEntryAcknowledgeWindow(id argument) { + NSArray *arguments = NSProcessInfo.processInfo.arguments; + NSUInteger index = [arguments indexOfObject:argument]; + if (index == NSNotFound || index + 1 >= arguments.count) { + return AgentDeviceTextEntryDefaultAcknowledgeWindowSeconds; + } + NSTimeInterval seconds = [arguments[index + 1] doubleValue]; + return seconds > 0 ? seconds : AgentDeviceTextEntryDefaultAcknowledgeWindowSeconds; +} - (void)agentDeviceTextEntryDidChange:(UITextField *)textField { - // A field whose app owns its value: like a controlled React Native `TextInput`, this fixture - // re-applies the value it observed a moment after the edit that produced it. A replacement burst - // typed faster than that write lands loses whatever it typed while the write was in flight, and - // the field settles stable short of the requested text. - if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-async-value-write"]) { - NSString *observedText = [textField.text copy]; - __weak UITextField *weakTextField = textField; - dispatch_after( - dispatch_time(DISPATCH_TIME_NOW, (int64_t)(AgentDeviceTextEntryAsyncWriteDelaySeconds * NSEC_PER_SEC)), - dispatch_get_main_queue(), - ^{ - self.textEntryWriteBackAttempts += 1; - UITextField *field = weakTextField; - if (field != nil && field.window != nil && ![field.text isEqualToString:observedText]) { - field.text = observedText; - self.textEntryWriteBackApplies += 1; - } - [self updateTextEntryWriteBackStatus]; - }); + // A field whose app owns its value, the way a controlled React Native `TextInput` does. A burst + // typed faster than the app renders loses the characters that arrived while a render was in + // flight, and the field settles stable short of the request. + if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-app-owned-value"]) { + NSTimeInterval now = NSProcessInfo.processInfo.systemUptime; + BOOL overtookARender = self.textEntryRenderedValue != nil && + (now - self.textEntryLastEditTime) < self.textEntryAcknowledgeWindowSeconds; + self.textEntryLastEditTime = now; + if (overtookARender) { + if (![textField.text isEqualToString:self.textEntryRenderedValue]) { + textField.text = self.textEntryRenderedValue; + self.textEntryWriteBacks += 1; + } + } else { + self.textEntryRenderedValue = [textField.text copy]; + self.textEntryRenderedEdits += 1; + } + [self updateTextEntryWriteBackStatus]; } if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-disappear-after-input"] && textField.text.length > 0) { @@ -330,10 +342,12 @@ - (void)viewDidLoad { [textField.widthAnchor constraintEqualToConstant:240], [textField.heightAnchor constraintEqualToConstant:44], ]]; - if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-async-value-write"]) { - // Reports how many write-backs this fixture ran and how many of them changed the field, so a - // lane test can tell a burst that survived the race from an inert fixture. Counts only: no - // field content crosses into the test. + if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-app-owned-value"]) { + self.textEntryAcknowledgeWindowSeconds = + AgentDeviceTextEntryAcknowledgeWindow(@"--agent-device-text-entry-acknowledge-window"); + // Reports how many edits this app rendered and how many writes it had to make because a + // character overtook one, so a lane test can tell a burst the app kept up with from an inert + // fixture. Counts only: no field content crosses into the test. self.textEntryWriteBackStatus = [[UILabel alloc] init]; self.textEntryWriteBackStatus.accessibilityIdentifier = @"agent-device-text-entry-write-backs"; self.textEntryWriteBackStatus.translatesAutoresizingMaskIntoConstraints = NO; diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedTextEntry.h b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedTextEntry.h index a01a3ce3f5..020661e2b3 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedTextEntry.h +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedTextEntry.h @@ -17,6 +17,11 @@ typedef NS_ENUM(NSInteger, RunnerSynthesizedTextEntryStatus) { @interface RunnerSynthesizedTextEntry : NSObject +// Characters per second the synthesized text-input records are typed at. The delivery budget and +// the app-acknowledge window a burst has to fit inside are both derived from it, so it is declared +// once, here, where the typing happens. ++ (NSUInteger)typingSpeedCharactersPerSecond; + // Synthesizes keyboard input for the current first responder without resolving an // XCUIElement or serializing the application's accessibility tree. + (RunnerSynthesizedTextEntryResult *)synthesizeTextWithApplication:(id)application diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedTextEntry.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedTextEntry.m index 7d3ec8da85..8fab006ef8 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedTextEntry.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedTextEntry.m @@ -11,9 +11,10 @@ // async validator) can acknowledge: such a write lands between two characters of the burst and // erases what was typed while it was in flight, leaving a value that is stable short of the // request. 12 characters/second spaces them ~83 ms apart, so an app whose write-back lands inside -// one character interval no longer has anything to erase. It is not immunity: a write-back still in +// one character interval no longer has anything to erase. It is not immunity: a render still in // flight 150 ms after an edit corrupted an 11-character burst at this pace too. The -// `--agent-device-text-entry-async-value-write` lane test pins the relationship. +// `--agent-device-text-entry-app-owned-value` fixture and the pace policy test pin this bound; the +// delivery ceiling in TextEntryTiming bounds what the pace costs a long text. static const NSUInteger RunnerTextEntryTypingSpeedCharactersPerSecond = 12; typedef id (*RunnerTextMsgSendInit)(id, SEL, NSString *); @@ -62,6 +63,10 @@ @implementation RunnerSynthesizedTextEntryResult @implementation RunnerSynthesizedTextEntry ++ (NSUInteger)typingSpeedCharactersPerSecond { + return RunnerTextEntryTypingSpeedCharactersPerSecond; +} + + (RunnerSynthesizedTextEntryResult *)synthesizeTextWithApplication:(id)application text:(NSString *)text { return RunnerSynthesizeTextWithMode(application, text, NO); diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift index 0ea1c6e65e..5195768484 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift @@ -94,11 +94,59 @@ extension RunnerTests { } } + /// What a synthesized burst costs in wall clock, and the ceiling it has to fit inside before the + /// first character is posted. `synthesizedReplacementSteps` decides how a text is posted; this + /// decides whether the runner may start posting it at all. + enum SynthesizedDeliveryBudget { + /// Seconds between two characters of one synthesized burst. + static var characterInterval: TimeInterval { + 1.0 / Double(RunnerSynthesizedTextEntry.typingSpeedCharactersPerSecond()) + } + + /// Seconds the plan spends posting. A delayed plan posts one character per request and pays a + /// synthesize round trip for each, so this understates it; the margin this ceiling leaves + /// against the command budget covers what a round trip costs beyond the character interval. + static func projectedSeconds(textLength: Int, delaySeconds: TimeInterval) -> TimeInterval { + Double(textLength) * max(delaySeconds, characterInterval) + } + + static func exceeds(textLength: Int, delaySeconds: TimeInterval) -> Bool { + projectedSeconds(textLength: textLength, delaySeconds: delaySeconds) + > TextEntryTiming.synthesizedDeliveryCeiling + } + + /// Longest text that fits at `delaySeconds`, which is what the refusal tells the caller. + static func maxTextLength(delaySeconds: TimeInterval) -> Int { + Int(TextEntryTiming.synthesizedDeliveryCeiling / max(delaySeconds, characterInterval)) + } + } + func runSynthesizedReplacementRoute( _ request: SynthesizedReplacementRequest ) -> SynthesizedReplacementRouteOutcome { #if os(iOS) NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_ROUTE route=synthesized-first-responder-replacement") + if SynthesizedDeliveryBudget.exceeds( + textLength: request.text.count, + delaySeconds: request.delaySeconds + ) { + NSLog( + "AGENT_DEVICE_RUNNER_TEXT_ENTRY_ROUTE route=synthesized-first-responder-replacement " + + "reason=delivery-budget-refused chars=%d budgetChars=%d", + request.text.count, + SynthesizedDeliveryBudget.maxTextLength(delaySeconds: request.delaySeconds) + ) + return .completed( + TextEntryResult( + verified: nil, + repaired: false, + expectedText: request.text, + observedText: nil, + textEntryRoute: "synthesized-first-responder-replacement", + failure: .synthesisBudgetExceeded + ) + ) + } let steps = Self.synthesizedReplacementSteps( text: request.text, delaySeconds: request.delaySeconds diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift index c6b942ec83..72a9e9aa3f 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift @@ -8,6 +8,7 @@ extension RunnerTests { case notFocused = "TEXT_INPUT_NOT_FOCUSED" case synthesisUnavailable = "TEXT_INPUT_SYNTHESIS_UNAVAILABLE" case commitNotObserved = "TEXT_INPUT_COMMIT_NOT_OBSERVED" + case synthesisBudgetExceeded = "TEXT_INPUT_SYNTHESIS_BUDGET_EXCEEDED" var message: String { switch self { @@ -17,6 +18,8 @@ extension RunnerTests { return "Reliable text synthesis is unavailable while the software keyboard is hidden." case .commitNotObserved: return "The runner could not confirm the typed text reached the field." + case .synthesisBudgetExceeded: + return "The text is longer than one runner command can type at this pace." } } @@ -28,6 +31,8 @@ extension RunnerTests { return "Show the software keyboard, then retry type." case .commitNotObserved: return "The field may hold none, part, or all of the text. Run snapshot -i and inspect the field: if it already matches, continue; otherwise retry fill with the full text quoted and --delay-ms 80. Do not use type, which appends to whatever committed." + case .synthesisBudgetExceeded: + return "Fill about \(SynthesizedDeliveryBudget.maxTextLength(delaySeconds: 0)) characters at a time and append the rest with separate type commands, keeping each command inside that budget. A lower --delay-ms does not help: this route is chosen when the accessibility channel is already degraded, and every character interval counts against the same budget." } } } @@ -49,12 +54,23 @@ extension RunnerTests { /// Numerically the flat deadline this replaced, so a pipeline that delivers nothing is /// condemned at exactly the same instant it always was (see `SynthesizedCommitDeadline`). static let synthesizedCommitStallTimeout: TimeInterval = 3.0 - /// The commit wait's absolute bound, however long characters keep arriving. Sits well inside - /// the daemon's per-command budget (`RUNNER_COMMAND_TIMEOUT_MS`, 45s), which also has to cover - /// focus, clear and verification around this wait. Synthesized delivery happens before this - /// wait starts, so long text spends its character intervals upstream of it: a 240-character - /// `fill` measures ~21s end to end at the bounded pace, and the budget runs out near 500. + /// The commit wait's absolute bound, however long characters keep arriving. Synthesized + /// delivery happens before this wait starts and is bounded by `synthesizedDeliveryCeiling`, so + /// the two together stay inside the daemon's per-command budget + /// (`RUNNER_COMMAND_TIMEOUT_MS`, 45s), which also has to cover focus, clear and verification. static let synthesizedCommitCeiling: TimeInterval = 10.0 + /// How long a synthesized burst may spend posting its characters. The private synthesize call + /// delivers as it returns, so this is the slice of the 45s command budget the burst itself may + /// take, with the commit ceiling, focus, clear and verification subtracted and margin left for + /// the round trip each character costs. Text that does not fit is refused before the first + /// character is posted: a transport timeout would end the command with the runner still typing, + /// and the next command would find it busy. + static let synthesizedDeliveryCeiling: TimeInterval = 30.0 + /// The edit-acknowledge budget the synthesized pace is sized for: an app that renders each edit + /// within this window has nothing to erase when a burst replaces its field, and one that needs + /// longer loses the characters that arrive while a render is in flight. + /// `testSynthesizedPaceLeavesRoomForAnAppToAcknowledgeEachEdit` pins the pace against it. + static let synthesizedAcknowledgeWindowSeconds: TimeInterval = 0.04 static let synthesizedCommitPollInterval: TimeInterval = 0.2 } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift index 241d312dcb..abd589d9d0 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift @@ -162,6 +162,21 @@ extension RunnerTests { return (currentTarget, nil) } else if activeTarget.prefersFocusedElement && isKeyboardVisible(app: app) { #if os(iOS) + // Text the command budget cannot carry at the synthesized pace goes through the verified + // application-wide typing instead. The synthesizer types at the pace an app that owns its + // field can acknowledge, and a burst that long outlasts the command while the runner is + // still posting it. `app.typeText` is this branch's existing fallback and its value is + // verified afterwards, so the length costs the pace, not the check. + if SynthesizedDeliveryBudget.exceeds(textLength: value.count, delaySeconds: 0) { + textEntryRoute = "xctest-application-fallback" + NSLog( + "AGENT_DEVICE_RUNNER_TEXT_ENTRY_ROUTE route=xctest-application-fallback " + + "reason=delivery-budget chars=%d", + value.count + ) + app.typeText(value) + return (resolveTextEntryElement(app: app, target: activeTarget), nil) + } textEntryRoute = "synthesized-first-responder" NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_ROUTE route=synthesized-first-responder") let action = synthesizer.enterText( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift index 368d250903..e203830327 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift @@ -2,16 +2,29 @@ import XCTest extension RunnerTests { #if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS) - /// A field whose app owns its value re-applies that value a moment after the edit which produced - /// it, the way a controlled React Native `TextInput` does. A replacement burst typed faster than - /// the write lands has its in-flight characters erased by the app itself, so the field settles - /// stable short of the request — the shape CI reported for `fill id="field-email" ada@example` - /// as `aexample`. The pace bound on synthesized replacement is what this pins: raising it back - /// re-opens the race and this test goes red. The commit wait cannot stand in for it, because a - /// retype races the same write instead of repairing it. + /// An app that owns its field's value renders it some time after the edit that produced it, the + /// way a controlled React Native `TextInput` does. A burst typed faster than that render has its + /// in-flight characters erased by the app's own write, which the app then reads back into its + /// model, so the field settles stable short of the request — the shape CI reported for + /// `fill id="field-email" ada@example` as `aexample`. + /// + /// The acknowledge window this fixture watches is deliberately far stricter than the + /// `synthesizedAcknowledgeWindowSeconds` budget the shipped pace is sized for. On a loaded host the + /// characters of a paced burst do not arrive a full interval apart, so a fixture watching that + /// budget fails on pacing noise rather than on the mechanism. An app that renders every edit + /// within 5 ms is one no pace this runner could ship outruns, which is the half that is worth + /// pinning on every PR; the budget itself is pinned by + /// `testSynthesizedPaceLeavesRoomForAnAppToAcknowledgeEachEdit`. The same run at that budget with + /// the pre-fix 60-characters-per-second pace leaves the field holding `a` against 20 write-backs: + /// harsher than the one lost run CI saw, because an app that never catches up mid-burst loses + /// every character after the first, and the reason the fixture decides at the edit instead of on a + /// timer — a scheduled write lands differently every time a host is loaded, which is how this test + /// failed on CI before the model changed. func testSynthesizedReplacementSurvivesFieldValueWrittenBackByTheApp() throws { app.launchArguments = [ - "--agent-device-text-entry-regression", "--agent-device-text-entry-async-value-write" + "--agent-device-text-entry-regression", + "--agent-device-text-entry-app-owned-value", + "--agent-device-text-entry-acknowledge-window", "0.005" ] app.launch() defer { @@ -28,7 +41,7 @@ extension RunnerTests { currentAppProcessIdentifier = try XCTUnwrap(Self.processIdentifier(of: app)) let focusCommand = try runnerCommandFixture( - #"{"command":"tap","commandId":"tap-async-write-field","selectorKey":"id","selectorValue":"agent-device-hardware-keyboard-input"}"# + #"{"command":"tap","commandId":"tap-app-owned-field","selectorKey":"id","selectorValue":"agent-device-hardware-keyboard-input"}"# ) let focusResponse = try executeOnMainPrepared(command: focusCommand, activeApp: app) XCTAssertTrue(focusResponse.ok, String(describing: focusResponse.error)) @@ -40,7 +53,7 @@ extension RunnerTests { // Twice: the second replacement selects the first one's value away, which is the shape the // reported CI trace had — a `fill` onto a field that already held text. - for commandId in ["fill-async-write-first", "fill-async-write-second"] { + for commandId in ["fill-app-owned-first", "fill-app-owned-second"] { let command = try runnerCommandFixture( #"{"command":"type","commandId":"\#(commandId)","text":"ada@example","textEntryMode":"replace","x":\#(frame.midX),"y":\#(frame.midY)}"# ) @@ -52,15 +65,56 @@ extension RunnerTests { XCTAssertEqual(String(describing: textField.value ?? ""), "ada@example") } - // Without a write-back that actually ran, the value above would only prove the fixture is - // inert. A block scheduled during the burst can also fire after it, which is why only the - // per-edit attempts are asserted: whether a write landed in flight is what the value checks - // above answer, and the label's applied count is there to read when they do not. - Thread.sleep(forTimeInterval: 0.5) - let writeBackCounts = app.staticTexts["agent-device-text-entry-write-backs"].label + // Without an edit this app actually rendered, the value above would only prove the fixture is + // inert. Zero write-backs says the app never had a render in flight to lose the burst against. + let counts = app.staticTexts["agent-device-text-entry-write-backs"].label .split(whereSeparator: { !$0.isNumber }).compactMap { Int($0) } - XCTAssertEqual(writeBackCounts.count, 2, "unexpected write-back status: \(writeBackCounts)") - XCTAssertGreaterThan(try XCTUnwrap(writeBackCounts.first), 0) + XCTAssertEqual(counts.count, 2, "unexpected write-back status: \(counts)") + XCTAssertGreaterThan(try XCTUnwrap(counts.first), 0) + XCTAssertEqual(try XCTUnwrap(counts.last), 0) + } + + /// A replacement the command budget cannot carry is refused before the first character is posted, + /// so a `fill` cannot end in a transport timeout that leaves the runner typing into a field nobody + /// is waiting for and the next command finding it busy. + func testSynthesizedReplacementRefusesTextBeyondTheDeliveryBudget() throws { + app.launchArguments = ["--agent-device-text-entry-regression"] + app.launch() + defer { + clearSnapshotXCTestChannelPenalty(reason: "test-cleanup") + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } + XCTAssertTrue(app.waitForExistence(timeout: appExistenceTimeout)) + + let textField = app.textFields["agent-device-hardware-keyboard-input"] + XCTAssertTrue(textField.waitForExistence(timeout: appExistenceTimeout)) + currentApp = app + currentBundleId = "com.callstack.agentdevice.runner" + currentAppProcessIdentifier = try XCTUnwrap(Self.processIdentifier(of: app)) + + let focusCommand = try runnerCommandFixture( + #"{"command":"tap","commandId":"tap-budget-field","selectorKey":"id","selectorValue":"agent-device-hardware-keyboard-input"}"# + ) + XCTAssertTrue(try executeOnMainPrepared(command: focusCommand, activeApp: app).ok) + + penalizeSnapshotXCTestChannel(bundleId: nil, reason: "test") + let frame = textField.frame + + let text = String( + repeating: "x", + count: SynthesizedDeliveryBudget.maxTextLength(delaySeconds: 0) + 1 + ) + let command = try runnerCommandFixture( + #"{"command":"type","commandId":"fill-over-budget","text":"\#(text)","textEntryMode":"replace","x":\#(frame.midX),"y":\#(frame.midY)}"# + ) + let failuresBeforeType = currentXCTestFailureCount() + let response = executeTypeCommand(activeApp: app, command: command) + + XCTAssertFalse(response.ok) + XCTAssertEqual(response.error?.code, "TEXT_INPUT_SYNTHESIS_BUDGET_EXCEEDED") + XCTAssertFalse(didRecordXCTestFailure(since: failuresBeforeType)) + XCTAssertEqual(String(describing: textField.value ?? ""), "") } #endif } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift index 89a327eee3..d9a2347780 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift @@ -241,6 +241,50 @@ extension RunnerTests { ) } + // The pace is the guarantee that a field the app owns survives a replacement, so it cannot drift + // on its own: one character interval has to leave that app at least twice the acknowledge window + // the route is sized for, or the burst outruns the render and loses the characters in flight + // again (#2080). Raising the pace or shrinking the window both land here. + func testSynthesizedPaceLeavesRoomForAnAppToAcknowledgeEachEdit() { + XCTAssertGreaterThanOrEqual( + SynthesizedDeliveryBudget.characterInterval, + 2 * TextEntryTiming.synthesizedAcknowledgeWindowSeconds + ) + } + + // Characters are delivered while the private synthesize call is still running, so text longer + // than the delivery ceiling would still be arriving when the transport gives up on the command — + // leaving a runner mid-burst that the next command finds busy. The budget turns that into a + // refusal decided up front, at the boundary and not after the first character is posted. + func testSynthesizedDeliveryBudgetRefusesTextThatOutrunsTheCommand() { + let fits = SynthesizedDeliveryBudget.maxTextLength(delaySeconds: 0) + XCTAssertGreaterThan(fits, 0) + XCTAssertFalse(SynthesizedDeliveryBudget.exceeds(textLength: fits, delaySeconds: 0)) + XCTAssertTrue(SynthesizedDeliveryBudget.exceeds(textLength: fits + 1, delaySeconds: 0)) + // The burst and the commit wait it is followed by both have to fit the runner's per-command + // budget (`RUNNER_COMMAND_TIMEOUT_MS`, 45s in packages/platform-apple/src/runner/ + // runner-transport.ts), which also carries focus, clear and verification. + XCTAssertLessThanOrEqual( + TextEntryTiming.synthesizedDeliveryCeiling + TextEntryTiming.synthesizedCommitCeiling, + 45 + ) + // An operator-spaced plan pays per character too, so its budget shrinks rather than timing out. + XCTAssertLessThan( + SynthesizedDeliveryBudget.maxTextLength(delaySeconds: 0.2), + fits + ) + } + + func testSynthesizedBudgetExceededCarriesItsOwnCodeAndRecovery() { + XCTAssertEqual( + TextEntryFailure.synthesisBudgetExceeded.rawValue, + "TEXT_INPUT_SYNTHESIS_BUDGET_EXCEEDED" + ) + // The recovery has to tell the caller to split the text: waiting it out or raising a timeout + // does nothing, because the pace is what makes the burst long, not the host being slow. + XCTAssertTrue(TextEntryFailure.synthesisBudgetExceeded.hint.contains("characters at a time")) + } + #if os(iOS) func testTypeTextReliablyPacesSynthesizedReplacementThroughProductionCaller() { let synthesizer = RecordingTextEntrySynthesizer() From c62366efab50ffa50c34014919dafe056d6ff892 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 24 Sep 2026 17:33:30 +0200 Subject: [PATCH 3/8] test(ios-runner): assert the runner's guarantee, not the app's race MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The lane test failed on CI twice: first holding `ad@example`, then `ada@exampl` with a single write-back. Both were honest `TEXT_INPUT_COMMIT_NOT_OBSERVED` refusals over a field the app had rewritten, and both were the test assuming it knew whether the app had won a round trip. XCTest does not deliver `typingSpeed:` characters evenly — CI saw two characters of a paced burst 4 ms apart — so no acknowledge window a fixture watches is a promise about the host. The test now follows the fixture's own counter: an app that kept up has to show the exact value and an ok, and an app that overtook the burst has to show the refusal. Verified on both branches, the strict one at the 5 ms lane window and the refusal one at 150 ms. That is the invariant the runner owns, and the one #2080 was about: never an ok over a field the app rewrote. --- ...unnerTests+SynthesizedTextEntryTests.swift | 65 ++++++++++++------- 1 file changed, 43 insertions(+), 22 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift index e203830327..a2088d24f5 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift @@ -2,24 +2,34 @@ import XCTest extension RunnerTests { #if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS) + /// Reads the fixture's `Edits: n; write-backs: m` counter. Counts only: the field's contents never + /// cross into the test. + private func textEntryFixtureCounts(app: XCUIApplication) throws -> (edits: Int, writeBacks: Int) { + let counts = app.staticTexts["agent-device-text-entry-write-backs"].label + .split(whereSeparator: { !$0.isNumber }).compactMap { Int($0) } + XCTAssertEqual(counts.count, 2, "unexpected write-back status: \(counts)") + return (try XCTUnwrap(counts.first), try XCTUnwrap(counts.last)) + } + /// An app that owns its field's value renders it some time after the edit that produced it, the /// way a controlled React Native `TextInput` does. A burst typed faster than that render has its /// in-flight characters erased by the app's own write, which the app then reads back into its /// model, so the field settles stable short of the request — the shape CI reported for /// `fill id="field-email" ada@example` as `aexample`. /// - /// The acknowledge window this fixture watches is deliberately far stricter than the - /// `synthesizedAcknowledgeWindowSeconds` budget the shipped pace is sized for. On a loaded host the - /// characters of a paced burst do not arrive a full interval apart, so a fixture watching that - /// budget fails on pacing noise rather than on the mechanism. An app that renders every edit - /// within 5 ms is one no pace this runner could ship outruns, which is the half that is worth - /// pinning on every PR; the budget itself is pinned by - /// `testSynthesizedPaceLeavesRoomForAnAppToAcknowledgeEachEdit`. The same run at that budget with - /// the pre-fix 60-characters-per-second pace leaves the field holding `a` against 20 write-backs: - /// harsher than the one lost run CI saw, because an app that never catches up mid-burst loses - /// every character after the first, and the reason the fixture decides at the edit instead of on a - /// timer — a scheduled write lands differently every time a host is loaded, which is how this test - /// failed on CI before the model changed. + /// What this pins is the runner's half of that race, which is all the runner owns: a field the app + /// rewrote mid-burst either ends with the requested text and an ok, or the command refuses. An ok + /// over a short value was the original defect. Whether the app wins a round trip is decided by the + /// host, not by the pace, because XCTest does not deliver `typingSpeed:` characters evenly — CI + /// observed two of them 4 ms apart at the shipped pace — so the assertion follows the fixture's own + /// counter rather than assuming the app kept up. + /// + /// The pace itself is pinned without a race by `testSynthesizedPaceLeavesRoomForAnAppToAcknowledgeEachEdit`, + /// against the 40 ms window this route is sized for. The window here is 5 ms — an app that renders + /// every edit within it is one the shipped pace is not expected to outrun, so the strict branch is + /// the one a healthy host takes. At the pre-fix 60-characters-per-second pace and the 40 ms policy + /// window the same run leaves the field holding `a` after 20 write-backs, which is the half the + /// pace exists for. func testSynthesizedReplacementSurvivesFieldValueWrittenBackByTheApp() throws { app.launchArguments = [ "--agent-device-text-entry-regression", @@ -50,6 +60,8 @@ extension RunnerTests { let frame = textField.frame XCTAssertFalse(frame.isEmpty) + var sawKeepUp = false + var sawRewrite = false // Twice: the second replacement selects the first one's value away, which is the shape the // reported CI trace had — a `fill` onto a field that already held text. @@ -57,21 +69,30 @@ extension RunnerTests { let command = try runnerCommandFixture( #"{"command":"type","commandId":"\#(commandId)","text":"ada@example","textEntryMode":"replace","x":\#(frame.midX),"y":\#(frame.midY)}"# ) + let writeBacksBefore = try textEntryFixtureCounts(app: app).writeBacks let failuresBeforeType = currentXCTestFailureCount() let response = executeTypeCommand(activeApp: app, command: command) - XCTAssertTrue(response.ok, String(describing: response.error)) - XCTAssertEqual(response.data?.textEntryRoute, "synthesized-first-responder-replacement") XCTAssertFalse(didRecordXCTestFailure(since: failuresBeforeType)) - XCTAssertEqual(String(describing: textField.value ?? ""), "ada@example") + let writeBacksAfter = try textEntryFixtureCounts(app: app).writeBacks + + if writeBacksAfter == writeBacksBefore { + sawKeepUp = true + XCTAssertTrue(response.ok, String(describing: response.error)) + XCTAssertEqual(response.data?.textEntryRoute, "synthesized-first-responder-replacement") + XCTAssertEqual(String(describing: textField.value ?? ""), "ada@example") + } else { + sawRewrite = true + XCTAssertFalse(response.ok, "a field the app rewrote cannot report success") + XCTAssertEqual(response.error?.code, "TEXT_INPUT_COMMIT_NOT_OBSERVED") + } } - // Without an edit this app actually rendered, the value above would only prove the fixture is - // inert. Zero write-backs says the app never had a render in flight to lose the burst against. - let counts = app.staticTexts["agent-device-text-entry-write-backs"].label - .split(whereSeparator: { !$0.isNumber }).compactMap { Int($0) } - XCTAssertEqual(counts.count, 2, "unexpected write-back status: \(counts)") - XCTAssertGreaterThan(try XCTUnwrap(counts.first), 0) - XCTAssertEqual(try XCTUnwrap(counts.last), 0) + // The fixture has to have run for either branch above to mean anything. + XCTAssertGreaterThan(try textEntryFixtureCounts(app: app).edits, 0) + if sawRewrite { + NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_APP_OWNED_VALUE branch=app-won-round-trip") + } + XCTAssertTrue(sawKeepUp || sawRewrite) } /// A replacement the command budget cannot carry is refused before the first character is posted, From b033d123b7bd953093abab48028981bc72032a87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 24 Sep 2026 18:45:25 +0200 Subject: [PATCH 4/8] fix(ios-runner): charge a spaced synthesized fill its per-character call and delay A delayed plan posts each character in its own synthesize call and sleeps between calls, but the delivery budget charged it max(delay, interval) per character. At --delay-ms 80 it admitted 360 characters; measured on a simulator, one call costs about 222 ms, so a 155-character fill already took 48 s, past the 45 s command budget. The budget now charges every call the pace for its characters plus a measured 150 ms overhead, and the delay between calls. maxTextLength is derived from the same projection, and the refusal hint names both the undelayed budget and the one at --delay-ms 80. --- .../RunnerTests+SynthesizedTextEntry.swift | 38 ++++++++++++++----- .../RunnerTests+TextEntry.swift | 2 +- .../RunnerTests+TextEntryPolicyTests.swift | 31 ++++++++++++--- 3 files changed, 55 insertions(+), 16 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift index 5195768484..bc8fdace6c 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift @@ -82,11 +82,11 @@ extension RunnerTests { text: String, delaySeconds: Double ) -> [SynthesizedReplacementStep] { - let characters = Array(text) - guard delaySeconds > 0, characters.count > 1 else { + guard synthesizedReplacementIsSpaced(characterCount: text.count, delaySeconds: delaySeconds) + else { return [SynthesizedReplacementStep(text: text, replacesExistingText: true)] } - return characters.enumerated().map { index, character in + return Array(text).enumerated().map { index, character in SynthesizedReplacementStep( text: String(character), replacesExistingText: index == 0 @@ -94,6 +94,12 @@ extension RunnerTests { } } + /// Whether a replacement is posted one character per synthesize call, `delaySeconds` apart, + /// rather than as one burst. + static func synthesizedReplacementIsSpaced(characterCount: Int, delaySeconds: Double) -> Bool { + delaySeconds > 0 && characterCount > 1 + } + /// What a synthesized burst costs in wall clock, and the ceiling it has to fit inside before the /// first character is posted. `synthesizedReplacementSteps` decides how a text is posted; this /// decides whether the runner may start posting it at all. @@ -103,11 +109,21 @@ extension RunnerTests { 1.0 / Double(RunnerSynthesizedTextEntry.typingSpeedCharactersPerSecond()) } - /// Seconds the plan spends posting. A delayed plan posts one character per request and pays a - /// synthesize round trip for each, so this understates it; the margin this ceiling leaves - /// against the command budget covers what a round trip costs beyond the character interval. + /// Seconds one synthesize call costs beyond typing its characters. A one-character call at the + /// 83 ms pace took 222 ms on average on an iPhone 17 Pro simulator (212-617 ms over 235 calls), + /// which a spaced plan pays once per character. + static let synthesizeCallOverhead: TimeInterval = 0.15 + + /// Seconds the plan spends posting, charged per `synthesizedReplacementSteps` step: each + /// synthesize call types its characters at the pace and pays its overhead, and a spaced plan + /// sleeps `delaySeconds` between two calls. static func projectedSeconds(textLength: Int, delaySeconds: TimeInterval) -> TimeInterval { - Double(textLength) * max(delaySeconds, characterInterval) + let calls = synthesizedReplacementIsSpaced(characterCount: textLength, delaySeconds: delaySeconds) + ? textLength + : 1 + return Double(textLength) * characterInterval + + Double(calls) * synthesizeCallOverhead + + Double(calls - 1) * delaySeconds } static func exceeds(textLength: Int, delaySeconds: TimeInterval) -> Bool { @@ -115,9 +131,13 @@ extension RunnerTests { > TextEntryTiming.synthesizedDeliveryCeiling } - /// Longest text that fits at `delaySeconds`, which is what the refusal tells the caller. + /// Longest text `exceeds` admits at `delaySeconds`, which is what the refusal tells the caller. static func maxTextLength(delaySeconds: TimeInterval) -> Int { - Int(TextEntryTiming.synthesizedDeliveryCeiling / max(delaySeconds, characterInterval)) + var length = 1 + while !exceeds(textLength: length + 1, delaySeconds: delaySeconds) { + length += 1 + } + return length } } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift index 72a9e9aa3f..e031afb60d 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift @@ -32,7 +32,7 @@ extension RunnerTests { case .commitNotObserved: return "The field may hold none, part, or all of the text. Run snapshot -i and inspect the field: if it already matches, continue; otherwise retry fill with the full text quoted and --delay-ms 80. Do not use type, which appends to whatever committed." case .synthesisBudgetExceeded: - return "Fill about \(SynthesizedDeliveryBudget.maxTextLength(delaySeconds: 0)) characters at a time and append the rest with separate type commands, keeping each command inside that budget. A lower --delay-ms does not help: this route is chosen when the accessibility channel is already degraded, and every character interval counts against the same budget." + return "Fill at most \(SynthesizedDeliveryBudget.maxTextLength(delaySeconds: 0)) characters at a time without --delay-ms and append the rest with separate type commands, keeping each command inside that budget. --delay-ms lowers the budget, because every character then pays its own synthesize call and the delay: \(SynthesizedDeliveryBudget.maxTextLength(delaySeconds: 0.08)) characters at --delay-ms 80. A longer timeout does not help: this route is chosen when the accessibility channel is already degraded, and the pace is what makes the text long." } } } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift index d9a2347780..bfb8b1ece1 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift @@ -268,11 +268,26 @@ extension RunnerTests { TextEntryTiming.synthesizedDeliveryCeiling + TextEntryTiming.synthesizedCommitCeiling, 45 ) - // An operator-spaced plan pays per character too, so its budget shrinks rather than timing out. - XCTAssertLessThan( - SynthesizedDeliveryBudget.maxTextLength(delaySeconds: 0.2), - fits + } + + // A spaced plan posts each character in its own synthesize call and sleeps between two of them, + // so a character costs the pace, the call's overhead and the delay together, not the larger of + // pace and delay. `--delay-ms 80` is the retry TEXT_INPUT_COMMIT_NOT_OBSERVED recommends. + func testSpacedDeliveryBudgetChargesEachCharacterItsCallAndDelay() { + let delay = 0.08 + let fits = SynthesizedDeliveryBudget.maxTextLength(delaySeconds: delay) + XCTAssertFalse(SynthesizedDeliveryBudget.exceeds(textLength: fits, delaySeconds: delay)) + XCTAssertTrue(SynthesizedDeliveryBudget.exceeds(textLength: fits + 1, delaySeconds: delay)) + XCTAssertEqual( + SynthesizedDeliveryBudget.projectedSeconds(textLength: 10, delaySeconds: delay) + - SynthesizedDeliveryBudget.projectedSeconds(textLength: 9, delaySeconds: delay), + SynthesizedDeliveryBudget.characterInterval + + SynthesizedDeliveryBudget.synthesizeCallOverhead + + delay, + accuracy: 1e-9 ) + XCTAssertLessThan(fits, SynthesizedDeliveryBudget.maxTextLength(delaySeconds: 0)) + XCTAssertLessThan(SynthesizedDeliveryBudget.maxTextLength(delaySeconds: 0.2), fits) } func testSynthesizedBudgetExceededCarriesItsOwnCodeAndRecovery() { @@ -281,8 +296,12 @@ extension RunnerTests { "TEXT_INPUT_SYNTHESIS_BUDGET_EXCEEDED" ) // The recovery has to tell the caller to split the text: waiting it out or raising a timeout - // does nothing, because the pace is what makes the burst long, not the host being slow. - XCTAssertTrue(TextEntryFailure.synthesisBudgetExceeded.hint.contains("characters at a time")) + // does nothing, because the pace is what makes the burst long, not the host being slow. A + // delayed request fits fewer characters, so the hint names both budgets rather than promising + // the undelayed one to a caller retrying with --delay-ms. + let hint = TextEntryFailure.synthesisBudgetExceeded.hint + XCTAssertTrue(hint.contains("\(SynthesizedDeliveryBudget.maxTextLength(delaySeconds: 0)) characters at a time")) + XCTAssertTrue(hint.contains("\(SynthesizedDeliveryBudget.maxTextLength(delaySeconds: 0.08)) characters at --delay-ms 80")) } #if os(iOS) From a27335ef41c5dc82af272c40c4450e644c95e974 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 24 Sep 2026 18:46:09 +0200 Subject: [PATCH 5/8] docs(fill): document TEXT_INPUT_SYNTHESIS_BUDGET_EXCEEDED next to the other text-entry codes --- website/docs/docs/commands.md | 1 + 1 file changed, 1 insertion(+) diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index e5b2974062..e0b921fce2 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -488,6 +488,7 @@ agent-device gesture transform 200 420 80 -40 2 35 700 # combined pan, zoom, and If `type` reports `TEXT_INPUT_NOT_FOCUSED`, focus a visible text input and retry; when accessibility does not expose the input, use a coordinate focus command before typing. On iOS, if `type "\n"` reports `TEXT_INPUT_SYNTHESIS_UNAVAILABLE` after tapping a field while the software keyboard is hidden, show the software keyboard, then retry. The runner reports this error instead of risking input through an unreliable text-entry path. On iOS, if `fill` reports `TEXT_INPUT_COMMIT_NOT_OBSERVED`, the runner could not confirm the typed text reached the field — either it did not land before the runner's deadline, or the expected final text is identical to the field's placeholder. In the latter case, accessibility cannot distinguish committed text from an empty field rendering that placeholder, even if the field held content before dispatch. The field may hold none, part, or all of the text: run `snapshot -i` and inspect it. If it already matches, continue; otherwise retry with the full text quoted and `fill --delay-ms 80`, which replaces the whole value. Do not use `type`, which appends to whatever committed. This covers the coordinate-driven `fill` route taken when the accessibility channel is under load, which observes the field after synthesizing; it is not a guarantee that every text-entry route verifies its result. +On iOS, if `fill` reports `TEXT_INPUT_SYNTHESIS_BUDGET_EXCEEDED`, the text is longer than that coordinate-driven route can type inside one runner command at its bounded pace, and nothing was typed. Fill at most the number of characters the hint names and append the rest with separate `type` commands. `--delay-ms` lowers that budget, because every character then pays its own synthesize call and the delay; a longer timeout does not help. Use plain `fill` or `type` first for ordinary login and form fields. Use `--delay-ms` on `type` or `fill` only when a debounced search field or search-as-you-type input actually misses characters, or when the app must receive incremental updates. Delayed typing intentionally prefers paced character entry over clipboard-style fallbacks so the target field receives each incremental update. On Android, `fill` also verifies text and treats IME-owned capture as a terminal failure instead of retrying against the wrong field. From 80bf831fe4878bd14ee2ebb92f29bc25123cf62c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 24 Sep 2026 20:14:11 +0200 Subject: [PATCH 6/8] fix(ios-runner): bound synthesized delivery by the main-thread watchdog and pin the pace the app sees A 358-character fill through the daemon was abandoned by the runner's 30 s main-thread watchdog with the burst still typing. The delivery ceiling was a free-standing 30 s sized against the 45 s transport budget, while the whole command runs under mainThreadExecutionTimeout. The ceiling is now derived from that watchdog minus a focus allowance and the commit wait's ceiling (18 s), so the admitted text fits with the longest commit wait: 214 characters undelayed, 57 at --delay-ms 80. The app-owned-value lane test accepted either outcome and stayed green at 60 characters per second. It now asserts the spacing the fixture app receives: a burst's characters average at least one acknowledge window apart. That is red at 60 (11 edits in about 110 ms) and green at 12 (about 780 ms). A strict "the app keeps up" assertion is not possible: at a 40 ms window, 12 characters per second still lost one character in 13 of 20 bursts, because XCTest spaces the characters unevenly. The comments that claimed survival are narrowed to match. The call overhead and the --delay-ms 80 recovery delay move into TextEntryTiming beside the other text-entry timings, and both hints read the recovery delay from there. --- .../AgentDeviceRunner/AgentDeviceRunnerApp.m | 58 +++--- .../RunnerSynthesizedTextEntry.m | 12 +- .../RunnerTests+CommandDispatch.swift | 10 +- .../RunnerTests+SnapshotExecution.swift | 2 +- .../RunnerTests+SynthesizedTextEntry.swift | 7 +- .../RunnerTests+TextEntry.swift | 44 +++-- .../RunnerTests+TextTyping.swift | 6 +- .../RunnerTests.swift | 2 +- .../RunnerTests+RecordingTests.swift | 2 +- ...unnerTests+SynthesizedTextEntryTests.swift | 180 +++++++++--------- .../RunnerTests+TextEntryPolicyTests.swift | 33 ++-- .../RunnerTests+TransportTests.swift | 2 +- 12 files changed, 189 insertions(+), 169 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m index 9444644ebd..daf60e715e 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m @@ -70,6 +70,9 @@ @interface AgentDeviceRunnerViewController : UIViewController @property(nonatomic, assign) NSUInteger textEntryWriteBacks; @property(nonatomic, copy, nullable) NSString *textEntryRenderedValue; @property(nonatomic, assign) NSTimeInterval textEntryLastEditTime; +@property(nonatomic, assign) NSTimeInterval textEntryBurstStartTime; +@property(nonatomic, assign) NSUInteger textEntryBurstEdits; +@property(nonatomic, assign) NSTimeInterval textEntryBurstMinGap; @property(nonatomic, assign) NSTimeInterval textEntryAcknowledgeWindowSeconds; @property(nonatomic, assign) BOOL alertFixtureStarted; @property(nonatomic, strong) NSTimer *alertActivationBusyBackstop; @@ -182,9 +185,14 @@ - (void)updateAlertActionStatus { } - (void)updateTextEntryWriteBackStatus { - self.textEntryWriteBackStatus.text = [NSString stringWithFormat:@"Edits: %lu; write-backs: %lu", - (unsigned long)self.textEntryRenderedEdits, - (unsigned long)self.textEntryWriteBacks]; + NSTimeInterval burstSpan = self.textEntryLastEditTime - self.textEntryBurstStartTime; + self.textEntryWriteBackStatus.text = [NSString + stringWithFormat:@"edits=%lu write-backs=%lu burst-edits=%lu burst-ms=%lu min-gap-ms=%lu", + (unsigned long)self.textEntryRenderedEdits, + (unsigned long)self.textEntryWriteBacks, + (unsigned long)self.textEntryBurstEdits, + (unsigned long)llround(burstSpan * 1000), + (unsigned long)llround(self.textEntryBurstMinGap * 1000)]; } - (void)presentAlertFixtureReplacement:(BOOL)replacement { @@ -244,32 +252,39 @@ - (void)viewDidAppear:(BOOL)animated { } #endif -// How fast an app that owns this field's value can acknowledge edits: one render per window. An -// edit that arrives inside that window overtook the render still in flight, so the value that -// render commits predates it and writing it erases the characters that got ahead of the app. The -// app then reads its own erasure back into its model, which is why the field stays wrong instead of -// healing when the burst finishes. The window is decided at the edit rather than scheduled, so a -// loaded host, which stretches the gaps between characters, can only make this app keep up better. -static const NSTimeInterval AgentDeviceTextEntryDefaultAcknowledgeWindowSeconds = 0.04; - -static NSTimeInterval AgentDeviceTextEntryAcknowledgeWindow(id argument) { +// How fast an app that owns this field's value can acknowledge edits: one render per window, passed +// by the test as `--agent-device-text-entry-acknowledge-window `. An edit that arrives +// inside that window overtook the render still in flight, so the value that render commits predates +// it and writing it erases the characters that got ahead of the app. The app then reads its own +// erasure back into its model, which is why the field stays wrong instead of healing when the burst +// finishes. The window is decided at the edit rather than scheduled, so a loaded host, which +// stretches the gaps between characters, can only make this app keep up better. +static NSTimeInterval AgentDeviceTextEntryAcknowledgeWindow(void) { NSArray *arguments = NSProcessInfo.processInfo.arguments; - NSUInteger index = [arguments indexOfObject:argument]; - if (index == NSNotFound || index + 1 >= arguments.count) { - return AgentDeviceTextEntryDefaultAcknowledgeWindowSeconds; - } - NSTimeInterval seconds = [arguments[index + 1] doubleValue]; - return seconds > 0 ? seconds : AgentDeviceTextEntryDefaultAcknowledgeWindowSeconds; + NSUInteger index = [arguments indexOfObject:@"--agent-device-text-entry-acknowledge-window"]; + return index == NSNotFound || index + 1 >= arguments.count ? 0 : [arguments[index + 1] doubleValue]; } +// Edits further apart than this belong to different bursts: one runner command's characters arrive +// well inside it, and two commands are separated by at least a commit-wait poll and a status read. +static const NSTimeInterval AgentDeviceTextEntryBurstBreakSeconds = 1.0; + - (void)agentDeviceTextEntryDidChange:(UITextField *)textField { // A field whose app owns its value, the way a controlled React Native `TextInput` does. A burst // typed faster than the app renders loses the characters that arrived while a render was in // flight, and the field settles stable short of the request. if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-app-owned-value"]) { NSTimeInterval now = NSProcessInfo.processInfo.systemUptime; - BOOL overtookARender = self.textEntryRenderedValue != nil && - (now - self.textEntryLastEditTime) < self.textEntryAcknowledgeWindowSeconds; + NSTimeInterval gap = now - self.textEntryLastEditTime; + BOOL overtookARender = self.textEntryRenderedValue != nil && gap < self.textEntryAcknowledgeWindowSeconds; + if (self.textEntryBurstEdits == 0 || gap > AgentDeviceTextEntryBurstBreakSeconds) { + self.textEntryBurstStartTime = now; + self.textEntryBurstEdits = 0; + self.textEntryBurstMinGap = 0; + } else if (self.textEntryBurstEdits == 1 || gap < self.textEntryBurstMinGap) { + self.textEntryBurstMinGap = gap; + } + self.textEntryBurstEdits += 1; self.textEntryLastEditTime = now; if (overtookARender) { if (![textField.text isEqualToString:self.textEntryRenderedValue]) { @@ -343,8 +358,7 @@ - (void)viewDidLoad { [textField.heightAnchor constraintEqualToConstant:44], ]]; if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-app-owned-value"]) { - self.textEntryAcknowledgeWindowSeconds = - AgentDeviceTextEntryAcknowledgeWindow(@"--agent-device-text-entry-acknowledge-window"); + self.textEntryAcknowledgeWindowSeconds = AgentDeviceTextEntryAcknowledgeWindow(); // Reports how many edits this app rendered and how many writes it had to make because a // character overtook one, so a lane test can tell a burst the app kept up with from an inert // fixture. Counts only: no field content crosses into the test. diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedTextEntry.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedTextEntry.m index 8fab006ef8..d76deeb4eb 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedTextEntry.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedTextEntry.m @@ -10,11 +10,13 @@ // its field's value and re-applies it after the edit (a controlled React Native `TextInput`, an // async validator) can acknowledge: such a write lands between two characters of the burst and // erases what was typed while it was in flight, leaving a value that is stable short of the -// request. 12 characters/second spaces them ~83 ms apart, so an app whose write-back lands inside -// one character interval no longer has anything to erase. It is not immunity: a render still in -// flight 150 ms after an edit corrupted an 11-character burst at this pace too. The -// `--agent-device-text-entry-app-owned-value` fixture and the pace policy test pin this bound; the -// delivery ceiling in TextEntryTiming bounds what the pace costs a long text. +// request. 12 characters/second spaces them ~83 ms apart on average, which reduces that loss but +// does not remove it: XCTest does not space the characters evenly, and two of them can reach the +// app a few milliseconds apart. Against a fixture app that acknowledges each edit within 40 ms, 60 +// characters/second left 1 of 11 characters in 20 of 20 bursts, and this pace left 10 or 11. The +// command refuses a field left short; back-pressure from the field (#2906) is what would prevent +// it. The app-owned-value lane test pins the average spacing the app sees, and the delivery budget +// in TextEntryTiming bounds what the pace costs a long text. static const NSUInteger RunnerTextEntryTypingSpeedCharactersPerSecond = 12; typedef id (*RunnerTextMsgSendInit)(id, SEL, NSString *); diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandDispatch.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandDispatch.swift index f58f712686..f58b463dca 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandDispatch.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandDispatch.swift @@ -144,7 +144,7 @@ extension RunnerTests { } return try runMainThreadWork( "command_execution", - timeout: mainThreadExecutionTimeout, + timeout: Self.mainThreadExecutionTimeout, timeoutError: mainThreadExecutionTimeoutError ) { try self.executeOnMainSafely(command: command, routeToSpringboard: routeToSpringboard) @@ -247,7 +247,7 @@ extension RunnerTests { while true { let failureCountBefore = try runMainThreadWork( "recorded_failure_count", - timeout: mainThreadExecutionTimeout, + timeout: Self.mainThreadExecutionTimeout, timeoutError: mainThreadExecutionTimeoutError ) { self.currentXCTestFailureCount() @@ -264,7 +264,7 @@ extension RunnerTests { } let recordedFailureResponse = try runMainThreadWork( "recorded_failure_count", - timeout: mainThreadExecutionTimeout, + timeout: Self.mainThreadExecutionTimeout, timeoutError: mainThreadExecutionTimeoutError ) { self.didRecordXCTestFailure(since: failureCountBefore) @@ -274,7 +274,7 @@ extension RunnerTests { if let recordedFailureResponse { try runMainThreadWork( "target_invalidation", - timeout: mainThreadExecutionTimeout, + timeout: Self.mainThreadExecutionTimeout, timeoutError: mainThreadExecutionTimeoutError ) { self.invalidateCachedTarget(reason: "xctest_recorded_failure") @@ -289,7 +289,7 @@ extension RunnerTests { hasRetried = true try runMainThreadWork( "target_invalidation", - timeout: mainThreadExecutionTimeout, + timeout: Self.mainThreadExecutionTimeout, timeoutError: mainThreadExecutionTimeoutError ) { self.invalidateCachedTarget(reason: "response_unavailable") diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotExecution.swift index ec0b2c8faa..59ea13b9fa 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotExecution.swift @@ -11,7 +11,7 @@ extension RunnerTests { private func executeSnapshotDispatchedOnce(command: Command) throws -> Response { let preparation: SnapshotCommandPreparation = try runMainThreadWork( "command_preparation", - timeout: mainThreadExecutionTimeout, + timeout: Self.mainThreadExecutionTimeout, timeoutError: mainThreadExecutionTimeoutError ) { () -> SnapshotCommandPreparation in switch try self.prepareActiveCommandContextSafely(command: command, routeToSpringboard: false) { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift index bc8fdace6c..554b96087b 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift @@ -109,11 +109,6 @@ extension RunnerTests { 1.0 / Double(RunnerSynthesizedTextEntry.typingSpeedCharactersPerSecond()) } - /// Seconds one synthesize call costs beyond typing its characters. A one-character call at the - /// 83 ms pace took 222 ms on average on an iPhone 17 Pro simulator (212-617 ms over 235 calls), - /// which a spaced plan pays once per character. - static let synthesizeCallOverhead: TimeInterval = 0.15 - /// Seconds the plan spends posting, charged per `synthesizedReplacementSteps` step: each /// synthesize call types its characters at the pace and pays its overhead, and a spaced plan /// sleeps `delaySeconds` between two calls. @@ -122,7 +117,7 @@ extension RunnerTests { ? textLength : 1 return Double(textLength) * characterInterval - + Double(calls) * synthesizeCallOverhead + + Double(calls) * TextEntryTiming.synthesizeCallOverhead + Double(calls - 1) * delaySeconds } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift index e031afb60d..ba76b29340 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift @@ -30,9 +30,13 @@ extension RunnerTests { case .synthesisUnavailable: return "Show the software keyboard, then retry type." case .commitNotObserved: - return "The field may hold none, part, or all of the text. Run snapshot -i and inspect the field: if it already matches, continue; otherwise retry fill with the full text quoted and --delay-ms 80. Do not use type, which appends to whatever committed." + return "The field may hold none, part, or all of the text. Run snapshot -i and inspect the field: if it already matches, continue; otherwise retry fill with the full text quoted and --delay-ms \(TextEntryTiming.recoveryDelayMilliseconds). Do not use type, which appends to whatever committed." case .synthesisBudgetExceeded: - return "Fill at most \(SynthesizedDeliveryBudget.maxTextLength(delaySeconds: 0)) characters at a time without --delay-ms and append the rest with separate type commands, keeping each command inside that budget. --delay-ms lowers the budget, because every character then pays its own synthesize call and the delay: \(SynthesizedDeliveryBudget.maxTextLength(delaySeconds: 0.08)) characters at --delay-ms 80. A longer timeout does not help: this route is chosen when the accessibility channel is already degraded, and the pace is what makes the text long." + let recoveryDelay = TextEntryTiming.recoveryDelayMilliseconds + let recoveryBudget = SynthesizedDeliveryBudget.maxTextLength( + delaySeconds: Double(recoveryDelay) / 1000 + ) + return "Fill at most \(SynthesizedDeliveryBudget.maxTextLength(delaySeconds: 0)) characters at a time without --delay-ms and append the rest with separate type commands, keeping each command inside that budget. --delay-ms lowers the budget, because every character then pays its own synthesize call and the delay: \(recoveryBudget) characters at --delay-ms \(recoveryDelay). A longer timeout does not help: this route is chosen when the accessibility channel is already degraded, and the pace is what makes the text long." } } } @@ -55,22 +59,30 @@ extension RunnerTests { /// condemned at exactly the same instant it always was (see `SynthesizedCommitDeadline`). static let synthesizedCommitStallTimeout: TimeInterval = 3.0 /// The commit wait's absolute bound, however long characters keep arriving. Synthesized - /// delivery happens before this wait starts and is bounded by `synthesizedDeliveryCeiling`, so - /// the two together stay inside the daemon's per-command budget - /// (`RUNNER_COMMAND_TIMEOUT_MS`, 45s), which also has to cover focus, clear and verification. + /// delivery happens before this wait starts and is bounded by `synthesizedDeliveryCeiling`. static let synthesizedCommitCeiling: TimeInterval = 10.0 - /// How long a synthesized burst may spend posting its characters. The private synthesize call - /// delivers as it returns, so this is the slice of the 45s command budget the burst itself may - /// take, with the commit ceiling, focus, clear and verification subtracted and margin left for - /// the round trip each character costs. Text that does not fit is refused before the first - /// character is posted: a transport timeout would end the command with the runner still typing, - /// and the next command would find it busy. - static let synthesizedDeliveryCeiling: TimeInterval = 30.0 - /// The edit-acknowledge budget the synthesized pace is sized for: an app that renders each edit - /// within this window has nothing to erase when a burst replaces its field, and one that needs - /// longer loses the characters that arrive while a render is in flight. - /// `testSynthesizedPaceLeavesRoomForAnAppToAcknowledgeEachEdit` pins the pace against it. + /// What a synthesized replacement spends before its first character: focusing the field took + /// 374–500 ms through the daemon on an iPhone 17 Pro simulator. + static let synthesizedReplacementFocusAllowance: TimeInterval = 2.0 + /// How long a synthesized burst may spend posting its characters: what the command's + /// main-thread watchdog leaves after focus and the longest commit wait. The private synthesize + /// call delivers as it returns, so text that does not fit is refused before the first character + /// is posted; otherwise the watchdog abandons the command with the runner still typing. + static let synthesizedDeliveryCeiling: TimeInterval = RunnerTests.mainThreadExecutionTimeout + - synthesizedReplacementFocusAllowance + - synthesizedCommitCeiling + /// The edit-acknowledge window the synthesized pace is sized for: on average, a burst's + /// characters reach the app at least this far apart. XCTest spaces them unevenly, so an app + /// with this window can still lose a character that arrives early; the command then refuses the + /// short value (#2906 tracks preventing it). The pace policy test and the app-owned-value lane + /// test pin the pace against it. static let synthesizedAcknowledgeWindowSeconds: TimeInterval = 0.04 + /// What one private synthesize call costs beyond typing its characters, which a `--delay-ms` + /// plan pays once per character. One-character calls at the shipped pace took 222 ms on average + /// on an iPhone 17 Pro simulator (212–617 ms over 235 calls), 83 ms of it the character. + static let synthesizeCallOverhead: TimeInterval = 0.15 + /// The spacing the `TEXT_INPUT_COMMIT_NOT_OBSERVED` recovery tells the caller to retry with. + static let recoveryDelayMilliseconds = 80 static let synthesizedCommitPollInterval: TimeInterval = 0.2 } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift index abd589d9d0..64852a45a4 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift @@ -163,9 +163,9 @@ extension RunnerTests { } else if activeTarget.prefersFocusedElement && isKeyboardVisible(app: app) { #if os(iOS) // Text the command budget cannot carry at the synthesized pace goes through the verified - // application-wide typing instead. The synthesizer types at the pace an app that owns its - // field can acknowledge, and a burst that long outlasts the command while the runner is - // still posting it. `app.typeText` is this branch's existing fallback and its value is + // application-wide typing instead. The synthesizer's pace is slowed for fields whose app + // owns the value, and a burst that long outlasts the command while the runner is still + // posting it. `app.typeText` is this branch's existing fallback and its value is // verified afterwards, so the length costs the pace, not the check. if SynthesizedDeliveryBudget.exceeds(textLength: value.count, delaySeconds: 0) { textEntryRoute = "xctest-application-fallback" diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift index 725261ff2a..473307e802 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift @@ -68,7 +68,7 @@ final class RunnerTests: XCTestCase { // interactions clear it before it can become stale. var textEntryTapWitness: TextEntryTapWitness? let maxRequestBytes = 2 * 1024 * 1024 - let mainThreadExecutionTimeout: TimeInterval = 30 + static let mainThreadExecutionTimeout: TimeInterval = 30 let appExistenceTimeout: TimeInterval = 30 let retryCooldown: TimeInterval = 0.2 let postSnapshotInteractionDelay: TimeInterval = 0.2 diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+RecordingTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+RecordingTests.swift index 0c97926342..8d2c75b439 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+RecordingTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+RecordingTests.swift @@ -448,7 +448,7 @@ extension RunnerTests { for _ in 0..<2 { try self.runMainThreadWork( "command_execution", - timeout: self.mainThreadExecutionTimeout, + timeout: Self.mainThreadExecutionTimeout, timeoutError: self.mainThreadExecutionTimeoutError ) { Thread.sleep(forTimeInterval: self.recordingFrameCaptureTimeout + 0.3) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift index a2088d24f5..83430eb52c 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift @@ -2,139 +2,139 @@ import XCTest extension RunnerTests { #if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS) - /// Reads the fixture's `Edits: n; write-backs: m` counter. Counts only: the field's contents never - /// cross into the test. - private func textEntryFixtureCounts(app: XCUIApplication) throws -> (edits: Int, writeBacks: Int) { - let counts = app.staticTexts["agent-device-text-entry-write-backs"].label - .split(whereSeparator: { !$0.isNumber }).compactMap { Int($0) } - XCTAssertEqual(counts.count, 2, "unexpected write-back status: \(counts)") - return (try XCTUnwrap(counts.first), try XCTUnwrap(counts.last)) + /// What the app-owned-value fixture reports about the edits it saw. Counts and timings only: the + /// field's contents never cross into the test. + struct AppOwnedFieldStatus { + let edits: Int + let writeBacks: Int + /// Edits in the latest burst, and the milliseconds between its first and last edit. + let burstEdits: Int + let burstMilliseconds: Int + let minimumGapMilliseconds: Int } - /// An app that owns its field's value renders it some time after the edit that produced it, the - /// way a controlled React Native `TextInput` does. A burst typed faster than that render has its - /// in-flight characters erased by the app's own write, which the app then reads back into its - /// model, so the field settles stable short of the request — the shape CI reported for - /// `fill id="field-email" ada@example` as `aexample`. - /// - /// What this pins is the runner's half of that race, which is all the runner owns: a field the app - /// rewrote mid-burst either ends with the requested text and an ok, or the command refuses. An ok - /// over a short value was the original defect. Whether the app wins a round trip is decided by the - /// host, not by the pace, because XCTest does not deliver `typingSpeed:` characters evenly — CI - /// observed two of them 4 ms apart at the shipped pace — so the assertion follows the fixture's own - /// counter rather than assuming the app kept up. - /// - /// The pace itself is pinned without a race by `testSynthesizedPaceLeavesRoomForAnAppToAcknowledgeEachEdit`, - /// against the 40 ms window this route is sized for. The window here is 5 ms — an app that renders - /// every edit within it is one the shipped pace is not expected to outrun, so the strict branch is - /// the one a healthy host takes. At the pre-fix 60-characters-per-second pace and the 40 ms policy - /// window the same run leaves the field holding `a` after 20 write-backs, which is the half the - /// pace exists for. - func testSynthesizedReplacementSurvivesFieldValueWrittenBackByTheApp() throws { - app.launchArguments = [ - "--agent-device-text-entry-regression", - "--agent-device-text-entry-app-owned-value", - "--agent-device-text-entry-acknowledge-window", "0.005" - ] - app.launch() - defer { - clearSnapshotXCTestChannelPenalty(reason: "test-cleanup") - invalidateCachedTarget(reason: "unit_test_cleanup") - app.terminate() + func appOwnedFieldStatus() throws -> AppOwnedFieldStatus { + let label = app.staticTexts["agent-device-text-entry-write-backs"].label + var fields: [String: Int] = [:] + for pair in label.split(separator: " ") { + let parts = pair.split(separator: "=") + if parts.count == 2, let value = Int(parts[1]) { fields[String(parts[0])] = value } } - XCTAssertTrue(app.waitForExistence(timeout: appExistenceTimeout)) + func field(_ name: String) throws -> Int { + try XCTUnwrap(fields[name], "fixture status lacks \(name): \(label)") + } + return AppOwnedFieldStatus( + edits: try field("edits"), + writeBacks: try field("write-backs"), + burstEdits: try field("burst-edits"), + burstMilliseconds: try field("burst-ms"), + minimumGapMilliseconds: try field("min-gap-ms") + ) + } + /// Launches the text-entry fixture, focuses its field, and penalizes the XCTest channel, so a + /// coordinate replacement takes the synthesized first-responder route. + func focusSynthesizedReplacementField(extraLaunchArguments: [String] = []) throws -> XCUIElement { + app.launchArguments = ["--agent-device-text-entry-regression"] + extraLaunchArguments + app.launch() + XCTAssertTrue(app.waitForExistence(timeout: appExistenceTimeout)) let textField = app.textFields["agent-device-hardware-keyboard-input"] XCTAssertTrue(textField.waitForExistence(timeout: appExistenceTimeout)) currentApp = app currentBundleId = "com.callstack.agentdevice.runner" currentAppProcessIdentifier = try XCTUnwrap(Self.processIdentifier(of: app)) - let focusCommand = try runnerCommandFixture( - #"{"command":"tap","commandId":"tap-app-owned-field","selectorKey":"id","selectorValue":"agent-device-hardware-keyboard-input"}"# + #"{"command":"tap","commandId":"tap-replacement-field","selectorKey":"id","selectorValue":"agent-device-hardware-keyboard-input"}"# ) let focusResponse = try executeOnMainPrepared(command: focusCommand, activeApp: app) XCTAssertTrue(focusResponse.ok, String(describing: focusResponse.error)) - penalizeSnapshotXCTestChannel(bundleId: nil, reason: "test") + return textField + } + func replaceSynthesizedFieldText( + _ textField: XCUIElement, + text: String, + commandId: String + ) throws -> Response { let frame = textField.frame - XCTAssertFalse(frame.isEmpty) - var sawKeepUp = false - var sawRewrite = false + let command = try runnerCommandFixture( + #"{"command":"type","commandId":"\#(commandId)","text":"\#(text)","textEntryMode":"replace","x":\#(frame.midX),"y":\#(frame.midY)}"# + ) + let failuresBeforeType = currentXCTestFailureCount() + let response = executeTypeCommand(activeApp: app, command: command) + XCTAssertFalse(didRecordXCTestFailure(since: failuresBeforeType)) + return response + } + + func tearDownSynthesizedReplacementField() { + clearSnapshotXCTestChannelPenalty(reason: "test-cleanup") + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } + + /// An app that owns its field's value renders it some time after the edit that produced it, the + /// way a controlled React Native `TextInput` does. A burst typed faster than that render has its + /// in-flight characters erased by the app's own write, which the app then reads back into its + /// model, so the field settles stable short of the request — the shape CI reported for + /// `fill id="field-email" ada@example` as `aexample` (#2080). + /// + /// Two halves, each independent of host timing: + /// - The pace: across a burst, the characters reach the app at least one acknowledge window apart + /// on average. At the pre-fix 60 characters per second they arrive about 13 ms apart and this + /// goes red. It is an average because XCTest does not space `typingSpeed:` characters evenly — + /// two of them can reach the app a few milliseconds apart at any pace — so whether an app with + /// this window keeps up with one particular burst is not something the runner can promise. + /// - The runner's: a field the app rewrote mid-burst never reports ok. An ok over a short value + /// was the original defect. + func testSynthesizedReplacementPacesAnAppOwnedFieldAtItsAcknowledgeWindow() throws { + let window = TextEntryTiming.synthesizedAcknowledgeWindowSeconds + let textField = try focusSynthesizedReplacementField(extraLaunchArguments: [ + "--agent-device-text-entry-app-owned-value", + "--agent-device-text-entry-acknowledge-window", String(window), + ]) + defer { tearDownSynthesizedReplacementField() } // Twice: the second replacement selects the first one's value away, which is the shape the // reported CI trace had — a `fill` onto a field that already held text. for commandId in ["fill-app-owned-first", "fill-app-owned-second"] { - let command = try runnerCommandFixture( - #"{"command":"type","commandId":"\#(commandId)","text":"ada@example","textEntryMode":"replace","x":\#(frame.midX),"y":\#(frame.midY)}"# - ) - let writeBacksBefore = try textEntryFixtureCounts(app: app).writeBacks - let failuresBeforeType = currentXCTestFailureCount() - let response = executeTypeCommand(activeApp: app, command: command) - XCTAssertFalse(didRecordXCTestFailure(since: failuresBeforeType)) - let writeBacksAfter = try textEntryFixtureCounts(app: app).writeBacks + let before = try appOwnedFieldStatus() + let response = try replaceSynthesizedFieldText(textField, text: "ada@example", commandId: commandId) + let after = try appOwnedFieldStatus() - if writeBacksAfter == writeBacksBefore { - sawKeepUp = true + XCTAssertGreaterThan(after.burstEdits, 1, "the fixture saw no burst") + XCTAssertGreaterThanOrEqual( + Double(after.burstMilliseconds), + Double(after.burstEdits - 1) * window * 1000, + "\(after.burstEdits) edits reached the app in \(after.burstMilliseconds) ms " + + "(closest pair \(after.minimumGapMilliseconds) ms)" + ) + if after.writeBacks == before.writeBacks { XCTAssertTrue(response.ok, String(describing: response.error)) XCTAssertEqual(response.data?.textEntryRoute, "synthesized-first-responder-replacement") XCTAssertEqual(String(describing: textField.value ?? ""), "ada@example") } else { - sawRewrite = true XCTAssertFalse(response.ok, "a field the app rewrote cannot report success") XCTAssertEqual(response.error?.code, "TEXT_INPUT_COMMIT_NOT_OBSERVED") } } - - // The fixture has to have run for either branch above to mean anything. - XCTAssertGreaterThan(try textEntryFixtureCounts(app: app).edits, 0) - if sawRewrite { - NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_APP_OWNED_VALUE branch=app-won-round-trip") - } - XCTAssertTrue(sawKeepUp || sawRewrite) } /// A replacement the command budget cannot carry is refused before the first character is posted, /// so a `fill` cannot end in a transport timeout that leaves the runner typing into a field nobody /// is waiting for and the next command finding it busy. func testSynthesizedReplacementRefusesTextBeyondTheDeliveryBudget() throws { - app.launchArguments = ["--agent-device-text-entry-regression"] - app.launch() - defer { - clearSnapshotXCTestChannelPenalty(reason: "test-cleanup") - invalidateCachedTarget(reason: "unit_test_cleanup") - app.terminate() - } - XCTAssertTrue(app.waitForExistence(timeout: appExistenceTimeout)) - - let textField = app.textFields["agent-device-hardware-keyboard-input"] - XCTAssertTrue(textField.waitForExistence(timeout: appExistenceTimeout)) - currentApp = app - currentBundleId = "com.callstack.agentdevice.runner" - currentAppProcessIdentifier = try XCTUnwrap(Self.processIdentifier(of: app)) - - let focusCommand = try runnerCommandFixture( - #"{"command":"tap","commandId":"tap-budget-field","selectorKey":"id","selectorValue":"agent-device-hardware-keyboard-input"}"# - ) - XCTAssertTrue(try executeOnMainPrepared(command: focusCommand, activeApp: app).ok) - - penalizeSnapshotXCTestChannel(bundleId: nil, reason: "test") - let frame = textField.frame + let textField = try focusSynthesizedReplacementField() + defer { tearDownSynthesizedReplacementField() } let text = String( repeating: "x", count: SynthesizedDeliveryBudget.maxTextLength(delaySeconds: 0) + 1 ) - let command = try runnerCommandFixture( - #"{"command":"type","commandId":"fill-over-budget","text":"\#(text)","textEntryMode":"replace","x":\#(frame.midX),"y":\#(frame.midY)}"# - ) - let failuresBeforeType = currentXCTestFailureCount() - let response = executeTypeCommand(activeApp: app, command: command) + let response = try replaceSynthesizedFieldText(textField, text: text, commandId: "fill-over-budget") XCTAssertFalse(response.ok) XCTAssertEqual(response.error?.code, "TEXT_INPUT_SYNTHESIS_BUDGET_EXCEEDED") - XCTAssertFalse(didRecordXCTestFailure(since: failuresBeforeType)) XCTAssertEqual(String(describing: textField.value ?? ""), "") } #endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift index bfb8b1ece1..c277b8669b 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift @@ -241,10 +241,10 @@ extension RunnerTests { ) } - // The pace is the guarantee that a field the app owns survives a replacement, so it cannot drift - // on its own: one character interval has to leave that app at least twice the acknowledge window - // the route is sized for, or the burst outruns the render and loses the characters in flight - // again (#2080). Raising the pace or shrinking the window both land here. + // The pace is what keeps a field the app owns from losing most of a replacement (#2080), so it + // cannot drift on its own: one character interval has to leave that app at least twice the + // acknowledge window the route is sized for. The host lane runs this on every PR; the iOS lane's + // app-owned-value test checks the spacing the app actually receives. func testSynthesizedPaceLeavesRoomForAnAppToAcknowledgeEachEdit() { XCTAssertGreaterThanOrEqual( SynthesizedDeliveryBudget.characterInterval, @@ -253,28 +253,21 @@ extension RunnerTests { } // Characters are delivered while the private synthesize call is still running, so text longer - // than the delivery ceiling would still be arriving when the transport gives up on the command — - // leaving a runner mid-burst that the next command finds busy. The budget turns that into a - // refusal decided up front, at the boundary and not after the first character is posted. + // than the delivery ceiling would still be arriving when the main-thread watchdog abandons the + // command. The budget turns that into a refusal decided up front, at the boundary and not after + // the first character is posted. func testSynthesizedDeliveryBudgetRefusesTextThatOutrunsTheCommand() { let fits = SynthesizedDeliveryBudget.maxTextLength(delaySeconds: 0) XCTAssertGreaterThan(fits, 0) XCTAssertFalse(SynthesizedDeliveryBudget.exceeds(textLength: fits, delaySeconds: 0)) XCTAssertTrue(SynthesizedDeliveryBudget.exceeds(textLength: fits + 1, delaySeconds: 0)) - // The burst and the commit wait it is followed by both have to fit the runner's per-command - // budget (`RUNNER_COMMAND_TIMEOUT_MS`, 45s in packages/platform-apple/src/runner/ - // runner-transport.ts), which also carries focus, clear and verification. - XCTAssertLessThanOrEqual( - TextEntryTiming.synthesizedDeliveryCeiling + TextEntryTiming.synthesizedCommitCeiling, - 45 - ) } // A spaced plan posts each character in its own synthesize call and sleeps between two of them, // so a character costs the pace, the call's overhead and the delay together, not the larger of - // pace and delay. `--delay-ms 80` is the retry TEXT_INPUT_COMMIT_NOT_OBSERVED recommends. + // pace and delay. The delay checked is the retry TEXT_INPUT_COMMIT_NOT_OBSERVED recommends. func testSpacedDeliveryBudgetChargesEachCharacterItsCallAndDelay() { - let delay = 0.08 + let delay = Double(TextEntryTiming.recoveryDelayMilliseconds) / 1000 let fits = SynthesizedDeliveryBudget.maxTextLength(delaySeconds: delay) XCTAssertFalse(SynthesizedDeliveryBudget.exceeds(textLength: fits, delaySeconds: delay)) XCTAssertTrue(SynthesizedDeliveryBudget.exceeds(textLength: fits + 1, delaySeconds: delay)) @@ -282,7 +275,7 @@ extension RunnerTests { SynthesizedDeliveryBudget.projectedSeconds(textLength: 10, delaySeconds: delay) - SynthesizedDeliveryBudget.projectedSeconds(textLength: 9, delaySeconds: delay), SynthesizedDeliveryBudget.characterInterval - + SynthesizedDeliveryBudget.synthesizeCallOverhead + + TextEntryTiming.synthesizeCallOverhead + delay, accuracy: 1e-9 ) @@ -301,7 +294,11 @@ extension RunnerTests { // the undelayed one to a caller retrying with --delay-ms. let hint = TextEntryFailure.synthesisBudgetExceeded.hint XCTAssertTrue(hint.contains("\(SynthesizedDeliveryBudget.maxTextLength(delaySeconds: 0)) characters at a time")) - XCTAssertTrue(hint.contains("\(SynthesizedDeliveryBudget.maxTextLength(delaySeconds: 0.08)) characters at --delay-ms 80")) + let recoveryDelay = TextEntryTiming.recoveryDelayMilliseconds + let recoveryBudget = SynthesizedDeliveryBudget.maxTextLength( + delaySeconds: Double(recoveryDelay) / 1000 + ) + XCTAssertTrue(hint.contains("\(recoveryBudget) characters at --delay-ms \(recoveryDelay)")) } #if os(iOS) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TransportTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TransportTests.swift index ffa3d56d36..bdbd9b5e8c 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TransportTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TransportTests.swift @@ -56,7 +56,7 @@ extension RunnerTests { box.result = result executed.fulfill() } - guard XCTWaiter.wait(for: [executed], timeout: mainThreadExecutionTimeout + 5) == .completed, + guard XCTWaiter.wait(for: [executed], timeout: Self.mainThreadExecutionTimeout + 5) == .completed, let result = box.result else { throw NSError( From a58713b503348184acb17bc3941eb0a88c925e7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 24 Sep 2026 21:03:34 +0200 Subject: [PATCH 7/8] fix(ios-runner): charge the delivery budget the whole type command The element-less, keyboard-visible branch tested the budget against the chunk it was about to post. An append peels its first character for warmup and a `--delay-ms` plan posts one character per call, so no chunk ever looked long and the paced synthesizer kept a text the command cannot carry: 215 characters arrived as 1 + 214, both inside the budget. The branch is now charged the command's own length and delay. That branch had no test, so it gained one: 215 characters through an unresolved target type application-wide, and the value arrives unverified because the branch has no element to read back. The fixture field keeps a real input view under `--agent-device-text-entry-soft-keyboard`, without which no branch requiring a visible keyboard is reachable. --- .../AgentDeviceRunner/AgentDeviceRunnerApp.m | 8 ++- .../RunnerTests+TextTyping.swift | 15 +++-- .../RunnerTests+TextTypingTests.swift | 62 +++++++++++++++++++ 3 files changed, 78 insertions(+), 7 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m index daf60e715e..4d70b4273c 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m @@ -345,7 +345,13 @@ - (void)viewDidLoad { UITextField *textField = [[UITextField alloc] init]; textField.accessibilityIdentifier = @"agent-device-hardware-keyboard-input"; textField.borderStyle = UITextBorderStyleRoundedRect; - textField.inputView = [[UIView alloc] initWithFrame:CGRectMake(0, 0, 1, 1)]; + // An empty input view keeps the software keyboard down, which is the hardware-keyboard responder + // these routes are addressed to. `--agent-device-text-entry-soft-keyboard` leaves the real input + // view in place, so a lane test can reach the branch that requires a visible keyboard. + if (![NSProcessInfo.processInfo.arguments + containsObject:@"--agent-device-text-entry-soft-keyboard"]) { + textField.inputView = [[UIView alloc] initWithFrame:CGRectMake(0, 0, 1, 1)]; + } [textField addTarget:self action:@selector(agentDeviceTextEntryDidChange:) forControlEvents:UIControlEventEditingChanged]; diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift index 64852a45a4..ddd5e7a29e 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift @@ -162,12 +162,15 @@ extension RunnerTests { return (currentTarget, nil) } else if activeTarget.prefersFocusedElement && isKeyboardVisible(app: app) { #if os(iOS) - // Text the command budget cannot carry at the synthesized pace goes through the verified - // application-wide typing instead. The synthesizer's pace is slowed for fields whose app - // owns the value, and a burst that long outlasts the command while the runner is still - // posting it. `app.typeText` is this branch's existing fallback and its value is - // verified afterwards, so the length costs the pace, not the check. - if SynthesizedDeliveryBudget.exceeds(textLength: value.count, delaySeconds: 0) { + // Text the command budget cannot carry at the synthesized pace goes through application-wide + // typing instead. The synthesizer's pace is slowed for fields whose app owns the value, and a + // burst that long outlasts the command while the runner is still posting it. The ceiling is + // what the command's watchdog leaves, so it is charged the whole command: an append peels its + // first character for warmup and a `--delay-ms` plan dispatches one character at a time, and + // neither chunk would look long on its own. This branch's target has no element to type into, + // so nothing can be read back afterwards: the value arrives unverified, as it does for this + // route's older synthesizer-unavailable fallback. + if SynthesizedDeliveryBudget.exceeds(textLength: text.count, delaySeconds: delaySeconds) { textEntryRoute = "xctest-application-fallback" NSLog( "AGENT_DEVICE_RUNNER_TEXT_ENTRY_ROUTE route=xctest-application-fallback " diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift index 214f599bd6..7c8c784ca6 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift @@ -148,6 +148,59 @@ extension RunnerTests { XCTAssertFalse(textField.exists) } + // Text past the delivery budget cannot be paced into a field the runner cannot resolve, so it goes + // through application-wide typing. The budget is charged the whole command, which is what the + // length below pins: an append peels its first character for warmup, so a per-dispatch charge + // would find both of its pieces inside the budget and pace all 215 characters. The target carries + // no element by construction, so nothing on that route can read the value back: the command reports + // it unverified and this test reads the field itself to show every character arrived. + func testOverBudgetTypeWithoutResolvableElementTypesApplicationWide() throws { + app.launchArguments = [ + "--agent-device-text-entry-regression", + "--agent-device-text-entry-soft-keyboard", + ] + app.launch() + addTeardownBlock { [self] in + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } + XCTAssertTrue(app.waitForExistence(timeout: appExistenceTimeout)) + + let textField = app.textFields["agent-device-hardware-keyboard-input"] + XCTAssertTrue(textField.waitForExistence(timeout: appExistenceTimeout)) + let tapCommand = try runnerCommandFixture( + #"{"command":"tap","commandId":"tap-soft-keyboard-input","selectorKey":"id","selectorValue":"agent-device-hardware-keyboard-input"}"# + ) + let tapResponse = try executeOnMainPrepared(command: tapCommand, activeApp: app) + XCTAssertTrue(tapResponse.ok, String(describing: tapResponse.error)) + try skipUnlessSoftwareKeyboardIsVisible() + + let text = String( + repeating: "x", + count: SynthesizedDeliveryBudget.maxTextLength(delaySeconds: 0) + 1 + ) + let failureCountBefore = currentXCTestFailureCount() + let result = typeTextReliably( + app: app, + target: TextEntryTarget( + element: nil, + refreshPoint: nil, + prefersFocusedElement: true, + fromTapWitness: true + ), + text: text, + delaySeconds: 0, + repairMode: .append, + synthesizer: PrivateXCTestTextEntrySynthesizer() + ) + + XCTAssertFalse(didRecordXCTestFailure(since: failureCountBefore)) + XCTAssertNil(result.failure) + XCTAssertEqual(result.textEntryRoute, "xctest-application-fallback") + XCTAssertNil(result.verified) + XCTAssertEqual(textField.value as? String, text) + } + private struct UnavailableTextEntrySynthesizer: TextEntrySynthesizing { func enterText( app _: XCUIApplication, @@ -189,5 +242,14 @@ extension RunnerTests { "software keyboard is up: this simulator cannot exercise the hidden-keyboard responder path" ) } + + // The mirror precondition. A simulator with a hardware keyboard attached can keep the software + // keyboard down even for a field that has a real input view, which is an environment fact. + private func skipUnlessSoftwareKeyboardIsVisible() throws { + try XCTSkipIf( + !isKeyboardVisible(app: app), + "software keyboard is down: this simulator cannot exercise the keyboard-visible typing branch" + ) + } #endif } From 7baa118bdd98fbb2483bfd755054f31032314dfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 25 Sep 2026 08:01:18 +0200 Subject: [PATCH 8/8] fix(ios-runner): charge the warmup split the type plan really makes The review found `projectedSeconds` charging one synthesize call for an append-mode `type` while the route posts a warmup character and then the rest, so the estimate undercounted one call and the wait between them. The over-budget branch that asks is the one with no element, so its warmup wait is one poll: `waitForWarmupValue` has no expected value to wait for. That branch now charges 2 calls and the poll, which is one character less than a replacement of the same length. Also: the budget hint and the docs named one limit and said "the number" for a delayed retry, which reads as the larger one; both now name the limit for the delay in play and say the delay falls between characters. The pace's header comment claimed the acknowledge window is derived from it, which it is not. The synthesized-replacement helper builds its command with JSONSerialization, and the over-budget lane test polls the field under its own deadline instead of reading `value` once on a route that verifies nothing. --- .../RunnerSynthesizedTextEntry.h | 7 ++-- .../RunnerTests+SynthesizedTextEntry.swift | 29 +++++++++++------ .../RunnerTests+TextEntry.swift | 2 +- .../RunnerTests+TextTyping.swift | 6 +++- ...unnerTests+SynthesizedTextEntryTests.swift | 14 ++++++-- .../RunnerTests+TextEntryPolicyTests.swift | 32 +++++++++++++++++++ .../RunnerTests+TextTypingTests.swift | 24 ++++++++++---- website/docs/docs/commands.md | 2 +- 8 files changed, 92 insertions(+), 24 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedTextEntry.h b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedTextEntry.h index 020661e2b3..cecbe07ed8 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedTextEntry.h +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedTextEntry.h @@ -17,9 +17,10 @@ typedef NS_ENUM(NSInteger, RunnerSynthesizedTextEntryStatus) { @interface RunnerSynthesizedTextEntry : NSObject -// Characters per second the synthesized text-input records are typed at. The delivery budget and -// the app-acknowledge window a burst has to fit inside are both derived from it, so it is declared -// once, here, where the typing happens. +// Characters per second the synthesized text-input records are typed at. Declared here, where the +// typing happens, so the delivery budget that bounds a burst is charged the same pace the app sees. +// The edit-acknowledge window that pace is sized for is a separate assumption about the app +// (TextEntryTiming.synthesizedAcknowledgeWindowSeconds), not a value derived from this one. + (NSUInteger)typingSpeedCharactersPerSecond; // Synthesizes keyboard input for the current first responder without resolving an diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift index 554b96087b..1d1e911d71 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift @@ -109,20 +109,31 @@ extension RunnerTests { 1.0 / Double(RunnerSynthesizedTextEntry.typingSpeedCharactersPerSecond()) } - /// Seconds the plan spends posting, charged per `synthesizedReplacementSteps` step: each - /// synthesize call types its characters at the pace and pays its overhead, and a spaced plan - /// sleeps `delaySeconds` between two calls. - static func projectedSeconds(textLength: Int, delaySeconds: TimeInterval) -> TimeInterval { - let calls = synthesizedReplacementIsSpaced(characterCount: textLength, delaySeconds: delaySeconds) - ? textLength - : 1 + /// Seconds the plan spends posting: each synthesize call types its characters at the pace and + /// pays its overhead, a spaced plan sleeps `delaySeconds` between two calls, and a plan that + /// peels one character as a warmup (`typeWarmup`) pays one more call and the wait before the + /// rest is posted. That wait is one poll here because the caller that asks has no element to + /// read the warmup character back from, so `waitForWarmupValue` has no value to wait for. + static func projectedSeconds( + textLength: Int, + delaySeconds: TimeInterval, + typeWarmup: Bool = false + ) -> TimeInterval { + let spaced = synthesizedReplacementIsSpaced(characterCount: textLength, delaySeconds: delaySeconds) + let warmupSplit = typeWarmup && textLength > 1 && !spaced + let calls = spaced ? textLength : (warmupSplit ? 2 : 1) return Double(textLength) * characterInterval + Double(calls) * TextEntryTiming.synthesizeCallOverhead + Double(calls - 1) * delaySeconds + + (warmupSplit ? TextEntryTiming.pollInterval : 0) } - static func exceeds(textLength: Int, delaySeconds: TimeInterval) -> Bool { - projectedSeconds(textLength: textLength, delaySeconds: delaySeconds) + static func exceeds( + textLength: Int, + delaySeconds: TimeInterval, + typeWarmup: Bool = false + ) -> Bool { + projectedSeconds(textLength: textLength, delaySeconds: delaySeconds, typeWarmup: typeWarmup) > TextEntryTiming.synthesizedDeliveryCeiling } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift index ba76b29340..96f3a3c3e3 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift @@ -36,7 +36,7 @@ extension RunnerTests { let recoveryBudget = SynthesizedDeliveryBudget.maxTextLength( delaySeconds: Double(recoveryDelay) / 1000 ) - return "Fill at most \(SynthesizedDeliveryBudget.maxTextLength(delaySeconds: 0)) characters at a time without --delay-ms and append the rest with separate type commands, keeping each command inside that budget. --delay-ms lowers the budget, because every character then pays its own synthesize call and the delay: \(recoveryBudget) characters at --delay-ms \(recoveryDelay). A longer timeout does not help: this route is chosen when the accessibility channel is already degraded, and the pace is what makes the text long." + return "Fill at most \(SynthesizedDeliveryBudget.maxTextLength(delaySeconds: 0)) characters at a time without --delay-ms and append the rest with separate type commands, keeping each command inside that budget. --delay-ms lowers the budget, because each character then gets its own synthesize call and each gap between characters pays the delay: \(recoveryBudget) characters at --delay-ms \(recoveryDelay). A longer timeout does not help: this route is chosen when the accessibility channel is already degraded, and the pace is what makes the text long." } } } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift index ddd5e7a29e..79abf8ebf7 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift @@ -170,7 +170,11 @@ extension RunnerTests { // neither chunk would look long on its own. This branch's target has no element to type into, // so nothing can be read back afterwards: the value arrives unverified, as it does for this // route's older synthesizer-unavailable fallback. - if SynthesizedDeliveryBudget.exceeds(textLength: text.count, delaySeconds: delaySeconds) { + if SynthesizedDeliveryBudget.exceeds( + textLength: text.count, + delaySeconds: delaySeconds, + typeWarmup: repairMode != .none + ) { textEntryRoute = "xctest-application-fallback" NSLog( "AGENT_DEVICE_RUNNER_TEXT_ENTRY_ROUTE route=xctest-application-fallback " diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift index 83430eb52c..641ee31e18 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift @@ -58,8 +58,18 @@ extension RunnerTests { commandId: String ) throws -> Response { let frame = textField.frame - let command = try runnerCommandFixture( - #"{"command":"type","commandId":"\#(commandId)","text":"\#(text)","textEntryMode":"replace","x":\#(frame.midX),"y":\#(frame.midY)}"# + // Assembled with JSONSerialization so a text carrying a quote or a backslash stays one command + // rather than invalid JSON. + let command = try JSONDecoder().decode( + Command.self, + from: JSONSerialization.data(withJSONObject: [ + "command": "type", + "commandId": commandId, + "text": text, + "textEntryMode": "replace", + "x": frame.midX, + "y": frame.midY, + ]) ) let failuresBeforeType = currentXCTestFailureCount() let response = executeTypeCommand(activeApp: app, command: command) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift index c277b8669b..a2fb1c1bf9 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift @@ -283,6 +283,38 @@ extension RunnerTests { XCTAssertLessThan(SynthesizedDeliveryBudget.maxTextLength(delaySeconds: 0.2), fits) } + // A `type` plan peels one character as a warmup and posts the rest afterwards, so the same text + // costs one synthesize call and one wait more than the single burst the replacement route posts. + // Without this the estimate charged a burst, which is what made the over-budget branch of the + // keyboard-visible route unreachable: 215 characters looked like 1 + 214, each inside the budget. + func testTypeWarmupSplitCostsOneMoreCallThanASingleBurst() { + let length = 20 + let withWarmup = SynthesizedDeliveryBudget.projectedSeconds( + textLength: length, + delaySeconds: 0, + typeWarmup: true + ) + XCTAssertGreaterThan( + withWarmup, + SynthesizedDeliveryBudget.projectedSeconds(textLength: length, delaySeconds: 0) + ) + XCTAssertEqual( + withWarmup - SynthesizedDeliveryBudget.projectedSeconds(textLength: length, delaySeconds: 0), + TextEntryTiming.synthesizeCallOverhead + TextEntryTiming.pollInterval, + accuracy: 1e-9 + ) + // The split mirrors the plan: a spaced `type` already posts per character, and a single + // character has no rest to post. + XCTAssertEqual( + SynthesizedDeliveryBudget.projectedSeconds(textLength: length, delaySeconds: 0.2, typeWarmup: true), + SynthesizedDeliveryBudget.projectedSeconds(textLength: length, delaySeconds: 0.2) + ) + XCTAssertEqual( + SynthesizedDeliveryBudget.projectedSeconds(textLength: 1, delaySeconds: 0, typeWarmup: true), + SynthesizedDeliveryBudget.projectedSeconds(textLength: 1, delaySeconds: 0) + ) + } + func testSynthesizedBudgetExceededCarriesItsOwnCodeAndRecovery() { XCTAssertEqual( TextEntryFailure.synthesisBudgetExceeded.rawValue, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift index 7c8c784ca6..737a53c1e1 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift @@ -149,11 +149,11 @@ extension RunnerTests { } // Text past the delivery budget cannot be paced into a field the runner cannot resolve, so it goes - // through application-wide typing. The budget is charged the whole command, which is what the - // length below pins: an append peels its first character for warmup, so a per-dispatch charge - // would find both of its pieces inside the budget and pace all 215 characters. The target carries - // no element by construction, so nothing on that route can read the value back: the command reports - // it unverified and this test reads the field itself to show every character arrived. + // through application-wide typing. The budget is charged the whole command, warmup split included: + // an append peels its first character for warmup, so a per-dispatch charge would find both of its + // pieces inside the budget and pace all these characters. The target carries no element by + // construction, so nothing on that route can read the value back: the command reports it + // unverified and this test reads the field itself to show every character arrived. func testOverBudgetTypeWithoutResolvableElementTypesApplicationWide() throws { app.launchArguments = [ "--agent-device-text-entry-regression", @@ -180,6 +180,8 @@ extension RunnerTests { count: SynthesizedDeliveryBudget.maxTextLength(delaySeconds: 0) + 1 ) let failureCountBefore = currentXCTestFailureCount() + // The target the `type` command builds when it cannot resolve an input but the keyboard is up: + // no element, no refresh point, focused-element preference. let result = typeTextReliably( app: app, target: TextEntryTarget( @@ -197,8 +199,16 @@ extension RunnerTests { XCTAssertFalse(didRecordXCTestFailure(since: failureCountBefore)) XCTAssertNil(result.failure) XCTAssertEqual(result.textEntryRoute, "xctest-application-fallback") - XCTAssertNil(result.verified) - XCTAssertEqual(textField.value as? String, text) + // This branch has no element to read, so the value arrives unverified and the command waited for + // nothing. The field is polled here, under its own deadline. + let valueDeadline = Date().addingTimeInterval(appExistenceTimeout) + var observed: String? + while Date() < valueDeadline { + observed = textField.value as? String + if observed == text { break } + Thread.sleep(forTimeInterval: 0.25) + } + XCTAssertEqual(observed, text) } private struct UnavailableTextEntrySynthesizer: TextEntrySynthesizing { diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index e0b921fce2..2a1b443fc5 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -488,7 +488,7 @@ agent-device gesture transform 200 420 80 -40 2 35 700 # combined pan, zoom, and If `type` reports `TEXT_INPUT_NOT_FOCUSED`, focus a visible text input and retry; when accessibility does not expose the input, use a coordinate focus command before typing. On iOS, if `type "\n"` reports `TEXT_INPUT_SYNTHESIS_UNAVAILABLE` after tapping a field while the software keyboard is hidden, show the software keyboard, then retry. The runner reports this error instead of risking input through an unreliable text-entry path. On iOS, if `fill` reports `TEXT_INPUT_COMMIT_NOT_OBSERVED`, the runner could not confirm the typed text reached the field — either it did not land before the runner's deadline, or the expected final text is identical to the field's placeholder. In the latter case, accessibility cannot distinguish committed text from an empty field rendering that placeholder, even if the field held content before dispatch. The field may hold none, part, or all of the text: run `snapshot -i` and inspect it. If it already matches, continue; otherwise retry with the full text quoted and `fill --delay-ms 80`, which replaces the whole value. Do not use `type`, which appends to whatever committed. This covers the coordinate-driven `fill` route taken when the accessibility channel is under load, which observes the field after synthesizing; it is not a guarantee that every text-entry route verifies its result. -On iOS, if `fill` reports `TEXT_INPUT_SYNTHESIS_BUDGET_EXCEEDED`, the text is longer than that coordinate-driven route can type inside one runner command at its bounded pace, and nothing was typed. Fill at most the number of characters the hint names and append the rest with separate `type` commands. `--delay-ms` lowers that budget, because every character then pays its own synthesize call and the delay; a longer timeout does not help. +On iOS, if `fill` reports `TEXT_INPUT_SYNTHESIS_BUDGET_EXCEEDED`, the text is longer than that coordinate-driven route can type inside one runner command at its bounded pace, and nothing was typed. Fill at most the character limit the hint names for your `--delay-ms`, and append the rest with separate `type` commands: the hint gives one limit without `--delay-ms` and a lower one for the delay it recommends. `--delay-ms` lowers the budget because each character then gets its own synthesize call and each gap between characters pays that delay; a longer timeout does not help. Use plain `fill` or `type` first for ordinary login and form fields. Use `--delay-ms` on `type` or `fill` only when a debounced search field or search-as-you-type input actually misses characters, or when the app must receive incremental updates. Delayed typing intentionally prefers paced character entry over clipboard-style fallbacks so the target field receives each incremental update. On Android, `fill` also verifies text and treats IME-owned capture as a terminal failure instead of retrying against the wrong field.