perf: reduce iOS input readiness and PNG encoding latency

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift

@@ -1,10 +1,5 @@
 import XCTest
 
-// Text-entry target acquisition: choosing the element a `type`/`fill` will address and getting
-// focus onto it — the one-shot tap witness, the post-tap stabilization, and the
-// `focusTextInputForTextEntry` entry point the command layer calls. Whether that target is ready
-// is RunnerTests+TextEntryReadiness.swift's question, and this file asks it rather than answering
-// it.
 extension RunnerTests {
   /// Books the tap as the address for the next bare `type`. `element` is a text input the caller
   /// classified BEFORE dispatching, or nil when the tap hit nothing worth typing into; that is the whole
@@ -71,7 +66,6 @@ extension RunnerTests {
 #if os(tvOS)
     return TextEntryStabilization(element: target, focusConfirmed: true)
 #else
-    let latest = target
     let keyboardVisibleAtEntry = keyboardVisibleBeforeTap ?? isKeyboardVisible(app: app)
     let deadline = Date().addingTimeInterval(TextEntryTiming.focusTimeout)
     while Date() < deadline {
@@ -81,11 +75,16 @@ extension RunnerTests {
       // focusedTextInput is intentionally nil on iOS; treat the keyboard transitioning to
       // visible after our tap as the focus-moved signal. Don't fast-path when it was already up.
       if keyboardBecameVisible(app: app, wasVisibleAtEntry: keyboardVisibleAtEntry) {
-        return TextEntryStabilization(element: latest, focusConfirmed: true)
+        return TextEntryStabilization(element: target, focusConfirmed: true)
       }
+#if os(iOS)
+      if let target, keyboardFocusConfirmed(app: app, element: target) {
+        return TextEntryStabilization(element: target, focusConfirmed: true)
+      }
+#endif
       sleepFor(TextEntryTiming.pollInterval)
     }
-    return TextEntryStabilization(element: latest, focusConfirmed: false)
+    return TextEntryStabilization(element: target, focusConfirmed: false)
 #endif
   }
 
@@ -151,10 +150,9 @@ extension RunnerTests {
       refreshPoint: requestedPoint,
       prefersFocusedElement: false
     )
-    let concreteTargetReady = keyboardVisibleBeforeTap && readyTarget.element != nil
-    let element = stabilized.focusConfirmed || concreteTargetReady
-      ? (stabilized.element ?? target)
-      : (waitForTextEntryReadiness(app: app, target: readyTarget) ?? stabilized.element ?? target)
+    let element = stabilized.focusConfirmed
+      ? readyTarget.element
+      : (waitForTextEntryReadiness(app: app, target: readyTarget) ?? readyTarget.element)
     return TextEntryTarget(
       element: element,
       refreshPoint: textEntryRefreshPoint(for: element, fallbackTo: requestedPoint),
@@ -186,10 +184,4 @@ extension RunnerTests {
     }
     return CGPoint(x: frame.midX, y: frame.midY)
   }
-
-  /// A focus-moved signal for iOS text entry, where `focusedTextInput` is intentionally nil.
-  /// The software keyboard TRANSITIONING from hidden (at entry) to visible means the field we
-  /// just tapped gained first-responder. If the keyboard was ALREADY up (e.g. back-to-back
-  /// fills into different fields), its visibility is not evidence focus moved to the new field,
-  /// so callers must keep waiting rather than typing into the previously-focused field.
 }

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift

@@ -1,9 +1,5 @@
 import XCTest
 
-// Text-entry readiness: whether the target is actually able to receive text yet. The waits, the
-// keyboard signals they read, and the focus corroboration that ends the hardware-keyboard window.
-// RunnerTests+TextEntryFocus.swift is the only caller of the two non-private entries here; nothing
-// in this file reaches back into it.
 extension RunnerTests {
   func focusedTextInput(app: XCUIApplication) -> XCUIElement? {
 #if os(iOS)
@@ -35,27 +31,21 @@ extension RunnerTests {
     timeout: TimeInterval = TextEntryTiming.readinessTimeout
   ) -> XCUIElement? {
 #if os(iOS)
-    var latest = resolveTextEntryElement(app: app, target: target)
+    let candidate = resolveTextEntryElement(app: app, target: target)
     let keyboardVisibleAtEntry = isKeyboardVisible(app: app)
     let deadline = Date().addingTimeInterval(timeout)
     var hardwareKeyboardFallback = Date().addingTimeInterval(
       min(TextEntryTiming.hardwareKeyboardFallbackTimeout, timeout)
     )
     var sawSoftwareKeyboard = false
     while Date() < deadline {
-      if let focused = focusedTextInput(app: app) {
-        latest = focused
-        if isKeyboardVisible(app: app) {
-          return focused
-        }
-      }
       // Fast-path on a keyboard hidden->visible transition: our tapped field gained focus, so
       // return immediately instead of burning the full readinessTimeout (warmup-first-char echo
       // + post-type verify/repair remain as drop safety nets). When the keyboard was ALREADY up
       // (back-to-back fills), this isn't a focus signal — fall through to the settle/timeout so
       // text isn't sent to the previously-focused field.
       if keyboardBecameVisible(app: app, wasVisibleAtEntry: keyboardVisibleAtEntry) {
-        return latest
+        return candidate
       }
       sawSoftwareKeyboard = sawSoftwareKeyboard || keyboardElementExists(app: app)
       // A responder that takes no software keyboard (hardware keyboard connected, or a custom
@@ -64,7 +54,7 @@ extension RunnerTests {
       // simulator state (#1874): on a loaded host the keyboard is merely late, and returning here
       // handed the caller an element that had not taken focus yet. Ask the target itself instead,
       // and re-arm rather than re-asking every poll — the query is cheap, not free.
-      if !sawSoftwareKeyboard, Date() >= hardwareKeyboardFallback, let candidate = latest {
+      if !sawSoftwareKeyboard, Date() >= hardwareKeyboardFallback, let candidate {
         if keyboardFocusConfirmed(app: app, element: candidate) {
           return candidate
         }
@@ -74,7 +64,7 @@ extension RunnerTests {
       }
       sleepFor(TextEntryTiming.pollInterval)
     }
-    return focusedTextInput(app: app) ?? latest
+    return candidate
 #else
     return resolveTextEntryElement(app: app, target: target)
 #endif
@@ -96,7 +86,7 @@ extension RunnerTests {
   /// field is then, which is the moment the read it replaced ran at.
   func waitForTextEntryReadinessAfterTap(app: XCUIApplication, element: XCUIElement) {
 #if os(iOS)
-    if waitForFocusedTextInput(app: app, timeout: TextEntryTiming.readinessTimeout) != nil {
+    if waitForTextEntryFocus(app: app, element: element, timeout: TextEntryTiming.readinessTimeout) {
       return
     }
     guard let settled = safely("TEXT_ENTRY_READINESS_FRAME", { try? element.snapshot() }) else {
@@ -105,20 +95,20 @@ extension RunnerTests {
     let frame = settled.frame
     if !frame.isEmpty {
       _ = tapAt(app: app, x: frame.midX, y: frame.midY)
-      _ = waitForFocusedTextInput(app: app, timeout: TextEntryTiming.readinessTimeout)
+      _ = waitForTextEntryFocus(app: app, element: element, timeout: TextEntryTiming.readinessTimeout)
     }
 #endif
   }
 
-  private func waitForFocusedTextInput(app: XCUIApplication, timeout: TimeInterval) -> XCUIElement? {
+  private func waitForTextEntryFocus(app: XCUIApplication, element: XCUIElement, timeout: TimeInterval) -> Bool {
     let deadline = Date().addingTimeInterval(timeout)
     while Date() < deadline {
-      if let focused = focusedTextInput(app: app) {
-        return focused
+      if keyboardFocusConfirmed(app: app, element: element) {
+        return true
       }
       sleepFor(TextEntryTiming.pollInterval)
     }
-    return focusedTextInput(app: app)
+    return keyboardFocusConfirmed(app: app, element: element)
   }
 
   func keyboardBecameVisible(app: XCUIApplication, wasVisibleAtEntry: Bool) -> Bool {
@@ -142,18 +132,17 @@ extension RunnerTests {
       // An element that no longer resolves reads as identifier "" and frame `.zero`, which would
       // match any focused element that also reports an empty frame. Require a real frame first,
       // so a dead handle cannot corroborate anything.
-      let frame = element.frame
-      guard !frame.isEmpty else {
+      guard element.exists, let expected = try? element.snapshot(), !expected.frame.isEmpty else {
         return false
       }
       let focused = app
         .descendants(matching: .any)
         .matching(NSPredicate(format: "hasKeyboardFocus == 1"))
         .firstMatch
-      guard focused.exists else {
+      guard focused.exists, let observed = try? focused.snapshot() else {
         return false
       }
-      return focused.identifier == element.identifier && focused.frame == frame
+      return observed.identifier == expected.identifier && observed.frame == expected.frame
     }
 #else
     return false

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryLatencyTests.swift

@@ -0,0 +1,25 @@
+import XCTest
+
+extension RunnerTests {
+#if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS)
+  @MainActor
+  func testConfirmedInputReadinessReturnsBeforeItsReadinessWindow() throws {
+    app.launchArguments = ["--agent-device-text-entry-regression"]
+    app.launch()
+    defer { invalidateCachedTarget(reason: "unit_test_cleanup"); app.terminate() }
+    XCTAssertTrue(app.waitForExistence(timeout: appExistenceTimeout))
+    let field = app.textFields["agent-device-hardware-keyboard-input"]
+    XCTAssertTrue(field.waitForExistence(timeout: appExistenceTimeout))
+    field.tap()
+    XCTAssertTrue(keyboardFocusConfirmed(app: app, element: field))
+    let failures = currentXCTestFailureCount()
+    let startedAt = Date()
+    waitForTextEntryReadinessAfterTap(app: app, element: field)
+    let elapsed = Date().timeIntervalSince(startedAt)
+    XCTAssertFalse(didRecordXCTestFailure(since: failures))
+    XCTAssertTrue(keyboardFocusConfirmed(app: app, element: field))
+    XCTAssertLessThan(elapsed, TextEntryTiming.readinessTimeout)
+  }
+
+#endif
+}

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift

@@ -234,6 +234,48 @@ extension RunnerTests {
     XCTAssertEqual(editableTextValue(for: nextScreenField, treatingPlaceholderAsEmpty: true), "")
   }
 
+  @MainActor
+  func testFillFailsWhenAppRemovesMaskedInputBeforeTextIsDelivered() throws {
+    let textField = try launchRemovableInputFixture(
+      "--agent-device-text-entry-masked-value",
+      "--agent-device-text-entry-disappear-after-input"
+    )
+
+    let failureCountBefore = currentXCTestFailureCount()
+    let response = executeTypeCommand(
+      activeApp: app,
+      command: try fillCommandFixture(commandId: "fill-disappearing-masked-input", text: "123456", at: textField)
+    )
+
+    XCTAssertFalse(didRecordXCTestFailure(since: failureCountBefore))
+    XCTAssertFalse(response.ok)
+    XCTAssertEqual(response.error?.code, "TEXT_INPUT_NOT_FOCUSED")
+    XCTAssertFalse(textField.exists)
+    XCTAssertEqual(app.textFields.count, 0)
+  }
+
+  @MainActor
+  func testFillFailsWhenAppReplacesMaskedInputBeforeTextIsDelivered() throws {
+    let textField = try launchRemovableInputFixture(
+      "--agent-device-text-entry-masked-value",
+      "--agent-device-text-entry-replace-after-input"
+    )
+
+    let failureCountBefore = currentXCTestFailureCount()
+    let response = executeTypeCommand(
+      activeApp: app,
+      command: try fillCommandFixture(commandId: "fill-replaced-masked-input", text: "123456", at: textField)
+    )
+
+    XCTAssertFalse(didRecordXCTestFailure(since: failureCountBefore))
+    XCTAssertFalse(response.ok)
+    XCTAssertEqual(response.error?.code, "TEXT_INPUT_NOT_FOCUSED")
+    XCTAssertFalse(textField.exists)
+    let nextScreenField = app.textFields["agent-device-auto-submit-next-screen-input"]
+    XCTAssertTrue(nextScreenField.exists)
+    XCTAssertEqual(editableTextValue(for: nextScreenField, treatingPlaceholderAsEmpty: true), "")
+  }
+
   // Neither input has an identifier, so the successor carries the same identity as the code field.
   // The runner cannot tell them apart and must not clear and retype the successor: it reports the
   // mismatch it read instead of repairing.