FuJacob · Joaov41 · May 22, 2026 · May 22, 2026 · May 22, 2026 · May 22, 2026
diff --git a/README.md b/README.md
@@ -76,6 +76,17 @@ Everything runs on-device. No hosted API, no cloud round-trip.
 - **Visual context** -- Screenshot OCR gives the model awareness of what's on screen
 - **Low latency** -- Optimized for fast response on Apple Silicon
 
+## Compared To The Original Upstream Code
+
+This working tree currently differs from the original upstream codebase in a few behaviorally important ways:
+
+- **Richer field context from Accessibility** -- suggestions can use focused-field metadata and nearby AX text, not just the text before the caret.
+- **Prompting that understands both sides of the caret** -- request building now carries the text after the caret as a constraint, which improves completions in the middle of existing text.
+- **Stronger output cleanup** -- normalization is more aggressive about dropping generic filler, assistant-style replies, copied OCR/UI fragments, repeated draft text, and bad whole-word echoes.
+- **Faster partial-word completion** -- common word tails can be completed locally through macOS spelling/completion APIs before falling back to model generation.
+- **Better non-keyboard update handling** -- autocomplete can reschedule when the focused text changes through Accessibility without a normal key event, which helps automation and some host-app/input-method paths.
+- **More careful visual-context behavior** -- screenshot OCR remains a best-effort prompt signal; Screen Recording improves visual context but does not block plain text-only autocomplete.
+
 ## Engines
 
 **Apple Intelligence [EXPERIMENTAL]**: uses Apple's on-device `FoundationModels` runtime on macOS 26 or later, no download required. Currently does not perform as well as the Open Source models. We're actively working on improving it.
@@ -93,7 +104,7 @@ You can also drop your own `.gguf` files into tabby's models folder and refresh
 
 1. Download the latest `tabby.dmg` from GitHub Releases.
 2. Drag `tabby.app` into `Applications` and launch it.
-3. Grant **Accessibility**, **Input Monitoring**, and **Screen Recording** when prompted.
+3. Grant **Accessibility** and **Input Monitoring** when prompted. Grant **Screen Recording** if you want screenshot-derived visual context.
 4. Pick an engine. Apple Intelligence if available, otherwise Open Source plus a model.
 5. Start typing in any supported editable field.
 
@@ -103,7 +114,7 @@ If macOS blocks first launch, right-click `tabby.app` → `Open`, or allow it in
 
 - **Accessibility**: read the focused text field's value and caret position.
 - **Input Monitoring**: detect global `Tab` presses for acceptance.
-- **Screen Recording**: capture a screenshot around the focused field for visual context (OCR).
+- **Screen Recording**: optional, used to capture a screenshot around the focused field for visual context (OCR).
 
 **Requires macOS 15.0 or later.** Apple Intelligence suggestions require macOS 26 or later; on earlier supported systems, use the Open Source engine.
 

diff --git a/tabby.xcodeproj/project.pbxproj b/tabby.xcodeproj/project.pbxproj
@@ -23,6 +23,7 @@
 		C10000062F91000100BBB006 /* ModelAndPresentationValueTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = C10000162F91000100BBB016 /* ModelAndPresentationValueTests.swift */; };
 		C10000072F91000100BBB007 /* SuggestionStateHelperTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = C10000172F91000100BBB017 /* SuggestionStateHelperTests.swift */; };
 		D10000012F92000100CCC001 /* TerminalAppDetectorTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D10000112F92000100CCC011 /* TerminalAppDetectorTests.swift */; };
+		E20000012FA0000100DDD001 /* SuggestionInserterTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = E20000112FA0000100DDD011 /* SuggestionInserterTests.swift */; };
 /* End PBXBuildFile section */
 
 /* Begin PBXContainerItemProxy section */
@@ -50,6 +51,7 @@
 		C10000162F91000100BBB016 /* ModelAndPresentationValueTests.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = ModelAndPresentationValueTests.swift; sourceTree = "<group>"; };
 		C10000172F91000100BBB017 /* SuggestionStateHelperTests.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = SuggestionStateHelperTests.swift; sourceTree = "<group>"; };
 		D10000112F92000100CCC011 /* TerminalAppDetectorTests.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = TerminalAppDetectorTests.swift; sourceTree = "<group>"; };
+		E20000112FA0000100DDD011 /* SuggestionInserterTests.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = SuggestionInserterTests.swift; sourceTree = "<group>"; };
 		F29623C5C0A67B992D383A3C /* LlamaPromptRendererTests.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = LlamaPromptRendererTests.swift; sourceTree = "<group>"; };
 		F9D35DB9E86506B9FAE1CFE9 /* ModelFileValidatorTests.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = ModelFileValidatorTests.swift; sourceTree = "<group>"; };
 /* End PBXFileReference section */
@@ -117,6 +119,7 @@
 				F9D35DB9E86506B9FAE1CFE9 /* ModelFileValidatorTests.swift */,
 				BAAEE25772008D75883F2655 /* DownloadFileRescuerTests.swift */,
 				D10000112F92000100CCC011 /* TerminalAppDetectorTests.swift */,
+				E20000112FA0000100DDD011 /* SuggestionInserterTests.swift */,
 			);
 			path = tabbyTests;
 			sourceTree = "<group>";
@@ -248,6 +251,7 @@
 				AF0F4C853CCA8B86BB5E28CD /* ModelFileValidatorTests.swift in Sources */,
 				B5788B37B93AFEC10EFD3108 /* DownloadFileRescuerTests.swift in Sources */,
 				D10000012F92000100CCC001 /* TerminalAppDetectorTests.swift in Sources */,
+				E20000012FA0000100DDD001 /* SuggestionInserterTests.swift in Sources */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 		};

diff --git a/tabby/App/Coordinators/SuggestionCoordinator+Acceptance.swift b/tabby/App/Coordinators/SuggestionCoordinator+Acceptance.swift
@@ -41,7 +41,7 @@ extension SuggestionCoordinator {
             return passTabThrough(reason: reason)
         }
 
-        guard suggestionInserter.insert(acceptedChunk) else {
+        guard commitAcceptedText(acceptedChunk, for: sessionForAcceptance) else {
             let message = suggestionInserter.lastErrorMessage ?? "Suggestion insertion failed."
             cancelPredictionWork()
             clearSuggestion(clearDiagnostics: true)
@@ -116,6 +116,16 @@ extension SuggestionCoordinator {
         }
     }
 
+    private func commitAcceptedText(_ acceptedChunk: String, for session: ActiveSuggestionSession) -> Bool {
+        switch session.acceptanceEdit {
+        case .insert:
+            return suggestionInserter.insert(acceptedChunk)
+
+        case let .replacePreviousCharacters(count):
+            return suggestionInserter.replacePreviousCharacters(count: count, with: acceptedChunk)
+        }
+    }
+
     /// Returns control of `Tab` to the host app and clears stale suggestion UI.
     func passTabThrough(reason: String) -> Bool {
         let generation = latestGenerationNumber

diff --git a/tabby/App/Coordinators/SuggestionCoordinator+Input.swift b/tabby/App/Coordinators/SuggestionCoordinator+Input.swift
@@ -13,7 +13,6 @@ extension SuggestionCoordinator {
             globallyEnabled: settingsSnapshot.isGloballyEnabled,
             disabledAppBundleIdentifiers: settingsSnapshot.disabledAppBundleIdentifiers,
             inputMonitoringGranted: permissionManager.inputMonitoringGranted,
-            screenRecordingGranted: permissionManager.screenRecordingGranted,
             focusSnapshot: focusModel.snapshot
         ) {
             handleSupportedSnapshot(focusModel.snapshot)
@@ -34,7 +33,6 @@ extension SuggestionCoordinator {
             globallyEnabled: settingsSnapshot.isGloballyEnabled,
             disabledAppBundleIdentifiers: settingsSnapshot.disabledAppBundleIdentifiers,
             inputMonitoringGranted: permissionManager.inputMonitoringGranted,
-            screenRecordingGranted: permissionManager.screenRecordingGranted,
             focusSnapshot: snapshot
         ) {
             disablePredictionsPreservingVisualContext(reason: disabledReason)
@@ -67,19 +65,63 @@ extension SuggestionCoordinator {
             clearSuggestion(clearDiagnostics: true)
             hideOverlay(reason: "Overlay hidden because the focused field changed.")
             state = .idle
+            lastSnapshotDrivenPredictionSignature = nil
         }
 
+        schedulePredictionIfFocusedTextChangedWithoutKeyEvent(focusedContext)
+
         if overlayState.isVisible {
             hideOverlay(reason: "Overlay hidden because no ready suggestion remains.")
         }
     }
 
+    /// Schedules a generation when Accessibility reports text changed but no global key event reached
+    /// `InputMonitor`. This covers automation paths like Computer Use and some host-app/input-method
+    /// combinations that mutate the text value without a normal `keyDown` event.
+    func schedulePredictionIfFocusedTextChangedWithoutKeyEvent(_ focusedContext: FocusedInputSnapshot) {
+        guard !isRefreshingFocusForInputEvent else {
+            _ = interactionState.materializeContext(from: focusedContext)
+            return
+        }
+
+        guard interactionState.activeSession == nil else {
+            return
+        }
+
+        guard let previousContext = interactionState.currentContext else {
+            _ = interactionState.materializeContext(from: focusedContext)
+            return
+        }
+
+        guard previousContext.processIdentifier == focusedContext.processIdentifier else {
+            _ = interactionState.materializeContext(from: focusedContext)
+            lastSnapshotDrivenPredictionSignature = nil
+            return
+        }
+
+        let signature = focusedContext.contentSignature
+        guard signature != previousContext.contentSignature else {
+            return
+        }
+
+        _ = interactionState.materializeContext(from: focusedContext)
+
+        guard focusedContext.selection.length == 0,
+              SuggestionRequestFactory.shouldGenerateSuggestion(for: focusedContext.precedingText),
+              lastSnapshotDrivenPredictionSignature != signature
+        else {
+            return
+        }
+
+        lastSnapshotDrivenPredictionSignature = signature
+        schedulePrediction()
+    }
+
     func handleInputEvent(_ event: CapturedInputEvent) -> Bool {
         if let disabledReason = SuggestionAvailabilityEvaluator.disabledReason(
             globallyEnabled: settingsSnapshot.isGloballyEnabled,
             disabledAppBundleIdentifiers: settingsSnapshot.disabledAppBundleIdentifiers,
             inputMonitoringGranted: permissionManager.inputMonitoringGranted,
-            screenRecordingGranted: permissionManager.screenRecordingGranted,
             focusSnapshot: focusModel.snapshot
         ) {
             disablePredictions(reason: disabledReason)
@@ -104,9 +146,12 @@ extension SuggestionCoordinator {
         }
 
         if event.shouldSchedulePrediction {
+            lastSnapshotDrivenPredictionSignature = nil
             // Capture AX state immediately at keystroke time so the debounce window
             // works with the freshest possible snapshot, not whenever the poll timer last fired.
+            isRefreshingFocusForInputEvent = true
             focusModel.refreshNow()
+            isRefreshingFocusForInputEvent = false
             schedulePrediction()
         }
 
@@ -137,7 +182,10 @@ extension SuggestionCoordinator {
                 clearDiagnostics: false
             )
             if event.shouldSchedulePrediction {
+                lastSnapshotDrivenPredictionSignature = nil
+                isRefreshingFocusForInputEvent = true
                 focusModel.refreshNow()
+                isRefreshingFocusForInputEvent = false
                 schedulePrediction()
             }
             return false
@@ -148,7 +196,10 @@ extension SuggestionCoordinator {
                 clearDiagnostics: false
             )
             if event.shouldSchedulePrediction {
+                lastSnapshotDrivenPredictionSignature = nil
+                isRefreshingFocusForInputEvent = true
                 focusModel.refreshNow()
+                isRefreshingFocusForInputEvent = false
                 schedulePrediction()
             }
             return false

diff --git a/tabby/App/Coordinators/SuggestionCoordinator+Lifecycle.swift b/tabby/App/Coordinators/SuggestionCoordinator+Lifecycle.swift
@@ -15,6 +15,7 @@ extension SuggestionCoordinator {
     func stop() {
         cancelPredictionWork()
         resetCachedGenerationContext()
+        lastSnapshotDrivenPredictionSignature = nil
         visualContextCoordinator.cancel(resetState: true)
         hideOverlay(reason: "Overlay hidden because Tabby stopped observing suggestions.")
         inputMonitor.onEvent = nil
@@ -29,6 +30,7 @@ extension SuggestionCoordinator {
     func prepareForRuntimeModelSwitch() {
         cancelPredictionWork()
         resetCachedGenerationContext()
+        lastSnapshotDrivenPredictionSignature = nil
         interactionState.resetAll()
         visualContextCoordinator.cancel(resetState: true)
         clearSuggestion(clearDiagnostics: true)
@@ -50,6 +52,7 @@ extension SuggestionCoordinator {
         settingsSnapshot = snapshot
         cancelPredictionWork()
         resetCachedGenerationContext()
+        lastSnapshotDrivenPredictionSignature = nil
         clearSuggestion(clearDiagnostics: true)
         hideOverlay(reason: "Overlay hidden because autocomplete settings changed.")
         state = .idle
@@ -72,7 +75,6 @@ extension SuggestionCoordinator {
             globallyEnabled: settingsSnapshot.isGloballyEnabled,
             disabledAppBundleIdentifiers: settingsSnapshot.disabledAppBundleIdentifiers,
             inputMonitoringGranted: permissionManager.inputMonitoringGranted,
-            screenRecordingGranted: permissionManager.screenRecordingGranted,
             focusSnapshot: focusModel.snapshot
         ) {
             schedulePrediction()

diff --git a/tabby/App/Coordinators/SuggestionCoordinator+Prediction.swift b/tabby/App/Coordinators/SuggestionCoordinator+Prediction.swift
@@ -11,7 +11,6 @@ extension SuggestionCoordinator {
             globallyEnabled: settingsSnapshot.isGloballyEnabled,
             disabledAppBundleIdentifiers: settingsSnapshot.disabledAppBundleIdentifiers,
             inputMonitoringGranted: permissionManager.inputMonitoringGranted,
-            screenRecordingGranted: permissionManager.screenRecordingGranted,
             focusSnapshot: focusModel.snapshot
         ) {
             disablePredictions(reason: disabledReason)
@@ -50,7 +49,6 @@ extension SuggestionCoordinator {
             globallyEnabled: settingsSnapshot.isGloballyEnabled,
             disabledAppBundleIdentifiers: settingsSnapshot.disabledAppBundleIdentifiers,
             inputMonitoringGranted: permissionManager.inputMonitoringGranted,
-            screenRecordingGranted: permissionManager.screenRecordingGranted,
             focusSnapshot: snapshot
         ) {
             disablePredictions(reason: disabledReason)
@@ -70,6 +68,39 @@ extension SuggestionCoordinator {
         }
 
         let context = interactionState.materializeContext(from: rawContext)
+
+        if let localSpellCorrection = LocalSpellCorrectionProvider.suggestion(for: context) {
+            latestGenerationNumber = context.generation
+            latestPromptPreview = "Local spell correction for current text."
+            latestRawModelOutput = SuggestionDebugLogger.debugPreview(localSpellCorrection.rawText)
+            logStage(
+                "local-spell-correction",
+                workID: workID,
+                generation: context.generation,
+                message: "Using local spell correction before model generation.",
+                rawOutput: localSpellCorrection.rawText,
+                normalizedOutput: localSpellCorrection.text
+            )
+            await apply(result: localSpellCorrection, workID: workID)
+            return
+        }
+
+        if let localWordCompletion = LocalWordCompletionProvider.suggestion(for: context) {
+            latestGenerationNumber = context.generation
+            latestPromptPreview = "Local word completion for current token."
+            latestRawModelOutput = SuggestionDebugLogger.debugPreview(localWordCompletion.rawText)
+            logStage(
+                "local-word-completion",
+                workID: workID,
+                generation: context.generation,
+                message: "Using local word completion before model generation.",
+                rawOutput: localWordCompletion.rawText,
+                normalizedOutput: localWordCompletion.text
+            )
+            await apply(result: localWordCompletion, workID: workID)
+            return
+        }
+
         let visualContextSummary = visualContextCoordinator.excerpt(for: context)
         let clipboardContext = settingsSnapshot.isClipboardContextEnabled
             ? clipboardContextProvider.currentContext()
@@ -134,7 +165,6 @@ extension SuggestionCoordinator {
             globallyEnabled: settingsSnapshot.isGloballyEnabled,
             disabledAppBundleIdentifiers: settingsSnapshot.disabledAppBundleIdentifiers,
             inputMonitoringGranted: permissionManager.inputMonitoringGranted,
-            screenRecordingGranted: permissionManager.screenRecordingGranted,
             focusSnapshot: snapshot
         ) {
 
@@ -204,7 +234,8 @@ extension SuggestionCoordinator {
         let session = interactionState.startSession(
             fullText: result.text,
             liveContext: liveContext,
-            latency: result.latency
+            latency: result.latency,
+            acceptanceEdit: result.acceptanceEdit
         )
         applySessionDiagnostics(session, acceptanceAction: "Generated new suggestion.")
         state = .ready(text: session.remainingText, latency: session.latency)
@@ -246,7 +277,6 @@ extension SuggestionCoordinator {
             globallyEnabled: settingsSnapshot.isGloballyEnabled,
             disabledAppBundleIdentifiers: settingsSnapshot.disabledAppBundleIdentifiers,
             inputMonitoringGranted: permissionManager.inputMonitoringGranted,
-            screenRecordingGranted: permissionManager.screenRecordingGranted,
             focusSnapshot: focusModel.snapshot
         )
 
@@ -329,6 +359,7 @@ extension SuggestionCoordinator {
     func disablePredictions(reason: String) {
         cancelPredictionWork()
         resetCachedGenerationContext()
+        lastSnapshotDrivenPredictionSignature = nil
         visualContextCoordinator.cancel(resetState: true)
         interactionState.resetAll()
         clearSuggestion(clearDiagnostics: true)
@@ -346,6 +377,7 @@ extension SuggestionCoordinator {
     func disablePredictionsPreservingVisualContext(reason: String) {
         cancelPredictionWork()
         resetCachedGenerationContext()
+        lastSnapshotDrivenPredictionSignature = nil
         interactionState.resetAll()
         clearSuggestion(clearDiagnostics: true)
         hideOverlay(reason: reason)

diff --git a/tabby/App/Coordinators/SuggestionCoordinator.swift b/tabby/App/Coordinators/SuggestionCoordinator.swift
@@ -56,6 +56,11 @@ final class SuggestionCoordinator: ObservableObject {
     // Async work and active-session storage now live in dedicated collaborators below.
     var cancellables = Set<AnyCancellable>()
     var settingsSnapshot: SuggestionSettingsSnapshot
+    // AX-only text mutations, such as automation or some input methods, can change the focused text
+    // without producing a CGEvent tap callback. This signature keeps that fallback from repeatedly
+    // scheduling the same snapshot while polling catches up.
+    var lastSnapshotDrivenPredictionSignature: String?
+    var isRefreshingFocusForInputEvent = false
     // Synchronous input/focus callbacks cannot directly `await`, so resets are represented as a
     // barrier task that the next generation must cross before it can ask the runtime for output.
     var cacheResetSequence: UInt64 = 0