diff --git a/KeyType/Logic/Completion/CompletionController.swift b/KeyType/Logic/Completion/CompletionController.swift index 0a6cadc..8d142c0 100644 --- a/KeyType/Logic/Completion/CompletionController.swift +++ b/KeyType/Logic/Completion/CompletionController.swift @@ -1551,6 +1551,10 @@ final class CompletionController { static func correctionSuffixWindow(from suffix: String) -> String { let trimmed = suffix.prefix(80) + // A whitespace-only window (the usual case right after typing "word ") carries no join + // signal — probing P(bare space | prefix + replacement) is pure tokenization noise that + // vetoes valid corrections (ADR-120). + guard trimmed.contains(where: { !$0.isWhitespace }) else { return "" } if let firstNonWhitespace = trimmed.first(where: { !$0.isWhitespace }), !firstNonWhitespace.isLetter, !firstNonWhitespace.isNumber { diff --git a/KeyType/Logic/Correction/SystemSpellcheckCorrectionDetector.swift b/KeyType/Logic/Correction/SystemSpellcheckCorrectionDetector.swift index 6868a0b..a662f9a 100644 --- a/KeyType/Logic/Correction/SystemSpellcheckCorrectionDetector.swift +++ b/KeyType/Logic/Correction/SystemSpellcheckCorrectionDetector.swift @@ -27,6 +27,15 @@ struct SystemSpellcheckCorrectionDetector: CorrectionDetecting { case let (.success(closed), _): target = closed case let (.failure, .success(current)): + // An open fragment that can still extend into a dictionary word is typing in + // progress (shou → should), not a typo: leave it to the completion lane (ADR-113) + // and only correct dead-end fragments. Without this gate the recalibrated validator + // (ADR-120) happily proves "show" more probable than "shou" mid-word and badge-spams + // while the user types. Mirrors the completion typo guard's closed-word-only rule. + guard !SystemWordRecognizer().canCompleteWord( + prefix: current.original, + language: context.detectedLanguage + ) else { return nil } target = current case (.failure, .failure): return nil diff --git a/Packages/CompletionUI/Sources/CompletionUI/CompletionUI.swift b/Packages/CompletionUI/Sources/CompletionUI/CompletionUI.swift index 309c7d7..9d2c73a 100644 --- a/Packages/CompletionUI/Sources/CompletionUI/CompletionUI.swift +++ b/Packages/CompletionUI/Sources/CompletionUI/CompletionUI.swift @@ -404,23 +404,35 @@ public struct CorrectionBadgeView: View { } public var body: some View { + // Strike only the letters that are wrong and embolden only the letters that change + // (ADR-119): defin̶a̶tely → definitely. A pure insertion has an empty original core and + // nothing to strike; the arrow still carries the meaning. + let diff = CorrectionLetterDiff.between(original, replacement) + let badgeFont = Font(font as CTFont) HStack(spacing: 6) { - Text(original) - .font(Font(font as CTFont)) - .foregroundStyle(Color(nsColor: .systemRed)) - .strikethrough(true, color: Color(nsColor: .systemRed)) - .lineLimit(1) - .fixedSize() + ( + Text(diff.prefix) + + Text(diff.originalCore).strikethrough(true, color: Color(nsColor: .systemRed)) + + Text(diff.suffix) + ) + .font(badgeFont) + .foregroundStyle(Color(nsColor: .systemRed)) + .lineLimit(1) + .fixedSize() Text("→") - .font(Font(font as CTFont)) + .font(badgeFont) .foregroundStyle(Color(nsColor: .secondaryLabelColor)) .lineLimit(1) .fixedSize() - Text(replacement) - .font(Font(font as CTFont)) - .foregroundStyle(Color(nsColor: .systemGreen)) - .lineLimit(1) - .fixedSize() + ( + Text(diff.prefix) + + Text(diff.replacementCore).fontWeight(.semibold) + + Text(diff.suffix) + ) + .font(badgeFont) + .foregroundStyle(Color(nsColor: .systemGreen)) + .lineLimit(1) + .fixedSize() } .padding(.horizontal, 8) .padding(.vertical, 4) diff --git a/Packages/CompletionUI/Sources/CompletionUI/CorrectionLetterDiff.swift b/Packages/CompletionUI/Sources/CompletionUI/CorrectionLetterDiff.swift new file mode 100644 index 0000000..3647059 --- /dev/null +++ b/Packages/CompletionUI/Sources/CompletionUI/CorrectionLetterDiff.swift @@ -0,0 +1,40 @@ +import Foundation + +/// Letter-level split between a misspelled word and its correction, so the badge can strike +/// through only the letters that are actually wrong (ADR-119) instead of the whole word: +/// `definately → definitely` renders as defin~a~tely → defin**i**tely. +/// +/// The split is the longest common prefix, then the longest common suffix of the remainders +/// (never overlapping the prefix). Either core may be empty — a pure insertion (`wich → which`) +/// has an empty original core and nothing to strike; a pure deletion has an empty replacement +/// core and nothing to embolden. +public struct CorrectionLetterDiff: Equatable, Sendable { + public let prefix: String + public let originalCore: String + public let replacementCore: String + public let suffix: String + + public static func between(_ original: String, _ replacement: String) -> CorrectionLetterDiff { + let o = Array(original) + let r = Array(replacement) + + var prefixLength = 0 + while prefixLength < o.count, prefixLength < r.count, o[prefixLength] == r[prefixLength] { + prefixLength += 1 + } + + var suffixLength = 0 + while suffixLength < o.count - prefixLength, + suffixLength < r.count - prefixLength, + o[o.count - 1 - suffixLength] == r[r.count - 1 - suffixLength] { + suffixLength += 1 + } + + return CorrectionLetterDiff( + prefix: String(o.prefix(prefixLength)), + originalCore: String(o[prefixLength..<(o.count - suffixLength)]), + replacementCore: String(r[prefixLength..<(r.count - suffixLength)]), + suffix: String(o.suffix(suffixLength)) + ) + } +} diff --git a/Packages/CompletionUI/Tests/CompletionUITests/CompletionUITests.swift b/Packages/CompletionUI/Tests/CompletionUITests/CompletionUITests.swift index 3601c90..d05de86 100644 --- a/Packages/CompletionUI/Tests/CompletionUITests/CompletionUITests.swift +++ b/Packages/CompletionUI/Tests/CompletionUITests/CompletionUITests.swift @@ -73,6 +73,29 @@ final class CompletionUITests: XCTestCase { XCTAssertEqual(resolver.placement(for: context)?.mode, .inline) } + func testCorrectionLetterDiffSplitsSubstitution() { + let diff = CorrectionLetterDiff.between("definately", "definitely") + XCTAssertEqual(diff, CorrectionLetterDiff(prefix: "defin", originalCore: "a", replacementCore: "i", suffix: "tely")) + } + + func testCorrectionLetterDiffSplitsTransposition() { + let diff = CorrectionLetterDiff.between("recieve", "receive") + XCTAssertEqual(diff, CorrectionLetterDiff(prefix: "rec", originalCore: "ie", replacementCore: "ei", suffix: "ve")) + } + + func testCorrectionLetterDiffPureInsertionHasEmptyOriginalCore() { + let diff = CorrectionLetterDiff.between("wich", "which") + XCTAssertEqual(diff, CorrectionLetterDiff(prefix: "w", originalCore: "", replacementCore: "h", suffix: "ich")) + } + + func testCorrectionLetterDiffDisjointWordsFallBackToWholeWordCores() { + let diff = CorrectionLetterDiff.between("teh", "the") + XCTAssertEqual(diff.prefix, "t") + XCTAssertEqual(diff.originalCore, "eh") + XCTAssertEqual(diff.replacementCore, "he") + XCTAssertEqual(diff.suffix, "") + } + @MainActor func testCorrectionBadgeLayoutPlacesBesideCaretAndClampsToField() { let font = NSFont.systemFont(ofSize: 14) diff --git a/Packages/ConstrainedGeneration/Sources/ConstrainedGeneration/CorrectionValidationScorer.swift b/Packages/ConstrainedGeneration/Sources/ConstrainedGeneration/CorrectionValidationScorer.swift index 9c3d927..eb4f301 100644 --- a/Packages/ConstrainedGeneration/Sources/ConstrainedGeneration/CorrectionValidationScorer.swift +++ b/Packages/ConstrainedGeneration/Sources/ConstrainedGeneration/CorrectionValidationScorer.swift @@ -2,22 +2,48 @@ import AutocompleteCore import Foundation import ModelRuntime +/// All word-vs-word comparisons are in **joint (summed) log-probability** — the log of the actual +/// probability of the text (ADR-120). The previous per-token *mean* systematically favoured +/// multi-token strings (their trailing pieces are near-deterministic), to the point of ranking a +/// misspelling above its correction: measured on the live model, " definitely" (1 token) scored +/// mean −8.09 vs " definately" (2 tokens) mean −6.56, while the joint scores are −8.09 vs −13.12 +/// — the correction wins by 5 nats once the maths stops lying. public struct CorrectionValidationThresholds: Equatable, Sendable { - public var minimumMeanLogProbability: Double + /// A replacement must beat the *original* — what the user actually typed — by at least this + /// many nats of joint log-probability. 0 means "the model must not prefer the original at + /// all": deliberate spellings, slang, and proper nouns the model knows keep their veto, and + /// this single gate replaces both the old absolute floor (which asked the wrong question — + /// "is this word a likely continuation?" — and suppressed valid corrections like an adverb + /// after "I") and the old original-is-much-better check. Dictionary flagging already + /// guarantees the replacement is a real word. + public var minimumAdvantageOverOriginal: Double + /// Runner-up separation (joint nats) for ordinary candidates. public var minimumMargin: Double public var minimumSuffixMeanLogProbability: Double public var priorPredictionConfidence: Double + /// Relaxed runner-up separation for *clear-cut* spellcheck corrections (dictionary-flagged + /// word, edit distance ≤ `clearCutMaximumEditDistance`, model-top guess). Common typos + /// attract look-alike checker guesses that compress the margin (definately → definitely vs + /// defiantly measured at 5.97 joint nats, but mean-scale margins sat at 0.15); genuinely + /// ambiguous near-ties stay suppressed, and the advantage-over-original gate is never + /// relaxed. See ADR-119/120. + public var clearCutMargin: Double + public var clearCutMaximumEditDistance: Int public init( - minimumMeanLogProbability: Double = -6.0, - minimumMargin: Double = 0.20, + minimumAdvantageOverOriginal: Double = 0.0, + minimumMargin: Double = 2.0, minimumSuffixMeanLogProbability: Double = -7.0, - priorPredictionConfidence: Double = 0.97 + priorPredictionConfidence: Double = 0.97, + clearCutMargin: Double = 0.5, + clearCutMaximumEditDistance: Int = 2 ) { - self.minimumMeanLogProbability = minimumMeanLogProbability + self.minimumAdvantageOverOriginal = minimumAdvantageOverOriginal self.minimumMargin = minimumMargin self.minimumSuffixMeanLogProbability = minimumSuffixMeanLogProbability self.priorPredictionConfidence = priorPredictionConfidence + self.clearCutMargin = clearCutMargin + self.clearCutMaximumEditDistance = clearCutMaximumEditDistance } } @@ -58,22 +84,34 @@ public final class CorrectionValidationScorer { return [candidate] } - let prefixTokens = try runtime.tokenizer.tokenize(prefixBeforeWord) + // Score across the word boundary the way the model actually tokenizes it (ADR-120, the + // correction-lane analogue of ADR-017's caret-boundary sanitization): an anchor ending in + // whitespace forces the word to tokenize standalone, a split BPE models essentially never + // produce after a space token, which floors every candidate's mean log-probability. Trim + // the trailing whitespace off the anchor and carry it as the words' leading separator so + // "I " + "definitely" is scored as "I" + " definitely" (one space-prefixed token). + let boundary = Self.splitTrailingWhitespace(of: prefixBeforeWord) + let prefixTokens = try runtime.tokenizer.tokenize(boundary.head) let original = candidates.first?.original ?? "" - let originalScore = try await meanLogProbability( - of: original, + let originalScore = try await jointLogProbability( + of: boundary.separator + original, anchor: prefixTokens ) var scored: [(candidate: CorrectionCandidate, score: Double, suffixScore: Double?)] = [] for candidate in candidates { try Task.checkCancellation() - let score = try await meanLogProbability(of: candidate.replacement, anchor: prefixTokens) - let suffixScore = suffixWindow.isEmpty + let score = try await jointLogProbability( + of: boundary.separator + candidate.replacement, + anchor: prefixTokens + ) + // Whitespace-only windows carry no join signal (defensive twin of the controller's + // producer-side check — probing a bare space token is tokenization noise, ADR-120). + let suffixScore = suffixWindow.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty ? nil : try await meanLogProbability( of: suffixWindow, - anchor: prefixTokens + runtime.tokenizer.tokenize(candidate.replacement) + anchor: prefixTokens + runtime.tokenizer.tokenize(boundary.separator + candidate.replacement) ) scored.append((candidate, score, suffixScore)) } @@ -83,18 +121,28 @@ public final class CorrectionValidationScorer { return scored.compactMap { entry in let margin = runnerUp.map { entry.score - $0 } ?? .infinity - let originalIsMuchBetter = originalScore > entry.score + max(1.0, thresholds.minimumMargin * 2) + let advantage = entry.score - originalScore let suffixPass = entry.suffixScore.map { $0 >= thresholds.minimumSuffixMeanLogProbability } ?? true let passesValidation: Bool if entry.candidate.source == .systemGrammarOnly { + // Grammar edits rewrite phrases the checker already vetted; the model only vetoes + // when it clearly prefers the user's phrasing (more than the standard margin). passesValidation = entry.score.isFinite - && !originalIsMuchBetter + && advantage >= -thresholds.minimumMargin && suffixPass } else { - passesValidation = entry.score >= thresholds.minimumMeanLogProbability - && margin >= thresholds.minimumMargin - && !originalIsMuchBetter - && suffixPass + passesValidation = Self.spellcheckValidationPasses( + advantage: advantage, + margin: margin, + suffixPass: suffixPass, + isClearCut: entry.candidate.source == .spellcheckOnly + && Self.boundedEditDistance( + entry.candidate.original.lowercased(), + entry.candidate.replacement.lowercased(), + cap: thresholds.clearCutMaximumEditDistance + ) != nil, + thresholds: thresholds + ) } guard passesValidation else { @@ -126,9 +174,84 @@ public final class CorrectionValidationScorer { } } + /// The pass/fail decision for a model-validated spellcheck candidate, extracted pure so tests + /// can assert real logged cases without a model. All quantities are joint log-probability + /// nats (ADR-120). + /// + /// `advantage` (replacement − original) is the primary gate: the correction must be at least + /// as plausible as what the user actually typed, which both admits valid-but-unlikely + /// continuations (an adverb after "I") and vetoes deliberate spellings the model knows. A + /// clear-cut candidate (dictionary-flagged + edit distance within the cap) only needs to beat + /// the runner-up by `clearCutMargin` — dominance, not the full separation bar — because + /// common typos systematically attract look-alike checker guesses. + static func spellcheckValidationPasses( + advantage: Double, + margin: Double, + suffixPass: Bool, + isClearCut: Bool, + thresholds: CorrectionValidationThresholds + ) -> Bool { + let requiredMargin = isClearCut ? thresholds.clearCutMargin : thresholds.minimumMargin + return advantage >= thresholds.minimumAdvantageOverOriginal + && margin >= requiredMargin + && suffixPass + } + + /// Splits `text` into everything before its trailing whitespace run and that run itself, + /// so the whitespace can lead the scored word instead of dangling off the anchor (ADR-120). + static func splitTrailingWhitespace(of text: String) -> (head: String, separator: String) { + var head = Substring(text) + var separator = "" + while let last = head.last, last.isWhitespace { + separator = String(last) + separator + head = head.dropLast() + } + return (String(head), separator) + } + + /// Levenshtein distance capped at `cap`: returns the distance when ≤ cap, else `nil`. + /// Early-outs on length difference and abandons rows whose minimum already exceeds the cap, + /// so pathological inputs stay cheap. + static func boundedEditDistance(_ a: String, _ b: String, cap: Int) -> Int? { + let s = Array(a), t = Array(b) + if abs(s.count - t.count) > cap { return nil } + if s.isEmpty { return t.count <= cap ? t.count : nil } + if t.isEmpty { return s.count <= cap ? s.count : nil } + + var previous = Array(0...t.count) + var current = [Int](repeating: 0, count: t.count + 1) + for i in 1...s.count { + current[0] = i + var rowMinimum = i + for j in 1...t.count { + let substitution = previous[j - 1] + (s[i - 1] == t[j - 1] ? 0 : 1) + current[j] = min(previous[j] + 1, current[j - 1] + 1, substitution) + rowMinimum = min(rowMinimum, current[j]) + } + if rowMinimum > cap { return nil } + swap(&previous, ¤t) + } + return previous[t.count] <= cap ? previous[t.count] : nil + } + + /// Joint log-probability of `text` after `anchor` — the log of the actual probability of the + /// string, used for all word-vs-word comparisons (ADR-120). + private func jointLogProbability(of text: String, anchor: [TokenID]) async throws -> Double { + try await totalLogProbability(of: text, anchor: anchor).total + } + + /// Per-token mean, kept for the suffix-join probe where the window text is identical across + /// candidates (so length normalisation can't distort a comparison) and the existing + /// `minimumSuffixMeanLogProbability` calibration still applies. private func meanLogProbability(of text: String, anchor: [TokenID]) async throws -> Double { + let (total, count) = try await totalLogProbability(of: text, anchor: anchor) + guard count > 0, total.isFinite else { return total } + return total / Double(count) + } + + private func totalLogProbability(of text: String, anchor: [TokenID]) async throws -> (total: Double, count: Int) { let tokens = try runtime.tokenizer.tokenize(text) - guard !tokens.isEmpty else { return -.infinity } + guard !tokens.isEmpty else { return (-.infinity, 0) } var suffix: [TokenID] = [] var total = 0.0 @@ -136,12 +259,12 @@ public final class CorrectionValidationScorer { try Task.checkCancellation() let logits = try await runtime.anchoredLogits(anchor: anchor, suffix: suffix) guard let logProbability = Self.logProbability(of: token, in: logits) else { - return -.infinity + return (-.infinity, tokens.count) } total += logProbability suffix.append(token) } - return total / Double(tokens.count) + return (total, tokens.count) } private static func logProbability(of token: TokenID, in logits: [TokenLogit]) -> Double? { diff --git a/Packages/ConstrainedGeneration/Tests/ConstrainedGenerationTests/Engine/CorrectionValidationScorerTests.swift b/Packages/ConstrainedGeneration/Tests/ConstrainedGenerationTests/Engine/CorrectionValidationScorerTests.swift index ad3e521..032dd66 100644 --- a/Packages/ConstrainedGeneration/Tests/ConstrainedGenerationTests/Engine/CorrectionValidationScorerTests.swift +++ b/Packages/ConstrainedGeneration/Tests/ConstrainedGenerationTests/Engine/CorrectionValidationScorerTests.swift @@ -1,5 +1,5 @@ import AutocompleteCore -import ConstrainedGeneration +@testable import ConstrainedGeneration import ModelRuntime import XCTest @@ -38,7 +38,9 @@ final class CorrectionValidationScorerTests: XCTestCase { XCTAssertEqual(result.first?.validation.method, .modelScore) } - func testDefaultMarginAllowsCloseSpellcheckRunnerUp() async throws { + func testCompressedNearTieMarginSuppressesEvenClearCut() async throws { + // logits 3.0 vs 2.76 → joint margin ~0.24 < clearCutMargin (0.5): a genuine near-tie + // between look-alike guesses stays suppressed. let scorer = CorrectionValidationScorer(runtime: Runtime(logitsByPath: [ [1]: [makeLogit(10, 3.0), makeLogit(11, 2.76), makeLogit(12, 0)] ])) @@ -48,12 +50,14 @@ final class CorrectionValidationScorerTests: XCTestCase { prefixBeforeWord: "in the " ) - XCTAssertEqual(result.map { $0.replacement }, ["middle"]) - XCTAssertGreaterThanOrEqual(result.first?.validation.margin ?? 0, 0.20) - XCTAssertLessThan(result.first?.validation.margin ?? .infinity, 0.35) + XCTAssertTrue(result.isEmpty) } - func testMisspelledOriginalDoesNotVetoClearSingleCandidate() async throws { + func testModelPreferringOriginalVetoesEvenSingleCandidate() async throws { + // The model rates the user's own string slightly higher (4.4 vs 4.0) → advantage < 0 → + // suppressed. This is the deliberate-spelling / proper-noun protection: any model + // preference for the original wins (ADR-120 strengthened this over the old + // "much better" slack). let scorer = CorrectionValidationScorer(runtime: Runtime(logitsByPath: [ [1]: [makeLogit(10, 4.0), makeLogit(12, 4.4), makeLogit(30, 0)] ])) @@ -63,6 +67,19 @@ final class CorrectionValidationScorerTests: XCTestCase { prefixBeforeWord: "in the " ) + XCTAssertTrue(result.isEmpty) + } + + func testReplacementPreferredOverOriginalShows() async throws { + let scorer = CorrectionValidationScorer(runtime: Runtime(logitsByPath: [ + [1]: [makeLogit(10, 4.4), makeLogit(12, 4.0), makeLogit(30, 0)] + ])) + + let result = try await scorer.validate( + candidates: [makeCandidate("middle")], + prefixBeforeWord: "in the " + ) + XCTAssertEqual(result.first?.replacement, "middle") } @@ -79,6 +96,24 @@ final class CorrectionValidationScorerTests: XCTestCase { XCTAssertTrue(result.isEmpty) } + func testWhitespaceOnlySuffixWindowSkipsTheJoinProbe() async throws { + // The live regression's final gate (ADR-120): right after typing "word ", the suffix + // window is a bare space — probing P(" " | prefix + replacement) is tokenization noise + // and must not veto. The stub tokenizer deliberately has no entry for " ". + let scorer = CorrectionValidationScorer(runtime: Runtime(logitsByPath: [ + [1]: [makeLogit(10, 5), makeLogit(12, 0)] + ])) + + let result = try await scorer.validate( + candidates: [makeCandidate("middle")], + prefixBeforeWord: "in the ", + suffixWindow: " " + ) + + XCTAssertEqual(result.first?.replacement, "middle") + XCTAssertNil(result.first?.validation.suffixJoinScore) + } + func testSuppressesWhenSuffixJoinIsWeak() async throws { let scorer = CorrectionValidationScorer(runtime: Runtime(logitsByPath: [ [2]: [makeLogit(10, 5), makeLogit(12, 0)], @@ -169,6 +204,111 @@ final class CorrectionValidationScorerTests: XCTestCase { XCTAssertEqual(result.first?.source, .systemGrammarValidatedByModel) } + // MARK: - Clear-cut margin relaxation (ADR-119) + + /// The live regression this retune exists for: predictions.log recorded + /// `CORRECT word="definately" -> SUPPRESS(lowModelMargin)`. On the real model the joint-nat + /// margin over "defiantly" is ~6, but look-alike guesses can compress it — a clear-cut + /// (dictionary-flagged, distance-1, model-top) guess must pass inside the compressed zone. + func testDefinatelyClearCutPassesInsideCompressedMarginZone() async throws { + // logits 3.0 vs 2.3 → joint margin ~0.70: above clearCutMargin (0.5), below minimumMargin (2.0). + let scorer = CorrectionValidationScorer(runtime: Runtime(logitsByPath: [ + [5]: [makeLogit(60, 3.0), makeLogit(61, 2.3), makeLogit(62, 0)] + ])) + + let result = try await scorer.validate( + candidates: [ + makeTypoCandidate(replacement: "definitely", id: "definitely"), + makeTypoCandidate(replacement: "defiantly", id: "defiantly") + ], + prefixBeforeWord: "I " + ) + + XCTAssertEqual(result.map(\.replacement), ["definitely"]) + XCTAssertEqual(result.first?.source, .spellcheckValidatedByModel) + let margin = try XCTUnwrap(result.first?.validation.margin) + XCTAssertGreaterThan(margin, 0.5) + XCTAssertLessThan(margin, 2.0, "test must exercise the clear-cut-only zone") + } + + func testClearCutNearTieStaysSuppressed() async throws { + // logits 3.0 vs 2.9 → joint margin ~0.1 < clearCutMargin: genuine ambiguity keeps the veto. + let scorer = CorrectionValidationScorer(runtime: Runtime(logitsByPath: [ + [5]: [makeLogit(60, 3.0), makeLogit(61, 2.9), makeLogit(62, 0)] + ])) + + let result = try await scorer.validate( + candidates: [ + makeTypoCandidate(replacement: "definitely", id: "definitely"), + makeTypoCandidate(replacement: "defiantly", id: "defiantly") + ], + prefixBeforeWord: "I " + ) + + XCTAssertTrue(result.isEmpty) + } + + func testDistantReplacementDoesNotGetClearCutRelaxation() async throws { + // "definately" → "define" is beyond the edit-distance cap, so the full 2.0-nat bar + // applies and a 0.70 margin still suppresses. + let scorer = CorrectionValidationScorer(runtime: Runtime(logitsByPath: [ + [5]: [makeLogit(63, 3.0), makeLogit(61, 2.3), makeLogit(62, 0)] + ])) + + let result = try await scorer.validate( + candidates: [ + makeTypoCandidate(replacement: "define", id: "define"), + makeTypoCandidate(replacement: "defiantly", id: "defiantly") + ], + prefixBeforeWord: "I " + ) + + XCTAssertTrue(result.isEmpty) + } + + func testSpellcheckDecisionBoundaries() { + let thresholds = CorrectionValidationThresholds() + func passes(advantage: Double = 3.0, margin: Double, suffixPass: Bool = true, clearCut: Bool) -> Bool { + CorrectionValidationScorer.spellcheckValidationPasses( + advantage: advantage, + margin: margin, + suffixPass: suffixPass, + isClearCut: clearCut, + thresholds: thresholds + ) + } + + XCTAssertTrue(passes(margin: 0.7, clearCut: true), "the compressed-margin zone opens for clear-cut") + XCTAssertFalse(passes(margin: 0.7, clearCut: false), "non-clear-cut keeps the 2.0-nat bar") + XCTAssertTrue(passes(margin: 2.2, clearCut: false), "well-separated candidates pass without relaxation") + XCTAssertFalse(passes(margin: 0.1, clearCut: true), "near-ties stay suppressed") + XCTAssertFalse(passes(advantage: -0.4, margin: 0.7, clearCut: true), + "any model preference for the original vetoes — never relaxed") + XCTAssertFalse(passes(margin: 0.7, suffixPass: false, clearCut: true), "suffix join still gates") + XCTAssertFalse(passes(margin: -0.3, clearCut: true), "a non-top candidate never passes") + } + + func testBoundedEditDistance() { + XCTAssertEqual(CorrectionValidationScorer.boundedEditDistance("definately", "definitely", cap: 2), 1) + XCTAssertEqual(CorrectionValidationScorer.boundedEditDistance("recieve", "receive", cap: 2), 2) + XCTAssertEqual(CorrectionValidationScorer.boundedEditDistance("colour", "color", cap: 2), 1) + XCTAssertNil(CorrectionValidationScorer.boundedEditDistance("definately", "define", cap: 2)) + XCTAssertNil(CorrectionValidationScorer.boundedEditDistance("cat", "elephant", cap: 2)) + XCTAssertEqual(CorrectionValidationScorer.boundedEditDistance("", "ab", cap: 2), 2) + XCTAssertNil(CorrectionValidationScorer.boundedEditDistance("", "abc", cap: 2)) + } + + private func makeTypoCandidate(replacement: String, id: String) -> CorrectionCandidate { + CorrectionCandidate( + original: "definately", + replacement: replacement, + originalRange: range, + confidence: 0.83, + source: .spellcheckOnly, + validation: .spellcheckOnly + ) + } + private func makeCandidate(_ replacement: String) -> CorrectionCandidate { CorrectionCandidate( original: "mdidle", @@ -187,20 +327,27 @@ private func makeLogit(_ id: TokenID, _ value: Float) -> TokenLogit { } private struct Tokenizer: ModelTokenizing { + // Anchors arrive whitespace-trimmed and words space-prefixed (ADR-120): the scorer moves the + // anchor's trailing separator onto the scored word so tokenization matches real model input. private let ids: [String: TokenID] = [ - "in the ": 1, - "Open the ": 2, - "I saw ": 3, - "These popsicles ": 4, - "middle": 10, - "muddle": 11, - "mdidle": 12, + "in the": 1, + "Open the": 2, + "I saw": 3, + "These popsicles": 4, + " middle": 10, + " muddle": 11, + " mdidle": 12, " of": 20, " config file": 21, - "an apple": 40, - "a apple": 41, - "is": 50, - "are": 51 + " an apple": 40, + " a apple": 41, + " is": 50, + " are": 51, + "I": 5, + " definitely": 60, + " defiantly": 61, + " definately": 62, + " define": 63 ] func tokenize(_ text: String) throws -> [TokenID] { diff --git a/docs/05-decisions.md b/docs/05-decisions.md index 017deda..73b1354 100644 --- a/docs/05-decisions.md +++ b/docs/05-decisions.md @@ -135,6 +135,7 @@ row here.** | 113 | Route prefix-only spellcheck replacements to completion | correction/completion | | 114 | Lower spellcheck correction model margin | correction/model-runtime | | 115 | Remove aggressive correction mode | correction/settings | +| 116 | Correction validation: joint log-probability scoring + guards | correction/model-runtime | --- @@ -3589,3 +3590,49 @@ text. Both are now closed: detector and correction validation thresholds. - Consequences: Correction behavior is simpler to reason about and the Settings UI has one fewer safety-related toggle. Previously stored user defaults for the removed key are ignored. + + +## ADR-116 — Correction validation: joint log-probability scoring, clear-cut margins, and guards + +- Date: 2026-07-11 +- Status: accepted +- Context: The autocorrect lane (ADR-108–115) detected typos but suppressed nearly every fix even + in ideal prose context (e.g. `definately` after "I " stayed suppressed as `lowModelMargin`). + Instrumenting `CorrectionValidationScorer` against a real GGUF found the thresholds were not the + problem — the scoring was, in three compounding ways, which explains the escalating threshold + surgery in ADR-113/114/115: + 1. The anchor kept its trailing space, so candidate words tokenized standalone — a split that + BPE vocabularies essentially never produce after a space token, flooring every score. + 2. Per-token **mean** log-probability systematically favoured multi-token strings (their tails + are near-deterministic): measured, `" definitely"` (1 token) scored mean −8.09 while + `" definately"` (2 tokens) scored −6.56 — the misspelling out-scored its own correction. The + joint (summed) log-probabilities are −8.09 vs −13.12, i.e. the correction wins by ~5 nats. + 3. The −6.0 absolute mean floor asked "is this a likely continuation?" — the wrong question for + a correction (an adverb after "I" is valid but unlikely), so every candidate died at the + floor. +- Decision: + - Move the anchor's trailing whitespace onto the scored word (the correction-lane analogue of + ADR-017's caret-boundary sanitization), so tokenization matches real model input. + - Compare candidates and the user's original on **joint (summed)** log-probability; the + per-token mean survives only for the suffix-join probe, where the window text is identical + across candidates. + - Replace the absolute floor and the "original is much better" slack with one gate: the + replacement's joint score must beat the user's own string by `minimumAdvantageOverOriginal` + (default 0 — any model preference for the original vetoes, which strengthens deliberate- + spelling / proper-noun protection). Dictionary flagging already guarantees the replacement is + a real word. + - Recalibrate margins to joint nats: `minimumMargin` 2.0, with a relaxed `clearCutMargin` 0.5 + for dictionary-flagged candidates within edit distance 2 that are the model's top guess + (common typos attract look-alike checker guesses that compress the margin). + - Skip the suffix-join probe for whitespace-only windows (probing a bare space token is + tokenization noise that vetoed corrections typed at the end of a sentence). + - Only correct an *open* current word when it is a dead-end fragment (no possible dictionary + completion); a fragment still extensible into a word is typing in progress and belongs to the + completion lane (ADR-113), mirroring the in-beam typo guard's closed-word-only rule (ADR-015). + - Render the badge with a per-letter diff (`CorrectionLetterDiff`): strike only the wrong + letters, embolden only the changed ones (`definately` -> `definitely`). +- Consequences: The live regression passes end to end — real-model verdict `definitely` at + confidence 0.95, joint margin ~5.4 (thin context) / ~6.0 (rich); textbook one-to-two-letter + typos surface while ambiguous near-ties and deliberate spellings stay suppressed. Thresholds are + now in interpretable units (nats of probability ratio). Correction latency is unchanged (joint + scoring decodes the same tokens as before). Verified by unit tests pinning the logged cases.