From 1cdd95e7b31d8f85f133107e4b28c58a373a17ea Mon Sep 17 00:00:00 2001 From: MarcusJRLee <7527115+MarcusJRLee@users.noreply.github.com> Date: Fri, 28 Aug 2026 01:01:19 -0400 Subject: [PATCH] Show Local AI pipeline evidence --- README.md | 5 ++ .../local_ai_pipeline_presentation.swift | 65 +++++++++++++++++++ .../local_ai_settings_view.swift | 64 +++++++++++++++++- .../voice_history_model.swift | 32 ++++++++- .../voice_history_view.swift | 26 ++++++++ .../local_ai_pipeline_presentation_test.swift | 53 +++++++++++++++ .../voice_history_model_test.swift | 5 ++ docs/architecture.md | 4 ++ .../0054_typed_local_ai_formatting_output.md | 2 + docs/product_brief.md | 2 + docs/user_guide.md | 23 +++++-- 11 files changed, 272 insertions(+), 9 deletions(-) create mode 100644 Sources/HardwareControllerApp/local_ai_pipeline_presentation.swift create mode 100644 Tests/HardwareControllerAppTests/local_ai_pipeline_presentation_test.swift diff --git a/README.md b/README.md index 493dea3..d57bee4 100644 --- a/README.md +++ b/README.md @@ -216,6 +216,11 @@ quotations, and dictionary terms. A provider error, invalid output, or three-second deadline delivers the deterministic Edited transcript once when the captured target is still safe. +**General → Local AI Dictation** names speech-to-text and formatting separately. +Its active-pipeline evidence shows each provider and model, typed output, +validation boundary, and deterministic fallback. History records whether each +formatted result was validated or used the Edited fallback. + ### Apple On-Device Apple On-Device refinement requires macOS 26, Apple Intelligence enabled, a diff --git a/Sources/HardwareControllerApp/local_ai_pipeline_presentation.swift b/Sources/HardwareControllerApp/local_ai_pipeline_presentation.swift new file mode 100644 index 0000000..1749849 --- /dev/null +++ b/Sources/HardwareControllerApp/local_ai_pipeline_presentation.swift @@ -0,0 +1,65 @@ +import Foundation +import HardwareControllerCore +import HardwareControllerMac + +struct LocalAIPipelinePresentation: Equatable, Sendable { + let speechProvider: String + let speechModel: String + let formattingProvider: String + let formattingModel: String + let formattingOutput: String + let effectiveCasing: String + let fallback: String + let validation: String + + init( + settings: LocalAISettings, + operatingSystemVersion: OperatingSystemVersion = + ProcessInfo.processInfo.operatingSystemVersion + ) { + speechProvider = "Apple On-Device" + speechModel = + operatingSystemVersion.majorVersion >= 26 + ? "SpeechAnalyzer + DictationTranscriber (OS-managed)" + : "SFSpeechRecognizer (OS-managed)" + formattingOutput = + "Typed paragraph/list blocks · prompt r\(VersionedLocalAIPromptBuilder.currentRevision)" + effectiveCasing = Self.casingTitle(settings.effectiveCasingPolicy) + fallback = "Edited transcript" + validation = "Protected text + semantic bounds" + + if settings.style.kind == .verbatim { + formattingProvider = "None — Verbatim" + formattingModel = "Not used" + } else { + switch settings.provider { + case .appleOnDevice: + formattingProvider = "Apple On-Device" + formattingModel = "SystemLanguageModel (OS-managed)" + case .ollama: + formattingProvider = "Ollama (localhost)" + formattingModel = Self.ollamaModelTitle(settings.ollamaModel) + } + } + } + + private static func ollamaModelTitle( + _ selection: LocalAIModelSelection + ) -> String { + guard let digest = selection.expectedDigest else { + return selection.name + } + return "\(selection.name) @ \(digest.prefix(12))…" + } + + private static func casingTitle(_ policy: VoiceCasingPolicy) -> String { + switch policy { + case .styleDefault: + "Style Default" + case .lowercaseProse: + "Lowercase Prose" + case .strictLowercase: + "Strict Lowercase" + } + } +} diff --git a/Sources/HardwareControllerApp/local_ai_settings_view.swift b/Sources/HardwareControllerApp/local_ai_settings_view.swift index d8cc7ff..8964417 100644 --- a/Sources/HardwareControllerApp/local_ai_settings_view.swift +++ b/Sources/HardwareControllerApp/local_ai_settings_view.swift @@ -11,7 +11,7 @@ struct LocalAISettingsSection: View { var body: some View { Section("Local AI Dictation") { - Picker("Provider", selection: providerBinding) { + Picker("Formatting provider", selection: providerBinding) { Text("Apple On-Device") .tag(LocalAIProviderKind.appleOnDevice) Text("Ollama") @@ -28,8 +28,17 @@ struct LocalAISettingsSection: View { .font(.caption) .foregroundStyle(.secondary) + Picker("Casing", selection: casingBinding) { + ForEach(VoiceCasingPolicy.allCases, id: \.self) { policy in + Text(casingTitle(policy)).tag(policy) + } + } + Text(casingDescription(settings.effectiveCasingPolicy)) + .font(.caption) + .foregroundStyle(.secondary) + if settings.provider == .ollama { - Picker("Model", selection: modelBinding) { + Picker("Formatting model", selection: modelBinding) { ForEach(modelOptions) { option in Text(modelTitle(option)).tag(option.name) } @@ -47,6 +56,22 @@ struct LocalAISettingsSection: View { .foregroundStyle(.secondary) } + VStack(alignment: .leading, spacing: 8) { + Text("Active pipeline") + .font(.headline) + LabeledContent("Speech provider", value: pipeline.speechProvider) + LabeledContent("Speech model", value: pipeline.speechModel) + LabeledContent( + "Formatting provider", + value: pipeline.formattingProvider + ) + LabeledContent("Formatting model", value: pipeline.formattingModel) + LabeledContent("Formatting output", value: pipeline.formattingOutput) + LabeledContent("Effective casing", value: pipeline.effectiveCasing) + LabeledContent("Validation", value: pipeline.validation) + LabeledContent("Fallback", value: pipeline.fallback) + } + LabeledContent("Status") { HStack(spacing: 6) { Image(systemName: readinessSymbol) @@ -209,6 +234,10 @@ struct LocalAISettingsSection: View { preferencesModel.localAISettings } + private var pipeline: LocalAIPipelinePresentation { + LocalAIPipelinePresentation(settings: settings) + } + private var selectedReadiness: LocalAIProviderReadiness { model.localAIReadiness.readiness(for: settings.provider) } @@ -260,6 +289,15 @@ struct LocalAISettingsSection: View { ) } + private var casingBinding: SwiftUI.Binding { + SwiftUI.Binding( + get: { settings.casingPolicy }, + set: { policy in + updateSettings { $0.casingPolicy = policy } + } + ) + } + private var modelBinding: SwiftUI.Binding { SwiftUI.Binding( get: { settings.ollamaModel.name }, @@ -412,6 +450,28 @@ struct LocalAISettingsSection: View { } } + private func casingTitle(_ policy: VoiceCasingPolicy) -> String { + switch policy { + case .styleDefault: + "Style Default" + case .lowercaseProse: + "Lowercase Prose" + case .strictLowercase: + "Strict Lowercase" + } + } + + private func casingDescription(_ policy: VoiceCasingPolicy) -> String { + switch policy { + case .styleDefault: + "Follows the selected Style." + case .lowercaseProse: + "Lowercases prose while preserving names, acronyms, and operational tokens." + case .strictLowercase: + "Lowercases prose while preserving URLs, email addresses, paths, code tokens, quoted text, and Dictionary values." + } + } + private var normalizedVocabularyEntry: String { vocabularyEntry.trimmingCharacters(in: .whitespacesAndNewlines) } diff --git a/Sources/HardwareControllerApp/voice_history_model.swift b/Sources/HardwareControllerApp/voice_history_model.swift index 2935825..211a8da 100644 --- a/Sources/HardwareControllerApp/voice_history_model.swift +++ b/Sources/HardwareControllerApp/voice_history_model.swift @@ -580,6 +580,11 @@ private actor DemoVoiceSessionHistory: VoiceSessionHistoryManaging { raw: "first install git then run bash version", formatted: "1. Install Git.\n2. Run `bash --version`.", style: .technical, + formattedBlock: VoiceFormattedBlock( + kind: .orderedList, + items: ["Install Git.", "Run `bash --version`."], + evidenceIndices: [0] + ), pinned: true ), Self.item( @@ -589,6 +594,12 @@ private actor DemoVoiceSessionHistory: VoiceSessionHistoryManaging { raw: "send the revised plan tomorrow", formatted: "send the revised plan tomorrow.", style: .casualMessage, + formattedBlock: VoiceFormattedBlock( + kind: .paragraph, + items: ["send the revised plan tomorrow."], + evidenceIndices: [0] + ), + formattingValidationStatus: .sourceFallback, pinned: false, audioExpirationReason: .byteLimit ), @@ -739,6 +750,8 @@ private actor DemoVoiceSessionHistory: VoiceSessionHistoryManaging { raw: String, formatted: String, style: VoiceStyle, + formattedBlock: VoiceFormattedBlock? = nil, + formattingValidationStatus: VoiceFormattingValidationStatus = .validated, pinned: Bool, audioExpirationReason: VoiceHistoryAudioExpirationReason? = nil, recoveryKind: VoiceHistoryRecoveryKind? = nil, @@ -797,7 +810,24 @@ private actor DemoVoiceSessionHistory: VoiceSessionHistoryManaging { style: style, provider: .appleOnDevice, modelIdentifier: "Apple SystemLanguageModel", - promptRevision: VersionedLocalAIPromptBuilder.currentRevision + promptRevision: VersionedLocalAIPromptBuilder.currentRevision, + formattedDocument: formattedBlock.map { block in + VoiceFormattedDocument( + rawText: raw, + style: style, + blocks: [block], + evidence: [ + VoiceFormattingEvidence( + rawUTF8StartOffset: 0, + rawUTF8EndOffset: raw.utf8.count, + provider: .appleOnDevice, + modelIdentifier: "Apple SystemLanguageModel", + promptRevision: VersionedLocalAIPromptBuilder.currentRevision + ) + ], + validationStatus: formattingValidationStatus + ) + } ), VoiceHistoryResult( sessionID: id, diff --git a/Sources/HardwareControllerApp/voice_history_view.swift b/Sources/HardwareControllerApp/voice_history_view.swift index 9e63a33..9494c06 100644 --- a/Sources/HardwareControllerApp/voice_history_view.swift +++ b/Sources/HardwareControllerApp/voice_history_view.swift @@ -397,6 +397,16 @@ struct VoiceHistoryView: View { if let promptRevision = result.promptRevision { evidenceLabel("Prompt", value: "r\(promptRevision)") } + if let formattedDocument = result.formattedDocument { + evidenceLabel( + "Validation", + value: formattedDocument.validationStatus.title + ) + evidenceLabel( + "Fallback", + value: formattedDocument.validationStatus.fallbackTitle + ) + } } .id(result.id) .accessibilityElement(children: .combine) @@ -724,6 +734,22 @@ extension LocalAIProviderKind { } } +extension VoiceFormattingValidationStatus { + fileprivate var title: String { + switch self { + case .validated: "Validated" + case .sourceFallback: "Provider output rejected" + } + } + + fileprivate var fallbackTitle: String { + switch self { + case .validated: "Not used" + case .sourceFallback: "Edited transcript" + } + } +} + extension VoiceSessionDeliveryOutcome { fileprivate var title: String { switch self { diff --git a/Tests/HardwareControllerAppTests/local_ai_pipeline_presentation_test.swift b/Tests/HardwareControllerAppTests/local_ai_pipeline_presentation_test.swift new file mode 100644 index 0000000..8c10439 --- /dev/null +++ b/Tests/HardwareControllerAppTests/local_ai_pipeline_presentation_test.swift @@ -0,0 +1,53 @@ +import Foundation +import Testing + +@testable import HardwareControllerApp +@testable import HardwareControllerCore + +struct LocalAIPipelinePresentationTest { + @Test + func distinguishesAppleSpeechFromOllamaFormatting() { + var settings = LocalAISettings.default + settings.provider = .ollama + settings.additionalInstructions = "only provide text in lowercase" + + let presentation = LocalAIPipelinePresentation( + settings: settings, + operatingSystemVersion: OperatingSystemVersion( + majorVersion: 26, + minorVersion: 0, + patchVersion: 0 + ) + ) + + #expect(presentation.speechProvider == "Apple On-Device") + #expect( + presentation.speechModel + == "SpeechAnalyzer + DictationTranscriber (OS-managed)" + ) + #expect(presentation.formattingProvider == "Ollama (localhost)") + #expect(presentation.formattingModel.contains("qwen3.5:4b")) + #expect(presentation.effectiveCasing == "Strict Lowercase") + #expect(presentation.fallback == "Edited transcript") + #expect(presentation.validation == "Protected text + semantic bounds") + } + + @Test + func verbatimMakesFormattingBypassExplicit() { + var settings = LocalAISettings.default + settings.style = .verbatim + + let presentation = LocalAIPipelinePresentation( + settings: settings, + operatingSystemVersion: OperatingSystemVersion( + majorVersion: 25, + minorVersion: 0, + patchVersion: 0 + ) + ) + + #expect(presentation.speechModel == "SFSpeechRecognizer (OS-managed)") + #expect(presentation.formattingProvider == "None — Verbatim") + #expect(presentation.formattingModel == "Not used") + } +} diff --git a/Tests/HardwareControllerAppTests/voice_history_model_test.swift b/Tests/HardwareControllerAppTests/voice_history_model_test.swift index ea50b04..9fd9c32 100644 --- a/Tests/HardwareControllerAppTests/voice_history_model_test.swift +++ b/Tests/HardwareControllerAppTests/voice_history_model_test.swift @@ -126,7 +126,12 @@ struct VoiceHistoryModelTest { #expect(results[1].sourceResultID == results[0].id) #expect(results[2].sourceResultID == results[1].id) #expect(results[1].provider == .appleOnDevice) + #expect(results[1].formattedDocument?.validationStatus == .validated) #expect(presentation.model.sessions[1].audioExpiredAt != nil) + #expect( + presentation.model.sessions[1].results[1].formattedDocument? + .validationStatus == .sourceFallback + ) #expect( presentation.model.sessions[1].audioExpirationReason == .byteLimit ) diff --git a/docs/architecture.md b/docs/architecture.md index 790af9b..1293fc1 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -466,6 +466,10 @@ renderer alone decides whether the captured target receives structure or one plain line. Validation runs on the canonical rendering before delivery. The sanitized provider test shares the three-second preparation-plus-generation deadline; settings changes and shutdown cancel it and suppress stale results. +General settings present speech and formatting provider/model identity as +separate active-pipeline evidence together with typed output, validation, and +fallback behavior. History renders the stored validation status and whether the +Edited fallback was used; it does not infer success from provider identity. The revision-2 Swift spoken-edit engine recognizes only exact, case-insensitive command phrases in immutable Raw text, so a Dictionary replacement cannot diff --git a/docs/decisions/0054_typed_local_ai_formatting_output.md b/docs/decisions/0054_typed_local_ai_formatting_output.md index 3720810..e029b22 100644 --- a/docs/decisions/0054_typed_local_ai_formatting_output.md +++ b/docs/decisions/0054_typed_local_ai_formatting_output.md @@ -56,3 +56,5 @@ Model output no longer owns list inference or lowercase safety. A larger model can improve prose, but it cannot replace deterministic semantics, canonical validation, or fallback. This decision does not change the ASR provider or supersede the existing Ollama recommendation. +Settings therefore name ASR and formatting evidence separately, while History +shows the stored validation result and deterministic fallback use. diff --git a/docs/product_brief.md b/docs/product_brief.md index e0cf376..e03687c 100644 --- a/docs/product_brief.md +++ b/docs/product_brief.md @@ -213,6 +213,8 @@ Given a focused editable field and a ready selected provider: - cancellation, process change, secure-status change, focus change, or caret change discards late model output and stores a typed delivery reason; - raw and refined recovery controls remain distinct. +- General settings identify speech and formatting providers/models separately; + History identifies validation acceptance and deterministic fallback use. ### Dictation handoff diff --git a/docs/user_guide.md b/docs/user_guide.md index 6f7383a..01ccd40 100644 --- a/docs/user_guide.md +++ b/docs/user_guide.md @@ -162,29 +162,36 @@ playback in History. Speech content is never logged. Open **General → Local AI Dictation** before assigning the Action: -1. Choose **Apple On-Device** or **Ollama**. +1. Choose **Apple On-Device** or **Ollama** under **Formatting provider**. 2. Choose **Natural**, **Casual Message**, **Formal**, **Technical**, or **Verbatim** under **Style**. Verbatim uses recognition, explicit spoken edits, and exact Dictionary replacements but skips the generative model. Casual Message prefers lowercase sentence starts while preserving required proper-name and Dictionary capitalization. -3. For Ollama, start the local service and install the recommended model: +3. Choose **Style Default**, **Lowercase Prose**, or **Strict Lowercase** under + **Casing**. Strict Lowercase protects operational tokens such as URLs, + email addresses, paths, code tokens, quoted text, and Dictionary values. +4. For Ollama, start the local service and install the recommended model: ```bash ollama pull qwen3.5:4b ``` -4. Choose an installed model and **5 minutes** or **Until app quits** retention. +5. Choose an installed formatting model and **5 minutes** or **Until app quits** + retention. On model change or quit, the app unloads a model it started. It leaves a model that was already running for another local Ollama client untouched. -5. Choose **Refresh Status**, then **Test Selected Provider**. The test uses a +6. Confirm **Active pipeline**. It names the speech provider and OS-managed + recognizer, formatting provider and model, typed output, validation, and + Edited fallback independently. +7. Choose **Refresh Status**, then **Test Selected Provider**. The test uses a fixed sanitized phrase without microphone or focused-field access. -6. Optionally expand **Personal dictionary**. Type a recognition term into its +8. Optionally expand **Personal dictionary**. Type a recognition term into its outlined field and choose **Add**, or fill both outlined replacement fields before choosing **Add**. You can also enable nearby-text context or provide formatting instructions. -7. Assign **Local AI Dictation** to a Control under Controller or Profiles. +9. Assign **Local AI Dictation** to a Control under Controller or Profiles. Apple On-Device requires macOS 26, Apple Intelligence enabled, a supported locale, and installed model assets. It never enables Private Cloud Compute. @@ -201,6 +208,10 @@ prompt-safety rules. The exact instruction `only provide text in lowercase` normalizes to Strict Lowercase and overrides Style capitalization while still protecting operational tokens. +History shows **Validation: Validated** and **Fallback: Not used** for accepted +formatter output. Rejected provider output shows **Provider output rejected** +and **Edited transcript** instead. + Nearby text is off by default. When enabled, the app reads at most a bounded window around the caret from an approved multiline, nonsecure Accessibility target for the current session. Single-line fields, browsers' compatibility