Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -216,6 +216,11 @@ quotations, and dictionary terms. A provider error, invalid output, or
three-second deadline delivers the deterministic Edited transcript once when
the captured target is still safe.

**General → Local AI Dictation** names speech-to-text and formatting separately.
Its active-pipeline evidence shows each provider and model, typed output,
validation boundary, and deterministic fallback. History records whether each
formatted result was validated or used the Edited fallback.

### Apple On-Device

Apple On-Device refinement requires macOS 26, Apple Intelligence enabled, a
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
import Foundation
import HardwareControllerCore
import HardwareControllerMac

struct LocalAIPipelinePresentation: Equatable, Sendable {
let speechProvider: String
let speechModel: String
let formattingProvider: String
let formattingModel: String
let formattingOutput: String
let effectiveCasing: String
let fallback: String
let validation: String

init(
settings: LocalAISettings,
operatingSystemVersion: OperatingSystemVersion =
ProcessInfo.processInfo.operatingSystemVersion
) {
speechProvider = "Apple On-Device"
speechModel =
operatingSystemVersion.majorVersion >= 26
? "SpeechAnalyzer + DictationTranscriber (OS-managed)"
: "SFSpeechRecognizer (OS-managed)"
formattingOutput =
"Typed paragraph/list blocks · prompt r\(VersionedLocalAIPromptBuilder.currentRevision)"
effectiveCasing = Self.casingTitle(settings.effectiveCasingPolicy)
fallback = "Edited transcript"
validation = "Protected text + semantic bounds"

if settings.style.kind == .verbatim {
formattingProvider = "None — Verbatim"
formattingModel = "Not used"
} else {
switch settings.provider {
case .appleOnDevice:
formattingProvider = "Apple On-Device"
formattingModel = "SystemLanguageModel (OS-managed)"
case .ollama:
formattingProvider = "Ollama (localhost)"
formattingModel = Self.ollamaModelTitle(settings.ollamaModel)
}
}
}

private static func ollamaModelTitle(
_ selection: LocalAIModelSelection
) -> String {
guard let digest = selection.expectedDigest else {
return selection.name
}
return "\(selection.name) @ \(digest.prefix(12))…"
}

private static func casingTitle(_ policy: VoiceCasingPolicy) -> String {
switch policy {
case .styleDefault:
"Style Default"
case .lowercaseProse:
"Lowercase Prose"
case .strictLowercase:
"Strict Lowercase"
}
}
}
64 changes: 62 additions & 2 deletions Sources/HardwareControllerApp/local_ai_settings_view.swift
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ struct LocalAISettingsSection: View {

var body: some View {
Section("Local AI Dictation") {
Picker("Provider", selection: providerBinding) {
Picker("Formatting provider", selection: providerBinding) {
Text("Apple On-Device")
.tag(LocalAIProviderKind.appleOnDevice)
Text("Ollama")
Expand All @@ -28,8 +28,17 @@ struct LocalAISettingsSection: View {
.font(.caption)
.foregroundStyle(.secondary)

Picker("Casing", selection: casingBinding) {
ForEach(VoiceCasingPolicy.allCases, id: \.self) { policy in
Text(casingTitle(policy)).tag(policy)
}
}
Text(casingDescription(settings.effectiveCasingPolicy))
.font(.caption)
.foregroundStyle(.secondary)

if settings.provider == .ollama {
Picker("Model", selection: modelBinding) {
Picker("Formatting model", selection: modelBinding) {
ForEach(modelOptions) { option in
Text(modelTitle(option)).tag(option.name)
}
Expand All @@ -47,6 +56,22 @@ struct LocalAISettingsSection: View {
.foregroundStyle(.secondary)
}

VStack(alignment: .leading, spacing: 8) {
Text("Active pipeline")
.font(.headline)
LabeledContent("Speech provider", value: pipeline.speechProvider)
LabeledContent("Speech model", value: pipeline.speechModel)
LabeledContent(
"Formatting provider",
value: pipeline.formattingProvider
)
LabeledContent("Formatting model", value: pipeline.formattingModel)
LabeledContent("Formatting output", value: pipeline.formattingOutput)
LabeledContent("Effective casing", value: pipeline.effectiveCasing)
LabeledContent("Validation", value: pipeline.validation)
LabeledContent("Fallback", value: pipeline.fallback)
}

LabeledContent("Status") {
HStack(spacing: 6) {
Image(systemName: readinessSymbol)
Expand Down Expand Up @@ -209,6 +234,10 @@ struct LocalAISettingsSection: View {
preferencesModel.localAISettings
}

private var pipeline: LocalAIPipelinePresentation {
LocalAIPipelinePresentation(settings: settings)
}

private var selectedReadiness: LocalAIProviderReadiness {
model.localAIReadiness.readiness(for: settings.provider)
}
Expand Down Expand Up @@ -260,6 +289,15 @@ struct LocalAISettingsSection: View {
)
}

private var casingBinding: SwiftUI.Binding<VoiceCasingPolicy> {
SwiftUI.Binding(
get: { settings.casingPolicy },
set: { policy in
updateSettings { $0.casingPolicy = policy }
}
)
}

private var modelBinding: SwiftUI.Binding<String> {
SwiftUI.Binding(
get: { settings.ollamaModel.name },
Expand Down Expand Up @@ -412,6 +450,28 @@ struct LocalAISettingsSection: View {
}
}

private func casingTitle(_ policy: VoiceCasingPolicy) -> String {
switch policy {
case .styleDefault:
"Style Default"
case .lowercaseProse:
"Lowercase Prose"
case .strictLowercase:
"Strict Lowercase"
}
}

private func casingDescription(_ policy: VoiceCasingPolicy) -> String {
switch policy {
case .styleDefault:
"Follows the selected Style."
case .lowercaseProse:
"Lowercases prose while preserving names, acronyms, and operational tokens."
case .strictLowercase:
"Lowercases prose while preserving URLs, email addresses, paths, code tokens, quoted text, and Dictionary values."
}
}

private var normalizedVocabularyEntry: String {
vocabularyEntry.trimmingCharacters(in: .whitespacesAndNewlines)
}
Expand Down
32 changes: 31 additions & 1 deletion Sources/HardwareControllerApp/voice_history_model.swift
Original file line number Diff line number Diff line change
Expand Up @@ -580,6 +580,11 @@ private actor DemoVoiceSessionHistory: VoiceSessionHistoryManaging {
raw: "first install git then run bash version",
formatted: "1. Install Git.\n2. Run `bash --version`.",
style: .technical,
formattedBlock: VoiceFormattedBlock(
kind: .orderedList,
items: ["Install Git.", "Run `bash --version`."],
evidenceIndices: [0]
),
pinned: true
),
Self.item(
Expand All @@ -589,6 +594,12 @@ private actor DemoVoiceSessionHistory: VoiceSessionHistoryManaging {
raw: "send the revised plan tomorrow",
formatted: "send the revised plan tomorrow.",
style: .casualMessage,
formattedBlock: VoiceFormattedBlock(
kind: .paragraph,
items: ["send the revised plan tomorrow."],
evidenceIndices: [0]
),
formattingValidationStatus: .sourceFallback,
pinned: false,
audioExpirationReason: .byteLimit
),
Expand Down Expand Up @@ -739,6 +750,8 @@ private actor DemoVoiceSessionHistory: VoiceSessionHistoryManaging {
raw: String,
formatted: String,
style: VoiceStyle,
formattedBlock: VoiceFormattedBlock? = nil,
formattingValidationStatus: VoiceFormattingValidationStatus = .validated,
pinned: Bool,
audioExpirationReason: VoiceHistoryAudioExpirationReason? = nil,
recoveryKind: VoiceHistoryRecoveryKind? = nil,
Expand Down Expand Up @@ -797,7 +810,24 @@ private actor DemoVoiceSessionHistory: VoiceSessionHistoryManaging {
style: style,
provider: .appleOnDevice,
modelIdentifier: "Apple SystemLanguageModel",
promptRevision: VersionedLocalAIPromptBuilder.currentRevision
promptRevision: VersionedLocalAIPromptBuilder.currentRevision,
formattedDocument: formattedBlock.map { block in
VoiceFormattedDocument(
rawText: raw,
style: style,
blocks: [block],
evidence: [
VoiceFormattingEvidence(
rawUTF8StartOffset: 0,
rawUTF8EndOffset: raw.utf8.count,
provider: .appleOnDevice,
modelIdentifier: "Apple SystemLanguageModel",
promptRevision: VersionedLocalAIPromptBuilder.currentRevision
)
],
validationStatus: formattingValidationStatus
)
}
),
VoiceHistoryResult(
sessionID: id,
Expand Down
26 changes: 26 additions & 0 deletions Sources/HardwareControllerApp/voice_history_view.swift
Original file line number Diff line number Diff line change
Expand Up @@ -397,6 +397,16 @@ struct VoiceHistoryView: View {
if let promptRevision = result.promptRevision {
evidenceLabel("Prompt", value: "r\(promptRevision)")
}
if let formattedDocument = result.formattedDocument {
evidenceLabel(
"Validation",
value: formattedDocument.validationStatus.title
)
evidenceLabel(
"Fallback",
value: formattedDocument.validationStatus.fallbackTitle
)
}
}
.id(result.id)
.accessibilityElement(children: .combine)
Expand Down Expand Up @@ -724,6 +734,22 @@ extension LocalAIProviderKind {
}
}

extension VoiceFormattingValidationStatus {
fileprivate var title: String {
switch self {
case .validated: "Validated"
case .sourceFallback: "Provider output rejected"
}
}

fileprivate var fallbackTitle: String {
switch self {
case .validated: "Not used"
case .sourceFallback: "Edited transcript"
}
}
}

extension VoiceSessionDeliveryOutcome {
fileprivate var title: String {
switch self {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
import Foundation
import Testing

@testable import HardwareControllerApp
@testable import HardwareControllerCore

struct LocalAIPipelinePresentationTest {
@Test
func distinguishesAppleSpeechFromOllamaFormatting() {
var settings = LocalAISettings.default
settings.provider = .ollama
settings.additionalInstructions = "only provide text in lowercase"

let presentation = LocalAIPipelinePresentation(
settings: settings,
operatingSystemVersion: OperatingSystemVersion(
majorVersion: 26,
minorVersion: 0,
patchVersion: 0
)
)

#expect(presentation.speechProvider == "Apple On-Device")
#expect(
presentation.speechModel
== "SpeechAnalyzer + DictationTranscriber (OS-managed)"
)
#expect(presentation.formattingProvider == "Ollama (localhost)")
#expect(presentation.formattingModel.contains("qwen3.5:4b"))
#expect(presentation.effectiveCasing == "Strict Lowercase")
#expect(presentation.fallback == "Edited transcript")
#expect(presentation.validation == "Protected text + semantic bounds")
}

@Test
func verbatimMakesFormattingBypassExplicit() {
var settings = LocalAISettings.default
settings.style = .verbatim

let presentation = LocalAIPipelinePresentation(
settings: settings,
operatingSystemVersion: OperatingSystemVersion(
majorVersion: 25,
minorVersion: 0,
patchVersion: 0
)
)

#expect(presentation.speechModel == "SFSpeechRecognizer (OS-managed)")
#expect(presentation.formattingProvider == "None — Verbatim")
#expect(presentation.formattingModel == "Not used")
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,12 @@ struct VoiceHistoryModelTest {
#expect(results[1].sourceResultID == results[0].id)
#expect(results[2].sourceResultID == results[1].id)
#expect(results[1].provider == .appleOnDevice)
#expect(results[1].formattedDocument?.validationStatus == .validated)
#expect(presentation.model.sessions[1].audioExpiredAt != nil)
#expect(
presentation.model.sessions[1].results[1].formattedDocument?
.validationStatus == .sourceFallback
)
#expect(
presentation.model.sessions[1].audioExpirationReason == .byteLimit
)
Expand Down
4 changes: 4 additions & 0 deletions docs/architecture.md
Original file line number Diff line number Diff line change
Expand Up @@ -466,6 +466,10 @@ renderer alone decides whether the captured target receives structure or one
plain line. Validation runs on the canonical rendering before delivery.
The sanitized provider test shares the three-second preparation-plus-generation
deadline; settings changes and shutdown cancel it and suppress stale results.
General settings present speech and formatting provider/model identity as
separate active-pipeline evidence together with typed output, validation, and
fallback behavior. History renders the stored validation status and whether the
Edited fallback was used; it does not infer success from provider identity.

The revision-2 Swift spoken-edit engine recognizes only exact, case-insensitive
command phrases in immutable Raw text, so a Dictionary replacement cannot
Expand Down
2 changes: 2 additions & 0 deletions docs/decisions/0054_typed_local_ai_formatting_output.md
Original file line number Diff line number Diff line change
Expand Up @@ -56,3 +56,5 @@ Model output no longer owns list inference or lowercase safety. A larger model
can improve prose, but it cannot replace deterministic semantics, canonical
validation, or fallback. This decision does not change the ASR provider or
supersede the existing Ollama recommendation.
Settings therefore name ASR and formatting evidence separately, while History
shows the stored validation result and deterministic fallback use.
2 changes: 2 additions & 0 deletions docs/product_brief.md
Original file line number Diff line number Diff line change
Expand Up @@ -213,6 +213,8 @@ Given a focused editable field and a ready selected provider:
- cancellation, process change, secure-status change, focus change, or caret
change discards late model output and stores a typed delivery reason;
- raw and refined recovery controls remain distinct.
- General settings identify speech and formatting providers/models separately;
History identifies validation acceptance and deterministic fallback use.

### Dictation handoff

Expand Down
Loading