diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6dffc9c..0ea65ac 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -38,7 +38,7 @@ jobs: git config user.email "github-actions[bot]@users.noreply.github.com" git add Textream/Textream.xcodeproj/project.pbxproj git commit -m "Bump version to ${{ steps.version.outputs.number }}" || echo "No changes to commit" - git push origin HEAD:refs/heads/main || echo "Push skipped" + git push origin HEAD:refs/heads/master || echo "Push skipped" build: needs: version @@ -146,17 +146,17 @@ jobs: if MacOS.version >= :tahoe sha256 "${SHA256_26}" - url "https://github.com/f/textream/releases/download/${VERSION}/Textream.dmg" + url "https://github.com/f/textream/releases/download/v#{version}/Textream.dmg" else sha256 "${SHA256_15}" - url "https://github.com/f/textream/releases/download/${VERSION}/Textream-macos15.dmg" + url "https://github.com/f/textream/releases/download/v#{version}/Textream-macos15.dmg" end name "Textream" desc "macOS teleprompter that highlights your script in real-time as you speak" homepage "https://github.com/f/textream" - depends_on macos: ">= :sequoia" + depends_on macos: :sequoia app "Textream.app" diff --git a/README.md b/README.md index d4fda41..fae126d 100644 --- a/README.md +++ b/README.md @@ -123,6 +123,19 @@ View your teleprompter on **any device** — phone, tablet, or another computer - **Configurable port** — Default port 7373, adjustable in advanced settings. - **Fully local** — All traffic stays on your local network. Nothing leaves your Wi-Fi. +### Director Mode + +Let someone else control your teleprompter remotely. A director can write, edit, and push scripts to your teleprompter in real time from any browser. + +- **Enable in Settings → Director** — Starts a dedicated HTTP + WebSocket server (default port 7575). +- **Remote web UI** — The director opens a mobile-friendly web page with a full-featured script editor. +- **Live text editing** — The director types or pastes a script, hits Go, and your teleprompter starts immediately with word tracking. +- **Read-locked highlighting** — Already-read text is highlighted and locked in the web editor. Only unread text remains editable. +- **Real-time sync** — Word progress, waveform, mic status, and audio levels stream to the director's browser at 10 Hz. +- **Single-page mode** — Director mode works with a single page of text. Multi-page scripts are not used. +- **Editor disabled** — When director mode is active, the macOS editor is replaced with a QR code overlay so the director has full control. +- **QR code** — Scan or share the QR code from Settings or the editor overlay to connect the director instantly. + ### File Support - **PowerPoint notes import** — Drop a .pptx file to extract presenter notes as pages. For Keynote or Google Slides, export to PowerPoint first. @@ -189,6 +202,7 @@ Textream/ ├── SettingsView.swift # Tabbed settings UI ├── MarqueeTextView.swift # Word flow layout and highlighting ├── BrowserServer.swift # Remote connection HTTP + WebSocket server + ├── DirectorServer.swift # Director mode HTTP + WebSocket server ├── PresentationNotesExtractor.swift # PPTX presenter notes extraction ├── UpdateChecker.swift # GitHub release update checker └── Assets.xcassets/ # App icon and colors @@ -204,6 +218,121 @@ textream://read?text=Hello%20world It also registers as a macOS Service, so you can select text in any app and send it to Textream via the Services menu. +## Director Mode API + +The Director Mode exposes an HTTP server and a WebSocket server on your local network. You can build your own director client using the protocol below. + +### Ports + +| Service | Default Port | Configurable in | +|---|---|---| +| **HTTP** (serves the built-in web UI) | `7575` | Settings → Director → Advanced | +| **WebSocket** (bidirectional communication) | `7576` (HTTP port + 1) | Automatic | + +### Connecting + +1. Open a WebSocket connection to `ws://:` (e.g. `ws://192.168.1.42:7576`). +2. The server immediately begins sending **state frames** as JSON at ~10 Hz once a script is active. +3. Send **command frames** as JSON to control the teleprompter. + +### Commands (Client → App) + +Send JSON messages over the WebSocket: + +#### `setText` — Start reading a new script + +```json +{ + "type": "setText", + "text": "Welcome everyone to today's live stream..." +} +``` + +Replaces the current text, starts word tracking, and opens the teleprompter overlay. This is equivalent to pressing **Go** in the built-in web UI. + +#### `updateText` — Edit unread text while active + +```json +{ + "type": "updateText", + "text": "Welcome everyone to today's live stream We changed the rest of the script...", + "readCharCount": 42 +} +``` + +Updates the full script text while preserving the read position. `readCharCount` is the number of characters already read (locked). Only text after this offset is replaced. Use this for live editing during a read. + +#### `stop` — Stop the teleprompter + +```json +{ + "type": "stop" +} +``` + +Stops word tracking and dismisses the overlay. + +### State (App → Client) + +The server broadcasts a JSON object on every tick (~100 ms): + +```json +{ + "words": ["Welcome", "everyone", "to", "today's", "live", "stream"], + "highlightedCharCount": 24, + "totalCharCount": 120, + "isActive": true, + "isDone": false, + "isListening": true, + "fontColor": "#F5F5F7", + "lastSpokenText": "Welcome everyone to today's", + "audioLevels": [0.12, 0.34, 0.08, ...] +} +``` + +| Field | Type | Description | +|---|---|---| +| `words` | `string[]` | The script split into words (same order as displayed in the overlay). | +| `highlightedCharCount` | `int` | Number of characters recognized so far. Use this to determine the read boundary. | +| `totalCharCount` | `int` | Total character count of the full script. | +| `isActive` | `bool` | `true` when the teleprompter overlay is visible and a script is loaded. | +| `isDone` | `bool` | `true` when `highlightedCharCount >= totalCharCount` (finished reading). | +| `isListening` | `bool` | `true` when the microphone is actively listening. | +| `fontColor` | `string` | CSS color of the text in the overlay (user preference). | +| `lastSpokenText` | `string` | Last recognized speech fragment. | +| `audioLevels` | `double[]` | Array of audio level samples (0.0–1.0) for waveform visualization. | + +When the overlay is not active, the server sends a frame with `isActive: false` and empty arrays. + +### Example: Minimal Python Client + +```python +import asyncio, json, websockets + +async def director(): + async with websockets.connect("ws://192.168.1.42:7576") as ws: + # Send a script + await ws.send(json.dumps({ + "type": "setText", + "text": "Hello everyone, welcome to the show." + })) + + # Listen for state updates + async for msg in ws: + state = json.loads(msg) + pct = 0 + if state["totalCharCount"] > 0: + pct = state["highlightedCharCount"] / state["totalCharCount"] * 100 + print(f"Progress: {pct:.0f}% Done: {state['isDone']}") + if state["isDone"]: + break + + # Stop + await ws.send(json.dumps({"type": "stop"})) + +asyncio.run(director()) +``` + ## License MIT diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..c07437f --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,67 @@ +# Security Policy + +## Supported Versions + +| Version | Supported | +|---------|-----------| +| Latest release | ✅ | +| Older releases | ❌ | + +Only the latest release receives security updates. Please keep Textream up to date. + +## Reporting a Vulnerability + +If you discover a security vulnerability in Textream, **please do not open a public issue.** + +Instead, report it privately: + +- **Email:** [fka@fka.dev](mailto:fka@fka.dev) +- **Subject:** `[SECURITY] Textream — ` + +Please include: + +1. A description of the vulnerability +2. Steps to reproduce the issue +3. Potential impact +4. Suggested fix (if any) + +You should receive an acknowledgment within **48 hours**. Once confirmed, a fix will be prioritized and released as soon as possible. + +## Security Considerations + +### On-Device Processing + +All speech recognition runs locally via Apple's Speech framework. No audio data, transcripts, or scripts are sent to external servers. There are no accounts, analytics, or telemetry. + +### Network Servers + +Textream includes two optional network servers that bind to your **local network only**: + +| Server | Default Port | Purpose | +|--------|-------------|---------| +| **Remote Connection** (BrowserServer) | `8080` | Read-only teleprompter mirror for a browser | +| **Director Mode** (DirectorServer) | `7575` / `7576` | Remote script editing via HTTP + WebSocket | + +**Important:** + +- Both servers are **disabled by default** and must be explicitly enabled in Settings. +- Servers listen on **all local interfaces** (`0.0.0.0`). Anyone on the same network can connect when enabled. +- There is **no authentication** on these servers. Do not enable them on untrusted or public networks. +- The HTTP server serves a single-page web UI. The WebSocket server handles real-time communication. +- Disable the servers when not in use. + +### Permissions + +Textream requests the following macOS permissions: + +- **Microphone** — Required for speech recognition and voice-activated features. +- **Speech Recognition** — Required for on-device word tracking. +- **Local Network** — Required when Remote Connection or Director Mode is enabled. + +No other permissions are requested or required. + +## Recommendations + +- Only enable network servers on trusted private networks. +- Disable Remote Connection and Director Mode when not actively in use. +- Keep Textream updated to the latest version via Homebrew or GitHub Releases. diff --git a/Textream/Textream.xcodeproj/project.pbxproj b/Textream/Textream.xcodeproj/project.pbxproj index df30b53..aa9a555 100644 --- a/Textream/Textream.xcodeproj/project.pbxproj +++ b/Textream/Textream.xcodeproj/project.pbxproj @@ -273,7 +273,7 @@ "$(inherited)", "@executable_path/../Frameworks", ); - MARKETING_VERSION = 1.4.0; + MARKETING_VERSION = 1.6.3; PRODUCT_BUNDLE_IDENTIFIER = dev.fka.textream; PRODUCT_NAME = "$(TARGET_NAME)"; REGISTER_APP_GROUPS = YES; @@ -312,7 +312,7 @@ "$(inherited)", "@executable_path/../Frameworks", ); - MARKETING_VERSION = 1.4.0; + MARKETING_VERSION = 1.6.3; PRODUCT_BUNDLE_IDENTIFIER = dev.fka.textream; PRODUCT_NAME = "$(TARGET_NAME)"; REGISTER_APP_GROUPS = YES; diff --git a/Textream/Textream/BrowserServer.swift b/Textream/Textream/BrowserServer.swift index c4676fa..d874247 100644 --- a/Textream/Textream/BrowserServer.swift +++ b/Textream/Textream/BrowserServer.swift @@ -18,6 +18,7 @@ struct BrowserState: Codable { let isListening: Bool let isDone: Bool let fontColor: String + let cueColor: String let hasNextPage: Bool let isActive: Bool let highlightWords: Bool @@ -186,21 +187,28 @@ class BrowserServer { let charCount: Int let mode = NotchSettings.shared.listeningMode + // Check if scroll already reached the end, to stop advancing the timer + let scrollDone = totalCharCount > 0 && charOffsetForWordProgress(timerWordProgress) >= totalCharCount switch mode { case .wordTracking: charCount = speechRecognizer?.recognizedCharCount ?? 0 case .classic: - timerWordProgress += NotchSettings.shared.scrollSpeed * 0.1 + if !scrollDone { + timerWordProgress += NotchSettings.shared.scrollSpeed * 0.1 + } charCount = charOffsetForWordProgress(timerWordProgress) case .silencePaused: - if speechRecognizer?.isListening == true && (speechRecognizer?.isSpeaking ?? false) { + if !scrollDone && speechRecognizer?.isListening == true && (speechRecognizer?.isSpeaking ?? false) { timerWordProgress += NotchSettings.shared.scrollSpeed * 0.1 } charCount = charOffsetForWordProgress(timerWordProgress) } let effective = min(charCount, totalCharCount) - let isDone = totalCharCount > 0 && effective >= totalCharCount + let rawDone = totalCharCount > 0 && effective >= totalCharCount + // In classic/silence-paused modes on the last page, suppress Done so the + // browser keeps showing the prompter text (speaker may still be talking). + let isDone = rawDone && (mode == .wordTracking || hasNextPage) let highlightWords = mode == .wordTracking @@ -212,6 +220,7 @@ class BrowserServer { isListening: speechRecognizer?.isListening ?? false, isDone: isDone, fontColor: NotchSettings.shared.fontColorPreset.cssColor, + cueColor: NotchSettings.shared.cueColorPreset.cssColor, hasNextPage: hasNextPage, isActive: true, highlightWords: highlightWords, @@ -224,7 +233,7 @@ class BrowserServer { let state = BrowserState( words: [], highlightedCharCount: 0, totalCharCount: 0, audioLevels: [], isListening: false, isDone: false, - fontColor: "#ffffff", hasNextPage: false, isActive: false, + fontColor: "#ffffff", cueColor: "#ffffff", hasNextPage: false, isActive: false, highlightWords: true, lastSpokenText: "" ) broadcast(state) @@ -416,18 +425,35 @@ class BrowserServer { } function rgba(rgb,a){return 'rgba('+rgb[0]+','+rgb[1]+','+rgb[2]+','+a+')';} + // Any letter or digit, in any script. An explicit range list silently + // demotes every unlisted script (Hebrew, Arabic, Greek, Thai…) to an + // annotation, so those words render as dim cues and never highlight. + const WORDY=/[\\p{L}\\p{N}]/u; + + // Scripts written right-to-left + const RTL_CHAR=/[\\p{Script=Hebrew}\\p{Script=Arabic}\\p{Script=Syriac}\\p{Script=Thaana}\\p{Script=Nko}\\p{Script=Samaritan}\\p{Script=Mandaic}\\p{Script=Adlam}]/gu; + // Detect annotation words: [bracket] or emoji-only (no letters/digits) function isAnnotation(w){ if(w.startsWith('[')&&w.endsWith(']'))return true; - return!/[a-zA-Z0-9\\u00C0-\\u024F\\u0400-\\u04FF\\u3000-\\u9FFF\\uAC00-\\uD7AF]/.test(w); + return!WORDY.test(w); } // Count letters+digits in a word function letterCount(w){ - let n=0;for(const ch of w)if(/[a-zA-Z0-9\\u00C0-\\u024F\\u0400-\\u04FF\\u3000-\\u9FFF\\uAC00-\\uD7AF]/.test(ch))n++; + let n=0;for(const ch of w)if(WORDY.test(ch))n++; return Math.max(1,n); } + // Base direction for the script: whichever side has more letters wins, so + // a Hebrew script opening with a Latin product name still reads right-to-left. + function baseDirection(words){ + const s=words.join(' '); + const rtl=(s.match(RTL_CHAR)||[]).length; + const letters=(s.match(/\\p{L}/gu)||[]).length; + return rtl>letters-rtl?'rtl':'ltr'; + } + /* ---- connection ---- */ function connect(){ @@ -457,7 +483,9 @@ class BrowserServer { const c=document.getElementById('text-container'), words=s.words||[], fc=s.fontColor||'#ffffff', + cc=s.cueColor||fc, rgb=parseColor(fc), + crgb=parseColor(cc), hlWords=s.highlightWords!==false, hcc=s.highlightedCharCount||0; @@ -465,6 +493,11 @@ class BrowserServer { const wordKey=words.length+'|'+(words[0]||'')+'|'+(words[words.length-1]||''); if(wordKey!==prevWordKey){ c.innerHTML=''; + // Let the browser's bidi engine order the line; it keeps embedded + // left-to-right runs (names, numbers) in reading order. + const dir=baseDirection(words); + c.setAttribute('dir',dir); + c.style.textAlign=dir==='rtl'?'right':'left'; let cp=0; for(let i=0;i? @FocusState private var isTextFocused: Bool private let defaultText = """ @@ -59,6 +61,89 @@ Happy presenting! [wave] service.pages.contains { !$0.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty } } + private var isRecording: Bool { + dictation.isRecording || dictation.isStarting + } + + private func scheduleLanguageDetection(for text: String) { + languageDetectionTask?.cancel() + languageSuggestion = nil + let pageIndex = service.currentPageIndex + let localeIdentifier = NotchSettings.shared.speechLocale + + languageDetectionTask = Task { @MainActor in + do { + try await Task.sleep(nanoseconds: 2_500_000_000) + } catch { + return + } + guard !Task.isCancelled, + service.currentPageIndex == pageIndex, + service.currentPageText == text, + NotchSettings.shared.speechLocale == localeIdentifier else { return } + + let suggestion = SpeechLanguageDetector.suggestion( + for: text, + currentLocaleIdentifier: localeIdentifier + ) + guard !Task.isCancelled, + suggestion?.detectedLanguageIdentifier != ignoredLanguageIdentifier else { return } + withAnimation(.easeInOut(duration: 0.2)) { + languageSuggestion = suggestion + } + } + } + + private func languageSuggestionBanner(_ suggestion: SpeechLanguageSuggestion) -> some View { + HStack(spacing: 10) { + Image(systemName: "character.bubble.fill") + .foregroundStyle(Color.accentColor) + + VStack(alignment: .leading, spacing: 2) { + Text("This text looks like \(suggestion.languageName).") + .font(.system(size: 12, weight: .semibold)) + Text("Speech is currently set to \(languageLabel).") + .font(.system(size: 11)) + .foregroundStyle(.secondary) + } + + Spacer(minLength: 8) + + Button("Use \(suggestion.languageName)") { + if isRecording { + stopRecording() + } + ignoredLanguageIdentifier = nil + languageSuggestion = nil + NotchSettings.shared.speechLocale = suggestion.localeIdentifier + } + .buttonStyle(.borderedProminent) + .controlSize(.small) + .help("Switch speech recognition to \(suggestion.localeName)") + + Button { + ignoredLanguageIdentifier = suggestion.detectedLanguageIdentifier + withAnimation(.easeInOut(duration: 0.2)) { + languageSuggestion = nil + } + } label: { + Image(systemName: "xmark") + } + .buttonStyle(.plain) + .foregroundStyle(.secondary) + .help("Dismiss language suggestion") + } + .padding(.horizontal, 12) + .padding(.vertical, 9) + .background(.regularMaterial, in: RoundedRectangle(cornerRadius: 10)) + .overlay { + RoundedRectangle(cornerRadius: 10) + .stroke(Color.accentColor.opacity(0.2)) + } + .padding(.horizontal, 20) + .padding(.top, 8) + } + @ViewBuilder private var waveformPill: some View { let pill = AudioWaveformView(levels: dictation.audioLevels, color: .red) @@ -172,7 +257,6 @@ Happy presenting! [wave] } } dictation.start() - isRecording = true } private func stopRecording() { @@ -182,11 +266,15 @@ Happy presenting! [wave] dictation.stop() dictation.onTextUpdate = nil dictation.onNewSegment = nil - isRecording = false } private var mainContent: some View { VStack(spacing: 0) { + if let languageSuggestion { + languageSuggestionBanner(languageSuggestion) + .transition(.move(edge: .top).combined(with: .opacity)) + } + ZStack { HighlightingTextEditor( text: currentText, @@ -224,7 +312,7 @@ Happy presenting! [wave] Spacer() ZStack { // Waveform pill centered to full width - if isRecording { + if dictation.isRecording { waveformPill .transition(.scale(scale: 0.8).combined(with: .opacity)) } @@ -240,13 +328,21 @@ Happy presenting! [wave] startRecording() } } label: { - Image(systemName: isRecording ? "pause.fill" : "mic.fill") - .font(.system(size: 16, weight: .semibold)) - .foregroundStyle(.white) - .frame(width: 44, height: 44) - .background(isRecording ? Color.orange : Color.red) - .clipShape(Circle()) - .shadow(color: .black.opacity(0.2), radius: 8, y: 4) + Group { + if dictation.isStarting { + ProgressView() + .controlSize(.small) + .tint(.white) + } else { + Image(systemName: isRecording ? "pause.fill" : "mic.fill") + .font(.system(size: 16, weight: .semibold)) + } + } + .foregroundStyle(.white) + .frame(width: 44, height: 44) + .background(isRecording ? Color.orange : Color.red) + .clipShape(Circle()) + .shadow(color: .black.opacity(0.2), radius: 8, y: 4) } .buttonStyle(.plain) .disabled(isRunning) @@ -408,15 +504,13 @@ Happy presenting! [wave] Group { if NotchSettings.shared.directorModeEnabled { directorOverlay - } else if service.pages.count > 1 { + } else { NavigationSplitView { pageSidebar } detail: { mainContent } .navigationSplitViewColumnWidth(min: 160, ideal: 200, max: 260) - } else { - mainContent } } .alert(dropAlertTitle, isPresented: Binding(get: { dropError != nil }, set: { if !$0 { dropError = nil } })) { @@ -424,12 +518,23 @@ Happy presenting! [wave] } message: { Text(dropError ?? "") } + .alert("Microphone Unavailable", isPresented: Binding( + get: { dictation.error != nil }, + set: { if !$0 { dictation.error = nil } } + )) { + Button("OK") { dictation.error = nil } + } message: { + Text(dictation.error ?? "") + } .frame(minWidth: 360, minHeight: 240) .background(.ultraThinMaterial) .toolbar { ToolbarItem(placement: .automatic) { HStack(spacing: 8) { Button { + if isRecording { + stopRecording() + } service.openFile() } label: { HStack(spacing: 4) { @@ -448,6 +553,9 @@ Happy presenting! [wave] // Add page button in toolbar Button { + if isRecording { + stopRecording() + } withAnimation(.easeInOut(duration: 0.2)) { service.pages.append("") service.currentPageIndex = service.pages.count - 1 @@ -498,6 +606,29 @@ Happy presenting! [wave] // Sync button state when app is re-activated (e.g. dock click) isRunning = service.overlayController.isShowing } + .onChange(of: service.currentPageText, initial: true) { _, text in + scheduleLanguageDetection(for: text) + } + .onChange(of: service.currentPageIndex) { _, _ in + if isRecording { + stopRecording() + } + ignoredLanguageIdentifier = nil + scheduleLanguageDetection(for: service.currentPageText) + } + .onChange(of: NotchSettings.shared.speechLocale) { _, _ in + if isRecording { + stopRecording() + } + ignoredLanguageIdentifier = nil + scheduleLanguageDetection(for: service.currentPageText) + } + .onDisappear { + languageDetectionTask?.cancel() + if isRecording { + stopRecording() + } + } .onAppear { // Set default text for the first page if empty if service.pages.count == 1 && service.pages[0].isEmpty { @@ -534,10 +665,11 @@ Happy presenting! [wave] get: { service.currentPageIndex }, set: { newValue in if let index = newValue { - DispatchQueue.main.async { - withAnimation(.easeInOut(duration: 0.15)) { - service.currentPageIndex = index - } + if isRecording { + stopRecording() + } + withAnimation(.easeInOut(duration: 0.15)) { + service.currentPageIndex = index } } } @@ -575,6 +707,9 @@ Happy presenting! [wave] .listStyle(.sidebar) .safeAreaInset(edge: .bottom) { Button { + if isRecording { + stopRecording() + } withAnimation(.easeInOut(duration: 0.2)) { service.pages.append("") service.currentPageIndex = service.pages.count - 1 @@ -595,6 +730,9 @@ Happy presenting! [wave] private func removePage(at index: Int) { guard service.pages.count > 1 else { return } + if isRecording { + stopRecording() + } withAnimation(.easeInOut(duration: 0.2)) { service.pages.remove(at: index) if service.currentPageIndex >= service.pages.count { @@ -616,7 +754,13 @@ Happy presenting! [wave] NSApp.windows.first?.makeKeyAndOrderFront(nil) } service.readPages.removeAll() - service.currentPageIndex = 0 + // If the current page is empty, find the first non-empty page + let currentText = service.currentPageText.trimmingCharacters(in: .whitespacesAndNewlines) + if currentText.isEmpty { + if let firstNonEmpty = service.pages.firstIndex(where: { !$0.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty }) { + service.currentPageIndex = firstNonEmpty + } + } service.readCurrentPage() isRunning = true } @@ -625,6 +769,9 @@ Happy presenting! [wave] private func handlePresentationDrop(url: URL) { guard service.confirmDiscardIfNeeded() else { return } + if isRecording { + stopRecording() + } isImporting = true DispatchQueue.global(qos: .userInitiated).async { diff --git a/Textream/Textream/DictationManager.swift b/Textream/Textream/DictationManager.swift index b97b496..d80db7e 100644 --- a/Textream/Textream/DictationManager.swift +++ b/Textream/Textream/DictationManager.swift @@ -13,6 +13,7 @@ import AppKit @Observable class DictationManager { var isRecording: Bool = false + var isStarting: Bool = false var audioLevels: [CGFloat] = Array(repeating: 0, count: 40) var error: String? @@ -27,68 +28,114 @@ class DictationManager { private var audioEngine = AVAudioEngine() private var configurationChangeObserver: Any? private var suppressConfigChange: Bool = false + private var pendingRestart: DispatchWorkItem? + private var requestLock = NSLock() + private var recognitionGeneration: Int = 0 + private var shouldRecord: Bool = false + private var retryCount: Int = 0 + private let maxRetries: Int = 10 // Tracks the committed text from previous recognition segments private var committedText: String = "" private var sessionGeneration: Int = 0 func start() { - guard !isRecording else { return } + guard !isRecording, !isStarting else { return } cleanup() committedText = "" - sessionGeneration += 1 + sessionGeneration &+= 1 + retryCount = 0 error = nil + shouldRecord = true + isStarting = true + requestMicrophoneAccessAndBegin(for: sessionGeneration) + } + + func stop() { + shouldRecord = false + sessionGeneration &+= 1 + isRecording = false + isStarting = false + cleanup() + } + + private func requestMicrophoneAccessAndBegin(for generation: Int) { + guard shouldRecord, sessionGeneration == generation else { return } // Check microphone permission switch AVCaptureDevice.authorizationStatus(for: .audio) { case .denied, .restricted: - error = "Microphone access denied. Open System Settings → Privacy & Security → Microphone." - return + fail("Microphone access denied. Open System Settings → Privacy & Security → Microphone.") + openMicrophoneSettings() case .notDetermined: AVCaptureDevice.requestAccess(for: .audio) { [weak self] granted in DispatchQueue.main.async { + guard let self, + self.shouldRecord, + self.sessionGeneration == generation else { return } if granted { - self?.requestSpeechAuthAndBegin() + self.requestSpeechAuthAndBegin(for: generation) } else { - self?.error = "Microphone access denied." + self.fail("Microphone access denied. Open System Settings → Privacy & Security → Microphone.") } } } - return case .authorized: - break + requestSpeechAuthAndBegin(for: generation) @unknown default: - break + fail("Microphone authorization is unavailable.") } - - requestSpeechAuthAndBegin() } - func stop() { - isRecording = false - cleanup() - } - - private func requestSpeechAuthAndBegin() { + private func requestSpeechAuthAndBegin(for generation: Int) { SFSpeechRecognizer.requestAuthorization { [weak self] status in DispatchQueue.main.async { + guard let self, + self.shouldRecord, + self.sessionGeneration == generation else { return } switch status { case .authorized: - self?.beginRecognition() + self.beginRecognition() default: - self?.error = "Speech recognition not authorized." + self.fail("Speech recognition not authorized. Open System Settings → Privacy & Security → Speech Recognition.") + self.openSpeechRecognitionSettings() } } } } + private func openMicrophoneSettings() { + if let url = URL(string: "x-apple.systempreferences:com.apple.preference.security?Privacy_Microphone") { + NSWorkspace.shared.open(url) + } + } + + private func openSpeechRecognitionSettings() { + if let url = URL(string: "x-apple.systempreferences:com.apple.preference.security?Privacy_SpeechRecognition") { + NSWorkspace.shared.open(url) + } + } + + private func fail(_ message: String) { + shouldRecord = false + isRecording = false + isStarting = false + error = message + cleanup() + } + private func cleanup() { + recognitionGeneration &+= 1 + pendingRestart?.cancel() + pendingRestart = nil if let observer = configurationChangeObserver { NotificationCenter.default.removeObserver(observer) configurationChangeObserver = nil } + requestLock.lock() recognitionRequest?.endAudio() recognitionRequest = nil + requestLock.unlock() recognitionTask?.cancel() recognitionTask = nil if audioEngine.isRunning { @@ -98,9 +145,13 @@ class DictationManager { } private func beginRecognition() { + guard shouldRecord else { return } + let expectedSessionGeneration = sessionGeneration cleanup() + guard shouldRecord, sessionGeneration == expectedSessionGeneration else { return } audioEngine = AVAudioEngine() + suppressConfigChange = false // Set selected microphone if configured let micUID = NotchSettings.shared.selectedMicUID @@ -120,42 +171,72 @@ class DictationManager { AudioUnitInitialize(audioUnit) } DispatchQueue.main.asyncAfter(deadline: .now() + 1.0) { [weak self] in - self?.suppressConfigChange = false + guard let self, + self.sessionGeneration == expectedSessionGeneration else { return } + self.suppressConfigChange = false } } speechRecognizer = SFSpeechRecognizer(locale: Locale(identifier: NotchSettings.shared.speechLocale)) - guard let speechRecognizer, speechRecognizer.isAvailable else { - error = "Speech recognizer not available" + guard let speechRecognizer else { + fail("Speech recognition isn't supported for the selected language.") + return + } + guard speechRecognizer.isAvailable else { + if retryCount < maxRetries { + retryCount += 1 + scheduleRestart(after: 0.5) + } else { + fail("Speech recognizer is not available.") + } return } recognitionRequest = SFSpeechAudioBufferRecognitionRequest() - guard let recognitionRequest else { return } + guard let recognitionRequest else { + fail("Unable to create a speech recognition request.") + return + } recognitionRequest.shouldReportPartialResults = true + recognitionRequest.taskHint = .dictation let inputNode = audioEngine.inputNode let recordingFormat = inputNode.outputFormat(forBus: 0) guard recordingFormat.sampleRate > 0, recordingFormat.channelCount > 0 else { - error = "Audio input unavailable" + if retryCount < maxRetries { + retryCount += 1 + scheduleRestart(after: 0.5) + } else { + fail("Audio input is unavailable.") + } return } + let monoFormat = AVAudioFormat( + commonFormat: recordingFormat.commonFormat, + sampleRate: recordingFormat.sampleRate, + channels: 1, + interleaved: recordingFormat.isInterleaved + ) + let tapFormat = recordingFormat.channelCount > 1 ? monoFormat : recordingFormat + // Observe audio configuration changes configurationChangeObserver = NotificationCenter.default.addObserver( forName: .AVAudioEngineConfigurationChange, object: audioEngine, queue: .main ) { [weak self] _ in - guard let self, !self.suppressConfigChange, self.isRecording else { return } + guard let self, + self.shouldRecord, + !self.suppressConfigChange else { return } self.restartRecognition() } inputNode.removeTap(onBus: 0) - inputNode.installTap(onBus: 0, bufferSize: 1024, format: nil) { [weak self] buffer, _ in - recognitionRequest.append(buffer) + inputNode.installTap(onBus: 0, bufferSize: 1024, format: tapFormat) { [weak self] buffer, _ in + self?.appendBuffer(buffer) guard let channelData = buffer.floatChannelData?[0] else { return } let frameLength = Int(buffer.frameLength) @@ -174,22 +255,29 @@ class DictationManager { } } - // Notify that a new recognition segment is starting - onNewSegment?() - + recognitionGeneration &+= 1 + let currentRecognitionGeneration = recognitionGeneration let currentGeneration = sessionGeneration recognitionTask = speechRecognizer.recognitionTask(with: recognitionRequest) { [weak self] result, error in guard let self else { return } if let result { let spoken = result.bestTranscription.formattedString DispatchQueue.main.async { - guard self.sessionGeneration == currentGeneration else { return } + guard self.sessionGeneration == currentGeneration, + self.recognitionGeneration == currentRecognitionGeneration else { return } + self.retryCount = 0 self.onTextUpdate?(spoken) + if result.isFinal { + self.restartRecognition() + } } } if error != nil { DispatchQueue.main.async { - guard self.recognitionRequest != nil, self.isRecording else { return } + guard self.sessionGeneration == currentGeneration, + self.recognitionGeneration == currentRecognitionGeneration, + self.recognitionRequest != nil, + self.shouldRecord else { return } self.restartRecognition() } } @@ -198,19 +286,51 @@ class DictationManager { do { audioEngine.prepare() try audioEngine.start() + guard shouldRecord, sessionGeneration == expectedSessionGeneration else { + cleanup() + return + } + retryCount = 0 + error = nil + isStarting = false isRecording = true + // Notify that a new recognition segment is starting + onNewSegment?() } catch { - self.error = "Audio engine failed: \(error.localizedDescription)" - isRecording = false + if retryCount < maxRetries { + retryCount += 1 + scheduleRestart(after: 0.5) + } else { + fail("Audio engine failed: \(error.localizedDescription)") + } } } + private func appendBuffer(_ buffer: AVAudioPCMBuffer) { + requestLock.lock() + recognitionRequest?.append(buffer) + requestLock.unlock() + } + private func restartRecognition() { - guard isRecording else { return } + guard shouldRecord else { return } + isRecording = false + isStarting = true cleanup() - DispatchQueue.main.asyncAfter(deadline: .now() + 0.3) { [weak self] in - guard let self, self.isRecording else { return } + scheduleRestart(after: 0.3) + } + + private func scheduleRestart(after delay: TimeInterval) { + pendingRestart?.cancel() + let expectedSessionGeneration = sessionGeneration + let work = DispatchWorkItem { [weak self] in + guard let self, + self.shouldRecord, + self.sessionGeneration == expectedSessionGeneration else { return } + self.pendingRestart = nil self.beginRecognition() } + pendingRestart = work + DispatchQueue.main.asyncAfter(deadline: .now() + delay, execute: work) } } diff --git a/Textream/Textream/DirectorServer.swift b/Textream/Textream/DirectorServer.swift index 3515619..a5dc207 100644 --- a/Textream/Textream/DirectorServer.swift +++ b/Textream/Textream/DirectorServer.swift @@ -18,6 +18,7 @@ struct DirectorState: Codable { let isDone: Bool let isListening: Bool let fontColor: String + let cueColor: String let lastSpokenText: String let audioLevels: [Double] } @@ -36,13 +37,23 @@ class DirectorServer { private var httpListener: NWListener? private var wsListener: NWListener? private var wsConnections: [NWConnection] = [] + private var authenticatedConnections: Set = [] private var broadcastTimer: Timer? + // Connection limit to prevent resource exhaustion (CWE-400) + private let maxConnections = 5 + + // Dedicated queue for broadcasting to avoid blocking the main/UI thread + private let broadcastQueue = DispatchQueue(label: "com.textream.director.broadcast") + // Security: shared secret token for WebSocket authentication + private var authToken: String = "" + // Content state private var words: [String] = [] private var totalCharCount: Int = 0 private weak var speechRecognizer: SpeechRecognizer? private var contentActive: Bool = false + private var lastBroadcastState: Data? // Callbacks var onSetText: ((String) -> Void)? @@ -58,6 +69,7 @@ class DirectorServer { func start() { stop() + authToken = Self.generateToken() startHTTPListener() startWSListener() } @@ -73,6 +85,7 @@ class DirectorServer { for conn in wsConnections { conn.cancel() } wsConnections.removeAll() + authenticatedConnections.removeAll() contentActive = false } @@ -129,9 +142,9 @@ class DirectorServer { } private func buildHTTPResponse() -> Data { - let html = Self.generateHTML(wsPort: wsPort) + let html = Self.generateHTML(wsPort: wsPort, authToken: authToken) let body = Data(html.utf8) - let header = "HTTP/1.1 200 OK\r\nContent-Type: text/html; charset=utf-8\r\nContent-Length: \(body.count)\r\nCache-Control: no-cache\r\nConnection: close\r\n\r\n" + let header = "HTTP/1.1 200 OK\r\nContent-Type: text/html; charset=utf-8\r\nContent-Length: \(body.count)\r\nCache-Control: no-store\r\nConnection: close\r\n\r\n" return Data(header.utf8) + body } @@ -157,14 +170,28 @@ class DirectorServer { } private func handleWSConnection(_ conn: NWConnection) { + guard wsConnections.count < maxConnections else { + conn.cancel() + return + } conn.start(queue: .main) wsConnections.append(conn) receiveWSMessage(conn) + // Auto-disconnect unauthenticated connections after 5 seconds + let connId = ObjectIdentifier(conn) + DispatchQueue.main.asyncAfter(deadline: .now() + 5) { [weak self] in + guard let self else { return } + if !self.authenticatedConnections.contains(connId) { + conn.cancel() + } + } + conn.stateUpdateHandler = { [weak self] state in switch state { case .failed, .cancelled: self?.wsConnections.removeAll { $0 === conn } + self?.authenticatedConnections.remove(ObjectIdentifier(conn)) default: break } } @@ -174,33 +201,54 @@ class DirectorServer { conn.receiveMessage { [weak self] data, _, _, error in if error != nil { conn.cancel(); return } if let data { - self?.handleIncomingMessage(data) + self?.handleIncomingMessage(data, from: conn) } self?.receiveWSMessage(conn) } } - private func handleIncomingMessage(_ data: Data) { + private func handleIncomingMessage(_ data: Data, from conn: NWConnection) { guard let command = try? JSONDecoder().decode(DirectorCommand.self, from: data) else { return } + let connId = ObjectIdentifier(conn) DispatchQueue.main.async { [weak self] in + guard let self else { return } + + // First message must be authentication + if !self.authenticatedConnections.contains(connId) { + if command.type == "auth", command.text == self.authToken { + self.authenticatedConnections.insert(connId) + } else { + conn.cancel() + } + return + } + switch command.type { case "setText": if let text = command.text { - self?.onSetText?(text) + self.onSetText?(text) } case "updateText": if let text = command.text, let readCharCount = command.readCharCount { - self?.onUpdateText?(text, readCharCount) + self.onUpdateText?(text, readCharCount) } case "stop": - self?.onStop?() + self.onStop?() default: break } } } + // MARK: - Token Generation + + private static func generateToken() -> String { + var bytes = [UInt8](repeating: 0, count: 32) + _ = SecRandomCopyBytes(kSecRandomDefault, bytes.count, &bytes) + return bytes.map { String(format: "%02x", $0) }.joined() + } + // MARK: - Broadcasting private func startBroadcasting() { @@ -225,6 +273,7 @@ class DirectorServer { isDone: isDone, isListening: speechRecognizer?.isListening ?? false, fontColor: NotchSettings.shared.fontColorPreset.cssColor, + cueColor: NotchSettings.shared.cueColorPreset.cssColor, lastSpokenText: speechRecognizer?.lastSpokenText ?? "", audioLevels: (speechRecognizer?.audioLevels ?? []).map { Double($0) } ) @@ -235,7 +284,7 @@ class DirectorServer { let state = DirectorState( words: [], highlightedCharCount: 0, totalCharCount: 0, isActive: false, isDone: false, isListening: false, - fontColor: "#ffffff", lastSpokenText: "", + fontColor: "#ffffff", cueColor: "#ffffff", lastSpokenText: "", audioLevels: [] ) broadcast(state) @@ -243,16 +292,26 @@ class DirectorServer { private func broadcast(_ state: DirectorState) { guard !wsConnections.isEmpty, let data = try? JSONEncoder().encode(state) else { return } + + // Skip broadcast if state hasn't changed + if let last = lastBroadcastState, last == data { return } + lastBroadcastState = data + + let connections = wsConnections.filter { authenticatedConnections.contains(ObjectIdentifier($0)) } + guard !connections.isEmpty else { return } let meta = NWProtocolWebSocket.Metadata(opcode: .text) let ctx = NWConnection.ContentContext(identifier: "ws", metadata: [meta]) - for conn in wsConnections { - conn.send(content: data, contentContext: ctx, completion: .idempotent) + + broadcastQueue.async { + for conn in connections { + conn.send(content: data, contentContext: ctx, completion: .idempotent) + } } } // MARK: - HTML Template - static func generateHTML(wsPort: UInt16) -> String { + static func generateHTML(wsPort: UInt16, authToken: String) -> String { """ @@ -360,9 +419,9 @@ class DirectorServer {
-
+
-
+
@@ -379,13 +438,14 @@ class DirectorServer { +

Up and running in seconds