From 7cdc6a3f9e0f32a028cd7264d23cc1d462e283da Mon Sep 17 00:00:00 2001 From: John Wang Date: Mon, 2 Nov 2026 07:51:36 -0800 Subject: [PATCH 01/16] feat(speech): add @plexusone/speech for browser speech in and out speak and listen wrap the Web Speech API with the handling browsers need: long text split into short utterances, a cancel before new speech and a retry when Chrome accepts speech but never starts it, pausing without the browser's pause, and continuous listening restarted until stopped. React hooks (useSpeech, useListen, useVoices) are in @plexusone/speech/react, and a standalone build exposes window.PlexusSpeech. Refs: RMI-SYSTEMFORGEWEB-031 Co-Authored-By: Claude Opus 5.5 --- packages/speech/package.json | 65 +++++ packages/speech/src/errors.ts | 16 ++ packages/speech/src/index.ts | 32 +++ packages/speech/src/react.ts | 199 +++++++++++++ packages/speech/src/recognition.ts | 224 +++++++++++++++ packages/speech/src/synthesis.ts | 430 +++++++++++++++++++++++++++++ packages/speech/src/text.ts | 65 +++++ packages/speech/tsconfig.json | 9 + packages/speech/tsup.config.ts | 23 ++ pnpm-lock.yaml | 16 ++ 10 files changed, 1079 insertions(+) create mode 100644 packages/speech/package.json create mode 100644 packages/speech/src/errors.ts create mode 100644 packages/speech/src/index.ts create mode 100644 packages/speech/src/react.ts create mode 100644 packages/speech/src/recognition.ts create mode 100644 packages/speech/src/synthesis.ts create mode 100644 packages/speech/src/text.ts create mode 100644 packages/speech/tsconfig.json create mode 100644 packages/speech/tsup.config.ts diff --git a/packages/speech/package.json b/packages/speech/package.json new file mode 100644 index 0000000..0942923 --- /dev/null +++ b/packages/speech/package.json @@ -0,0 +1,65 @@ +{ + "name": "@plexusone/speech", + "version": "0.1.0", + "description": "Browser text-to-speech and speech-to-text, with React hooks and a standalone build", + "homepage": "https://plexusone.github.io/systemforge-web/packages/speech/", + "main": "./dist/index.js", + "module": "./dist/index.mjs", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.mjs", + "require": "./dist/index.js" + }, + "./react": { + "types": "./dist/react.d.ts", + "import": "./dist/react.mjs", + "require": "./dist/react.js" + }, + "./browser": "./dist/speech.global.js" + }, + "files": [ + "dist" + ], + "sideEffects": false, + "publishConfig": { + "access": "public" + }, + "scripts": { + "build": "tsup", + "dev": "tsup --watch", + "lint": "eslint src/", + "typecheck": "tsc --noEmit" + }, + "peerDependencies": { + "react": "^19.0.0" + }, + "peerDependenciesMeta": { + "react": { + "optional": true + } + }, + "devDependencies": { + "@types/react": "^19.0.0", + "tsup": "^8.0.1", + "typescript": "^5.3.3" + }, + "keywords": [ + "systemforge", + "speech", + "text-to-speech", + "speech-to-text", + "web-speech-api" + ], + "author": "SystemForge Team", + "license": "MIT", + "repository": { + "type": "git", + "url": "https://github.com/plexusone/systemforge-web.git", + "directory": "packages/speech" + }, + "bugs": { + "url": "https://github.com/plexusone/systemforge-web/issues" + } +} diff --git a/packages/speech/src/errors.ts b/packages/speech/src/errors.ts new file mode 100644 index 0000000..d653768 --- /dev/null +++ b/packages/speech/src/errors.ts @@ -0,0 +1,16 @@ +/** + * A speech failure. `code` is `unsupported` when the browser lacks the API, + * otherwise the browser's own error code, such as `not-allowed` (no + * permission, or speech started without a user gesture), `audio-capture` + * (no microphone) or `synthesis-failed`. `start-timeout` means the browser + * accepted speech but never started it, even after a retry. + */ +export class SpeechError extends Error { + readonly code: string; + + constructor(code: string, message?: string) { + super(message ?? `speech error: ${code}`); + this.name = 'SpeechError'; + this.code = code; + } +} diff --git a/packages/speech/src/index.ts b/packages/speech/src/index.ts new file mode 100644 index 0000000..8383038 --- /dev/null +++ b/packages/speech/src/index.ts @@ -0,0 +1,32 @@ +export { SpeechError } from './errors'; +export { DEFAULT_MAX_CHUNK_CHARS, splitText } from './text'; +export { + getVoices, + isSynthesisSupported, + langMatches, + pickVoice, + speak, + stopSpeaking, +} from './synthesis'; +export type { + GetVoicesOptions, + SpeakInput, + SpeakOptions, + SpeakResult, + SpeakSegment, + Speaking, + SpeechState, + SynthesisEnv, + SynthesisLike, + UtteranceConstructor, +} from './synthesis'; +export { isRecognitionSupported, listen } from './recognition'; +export type { + ListenOptions, + ListenResult, + Listening, + RecognitionErrorEventLike, + RecognitionFactory, + RecognitionLike, + RecognitionResultEventLike, +} from './recognition'; diff --git a/packages/speech/src/react.ts b/packages/speech/src/react.ts new file mode 100644 index 0000000..5b7ff47 --- /dev/null +++ b/packages/speech/src/react.ts @@ -0,0 +1,199 @@ +import { useCallback, useEffect, useRef, useState } from 'react'; +import { SpeechError } from './errors'; +import { isRecognitionSupported, listen } from './recognition'; +import type { ListenOptions, Listening } from './recognition'; +import { getVoices, isSynthesisSupported, speak } from './synthesis'; +import type { + SpeakInput, + SpeakOptions, + SpeakResult, + Speaking, + SpeechState, + SynthesisEnv, +} from './synthesis'; + +/** The browser's voices, optionally for one language; empty until loaded. */ +export function useVoices(lang?: string, env: SynthesisEnv = {}): SpeechSynthesisVoice[] { + const [voices, setVoices] = useState([]); + const { synthesis } = env; + useEffect(() => { + let live = true; + getVoices({ lang, synthesis }).then((vs) => { + if (live) setVoices(vs); + }); + return () => { + live = false; + }; + }, [lang, synthesis]); + return voices; +} + +export interface UseSpeechResult { + /** Speaks, stopping anything already speaking. Options add to the hook's. */ + speak: (input: SpeakInput, options?: SpeakOptions) => Promise; + stop: () => void; + pause: () => void; + resume: () => void; + /** `idle` before the first speech. */ + state: SpeechState | 'idle'; + /** The segment being spoken, or -1. */ + segment: number; + error: SpeechError | undefined; + supported: boolean; +} + +/** Text-to-speech in a component. Speech stops when the component unmounts. */ +export function useSpeech(defaults: SpeakOptions = {}): UseSpeechResult { + const [state, setState] = useState('idle'); + const [segment, setSegment] = useState(-1); + const [error, setError] = useState(); + const handle = useRef(undefined); + // Identifies the latest speech, so callbacks from earlier speech are ignored. + const seq = useRef(0); + const defaultsRef = useRef(defaults); + useEffect(() => { + defaultsRef.current = defaults; + }); + + useEffect(() => () => handle.current?.stop(), []); + + const run = useCallback((input: SpeakInput, options: SpeakOptions = {}) => { + const merged = { ...defaultsRef.current, ...options }; + const id = ++seq.current; + const latest = () => seq.current === id; + setError(undefined); + setSegment(-1); + const h = speak(input, { + ...merged, + onSegment: (i, seg) => { + if (latest()) setSegment(i); + merged.onSegment?.(i, seg); + }, + }); + handle.current = h; + setState(h.state); + return h.done.then( + (r) => { + if (latest()) { + setState(r.status); + setSegment(-1); + } + return r; + }, + (e: SpeechError) => { + if (latest()) { + setState('error'); + setSegment(-1); + setError(e); + } + throw e; + } + ); + }, []); + + const stop = useCallback(() => handle.current?.stop(), []); + const pause = useCallback(() => { + handle.current?.pause(); + if (handle.current) setState(handle.current.state); + }, []); + const resume = useCallback(() => { + handle.current?.resume(); + if (handle.current) setState(handle.current.state); + }, []); + + return { + speak: run, + stop, + pause, + resume, + state, + segment, + error, + supported: isSynthesisSupported(defaults), + }; +} + +export interface UseListenResult { + start: (options?: ListenOptions) => Promise; + stop: () => void; + abort: () => void; + listening: boolean; + /** Final and interim text together. */ + text: string; + final: string; + interim: string; + error: SpeechError | undefined; + supported: boolean; +} + +/** Speech-to-text in a component. Listening stops when the component unmounts. */ +export function useListen(defaults: ListenOptions = {}): UseListenResult { + const [listening, setListening] = useState(false); + const [text, setText] = useState(''); + const [final, setFinal] = useState(''); + const [interim, setInterim] = useState(''); + const [error, setError] = useState(); + const handle = useRef(undefined); + const seq = useRef(0); + const defaultsRef = useRef(defaults); + useEffect(() => { + defaultsRef.current = defaults; + }); + + useEffect(() => () => handle.current?.abort(), []); + + const start = useCallback((options: ListenOptions = {}) => { + handle.current?.abort(); + const merged = { ...defaultsRef.current, ...options }; + const id = ++seq.current; + const latest = () => seq.current === id; + setError(undefined); + setText(''); + setFinal(''); + setInterim(''); + const h = listen({ + ...merged, + onResult: (r) => { + if (latest()) { + setText(r.text); + setFinal(r.final); + setInterim(r.interim); + } + merged.onResult?.(r); + }, + }); + handle.current = h; + setListening(h.listening); + return h.done.then( + (t) => { + if (latest()) { + setListening(false); + setInterim(''); + } + return t; + }, + (e: SpeechError) => { + if (latest()) { + setListening(false); + setError(e); + } + throw e; + } + ); + }, []); + + const stop = useCallback(() => handle.current?.stop(), []); + const abort = useCallback(() => handle.current?.abort(), []); + + return { + start, + stop, + abort, + listening, + text, + final, + interim, + error, + supported: isRecognitionSupported(defaults), + }; +} diff --git a/packages/speech/src/recognition.ts b/packages/speech/src/recognition.ts new file mode 100644 index 0000000..3ef395c --- /dev/null +++ b/packages/speech/src/recognition.ts @@ -0,0 +1,224 @@ +import { SpeechError } from './errors'; + +/** One recognition alternative, as the browser reports it. */ +interface AlternativeLike { + readonly transcript: string; +} + +/** One recognition result: alternatives, and whether it is final. */ +interface ResultLike { + readonly isFinal: boolean; + readonly length: number; + readonly [index: number]: AlternativeLike; +} + +/** The browser's SpeechRecognitionEvent, as far as this package reads it. */ +export interface RecognitionResultEventLike { + readonly resultIndex: number; + readonly results: { readonly length: number; readonly [index: number]: ResultLike }; +} + +/** The browser's SpeechRecognitionErrorEvent, as far as this package reads it. */ +export interface RecognitionErrorEventLike { + readonly error: string; +} + +/** The parts of the browser's SpeechRecognition this package uses. */ +export interface RecognitionLike { + lang: string; + continuous: boolean; + interimResults: boolean; + onresult: ((e: RecognitionResultEventLike) => void) | null; + onerror: ((e: RecognitionErrorEventLike) => void) | null; + onend: (() => void) | null; + onspeechstart: (() => void) | null; + start(): void; + stop(): void; + abort(): void; +} + +/** Creates a recognizer; set in tests. */ +export type RecognitionFactory = () => RecognitionLike; + +/** Recognized text so far. */ +export interface ListenResult { + /** Final and interim text together, as a user would see it typed. */ + text: string; + /** Text the browser will not revise. */ + final: string; + /** Text still being recognized, which may change. */ + interim: string; +} + +export interface ListenOptions { + /** BCP 47 language. Default `en-US`. */ + lang?: string; + /** Keep listening across pauses until stopped. Default true. */ + continuous?: boolean; + /** Report text while it is still being recognized. Default true. */ + interimResults?: boolean; + /** + * In continuous mode, stop after this many silences in a row (the browser + * reports one after several seconds without speech). Default 3; 0 never + * stops for silence. + */ + maxSilences?: number; + onResult?: (result: ListenResult) => void; + onError?: (error: SpeechError) => void; + /** Creates the recognizer; defaults to the browser's. */ + recognition?: RecognitionFactory; +} + +/** Listening in progress. */ +export interface Listening { + /** Resolves with the final text; rejects with a SpeechError. */ + readonly done: Promise; + readonly listening: boolean; + /** Stops listening and waits for the last results. */ + stop(): void; + /** Stops listening at once, dropping text still being recognized. */ + abort(): void; +} + +// Errors that end listening: no permission, no microphone, no service. +const FATAL = new Set([ + 'not-allowed', + 'service-not-allowed', + 'audio-capture', + 'network', + 'language-not-supported', + 'bad-grammar', +]); + +function browserRecognition(): RecognitionFactory | undefined { + const g = globalThis as { + SpeechRecognition?: new () => RecognitionLike; + webkitSpeechRecognition?: new () => RecognitionLike; + }; + const Ctor = g.SpeechRecognition ?? g.webkitSpeechRecognition; + return Ctor ? () => new Ctor() : undefined; +} + +/** Reports whether the browser can recognize speech. */ +export function isRecognitionSupported(options: Pick = {}): boolean { + return Boolean(options.recognition ?? browserRecognition()); +} + +/** + * Listens to the microphone and reports recognized text. In continuous mode + * the browser ends recognition on its own from time to time; this restarts + * it until stopped, a fatal error, or `maxSilences` silences in a row. + */ +export function listen(options: ListenOptions = {}): Listening { + const factory = options.recognition ?? browserRecognition(); + const continuous = options.continuous ?? true; + const maxSilences = options.maxSilences ?? 3; + + let active = true; + let final = ''; + let interim = ''; + let silences = 0; + let settled = false; + const rec = factory?.(); + let resolve!: (text: string) => void; + let reject!: (e: SpeechError) => void; + const done = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + + const handle: Listening = { + done, + get listening() { + return active; + }, + stop() { + if (!active) return; + active = false; + rec?.stop(); + }, + abort() { + if (!active) return; + active = false; + interim = ''; + rec?.abort(); + settle(); + }, + }; + + function settle(error?: SpeechError) { + active = false; + if (settled) return; + settled = true; + if (error) { + options.onError?.(error); + reject(error); + } else { + resolve(final.trim()); + } + } + + if (!rec) { + settle(new SpeechError('unsupported', 'this browser cannot recognize speech')); + return handle; + } + + const r = rec; + r.lang = options.lang ?? 'en-US'; + r.continuous = continuous; + r.interimResults = options.interimResults ?? true; + + r.onspeechstart = () => { + silences = 0; + }; + r.onresult = (e) => { + silences = 0; + interim = ''; + for (let i = e.resultIndex; i < e.results.length; i++) { + const res = e.results[i]; + const text = res.length > 0 ? res[0].transcript : ''; + if (res.isFinal) final = join(final, text); + else interim += text; + } + options.onResult?.({ + text: join(final, interim).trim(), + final: final.trim(), + interim: interim.trim(), + }); + }; + r.onerror = (e) => { + if (e.error === 'aborted') return; + if (FATAL.has(e.error)) { + settle(new SpeechError(e.error)); + return; + } + if (e.error === 'no-speech') { + silences++; + if (maxSilences > 0 && silences >= maxSilences) active = false; + } + }; + r.onend = () => { + if (active && continuous) { + try { + r.start(); + return; + } catch { + // The recognizer would not restart; finish with what was heard. + } + } + settle(); + }; + + try { + r.start(); + } catch (err) { + settle(new SpeechError('start-failed', err instanceof Error ? err.message : String(err))); + } + return handle; +} + +function join(a: string, b: string): string { + if (!a) return b; + if (!b) return a; + return /\s$/.test(a) || /^\s/.test(b) ? a + b : `${a} ${b}`; +} diff --git a/packages/speech/src/synthesis.ts b/packages/speech/src/synthesis.ts new file mode 100644 index 0000000..308d7be --- /dev/null +++ b/packages/speech/src/synthesis.ts @@ -0,0 +1,430 @@ +import { SpeechError } from './errors'; +import { DEFAULT_MAX_CHUNK_CHARS, splitText } from './text'; + +/** A piece of text to speak, and the silence after it. */ +export interface SpeakSegment { + text: string; + /** Silence after the text, in milliseconds. */ + pauseMs?: number; +} + +/** Plain text, or segments with pauses between them. */ +export type SpeakInput = string | readonly SpeakSegment[]; + +/** The parts of the browser's SpeechSynthesis this package uses. */ +export interface SynthesisLike { + readonly speaking: boolean; + readonly pending: boolean; + readonly paused: boolean; + speak(utterance: SpeechSynthesisUtterance): void; + cancel(): void; + pause(): void; + resume(): void; + getVoices(): SpeechSynthesisVoice[]; + addEventListener?(type: 'voiceschanged', listener: () => void): void; + removeEventListener?(type: 'voiceschanged', listener: () => void): void; +} + +/** Constructs an utterance; the browser's SpeechSynthesisUtterance. */ +export type UtteranceConstructor = new (text: string) => SpeechSynthesisUtterance; + +/** Where to speak; defaults to the browser's own. Set in tests. */ +export interface SynthesisEnv { + synthesis?: SynthesisLike; + Utterance?: UtteranceConstructor; +} + +export interface SpeakOptions extends SynthesisEnv { + /** BCP 47 language, such as `en-US`. Picks a voice when `voice` is not set. */ + lang?: string; + /** A voice, or a voice's `voiceURI` or `name`. */ + voice?: SpeechSynthesisVoice | string; + /** 0.1 to 10; 1 is normal speed. */ + rate?: number; + /** 0 to 2; 1 is normal. */ + pitch?: number; + /** 0 to 1. */ + volume?: number; + /** Longest text per utterance. See DEFAULT_MAX_CHUNK_CHARS. */ + maxChunkChars?: number; + /** Stops the speech when aborted. */ + signal?: AbortSignal; + /** + * How long an utterance may take to start before it is retried once, then + * reported as a `start-timeout` error. Default 3000. + */ + startTimeoutMs?: number; + /** Called as each segment starts, including one that is only a pause. */ + onSegment?: (index: number, segment: SpeakSegment) => void; +} + +export type SpeechState = 'speaking' | 'paused' | 'ended' | 'stopped' | 'error'; + +/** How speech finished: it reached the end, or it was stopped. */ +export interface SpeakResult { + status: 'ended' | 'stopped'; +} + +/** Speech in progress. */ +export interface Speaking { + /** Resolves when the speech ends or is stopped; rejects with a SpeechError. */ + readonly done: Promise; + readonly state: SpeechState; + stop(): void; + /** Pauses. Resuming repeats the utterance that was interrupted, at most one sentence. */ + pause(): void; + resume(): void; +} + +/** How long an utterance may take to start. See SpeakOptions.startTimeoutMs. */ +export const DEFAULT_START_TIMEOUT_MS = 3000; + +type Step = + | { kind: 'say'; text: string; segment: number; first: boolean } + | { kind: 'wait'; ms: number; segment: number; first: boolean }; + +// The browser speaks one thing at a time, so starting new speech stops the +// current one. +let current: { stop(): void } | undefined; + +// Speech left running or paused when a page goes away can stall the next +// page's speech in Chrome, so stop it as the page is hidden for good. +let pageHideBound = false; +function stopOnPageHide() { + if (pageHideBound) return; + const g = globalThis as { addEventListener?: (type: string, l: () => void) => void }; + if (typeof g.addEventListener !== 'function') return; + g.addEventListener('pagehide', () => current?.stop()); + pageHideBound = true; +} + +function browserSynthesis(): SynthesisLike | undefined { + const g = globalThis as { speechSynthesis?: SynthesisLike }; + return g.speechSynthesis; +} + +function browserUtterance(): UtteranceConstructor | undefined { + const g = globalThis as { SpeechSynthesisUtterance?: UtteranceConstructor }; + return g.SpeechSynthesisUtterance; +} + +/** Reports whether the browser can speak text. */ +export function isSynthesisSupported(env: SynthesisEnv = {}): boolean { + return Boolean((env.synthesis ?? browserSynthesis()) && (env.Utterance ?? browserUtterance())); +} + +function toSegments(input: SpeakInput): readonly SpeakSegment[] { + return typeof input === 'string' ? [{ text: input }] : input; +} + +function plan(segments: readonly SpeakSegment[], maxChars: number): Step[] { + const steps: Step[] = []; + segments.forEach((seg, segment) => { + let first = true; + for (const text of splitText(seg.text ?? '', maxChars)) { + steps.push({ kind: 'say', text, segment, first }); + first = false; + } + const ms = Math.max(0, seg.pauseMs ?? 0); + if (ms > 0) { + steps.push({ kind: 'wait', ms, segment, first }); + first = false; + } + }); + return steps; +} + +/** + * Speaks text or segments, with each segment's pause after it. Long text is + * split into several utterances (see `maxChunkChars`). Starting new speech + * stops speech already in progress. + */ +export function speak(input: SpeakInput, options: SpeakOptions = {}): Speaking { + const synth = options.synthesis ?? browserSynthesis(); + const Utterance = options.Utterance ?? browserUtterance(); + + let state: SpeechState = 'speaking'; + let resolve!: (r: SpeakResult) => void; + let reject!: (e: SpeechError) => void; + const done = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + + const handle: Speaking = { + done, + get state() { + return state; + }, + stop, + pause, + resume, + }; + + if (!synth || !Utterance) { + state = 'error'; + reject(new SpeechError('unsupported', 'this browser cannot speak text')); + return handle; + } + + const segments = toSegments(input); + const steps = plan(segments, options.maxChunkChars ?? DEFAULT_MAX_CHUNK_CHARS); + const voice = resolveVoice(synth, options); + let next = 0; + // The utterance being spoken. Holding it also keeps it from being + // garbage-collected before its end event, which loses the event in Chrome. + let utterance: SpeechSynthesisUtterance | undefined; + let timer: ReturnType | undefined; + let waitLeft = 0; + let waitStarted = 0; + // Watches that the current utterance starts; see startTimeoutMs. + let startTimer: ReturnType | undefined; + // Set while paused mid-utterance: resuming repeats that utterance. + let repeat = false; + const startTimeoutMs = options.startTimeoutMs ?? DEFAULT_START_TIMEOUT_MS; + + current?.stop(); + const self = { stop }; + current = self; + stopOnPageHide(); + // Always clear the queue first. When speech was paused and abandoned, here + // or in another tab, Chrome reports nothing speaking or pending, yet holds + // new speech without starting it or reporting an error. A cancel clears it. + synth.cancel(); + if (synth.paused) synth.resume(); + + options.signal?.addEventListener('abort', stop, { once: true }); + if (options.signal?.aborted) { + stop(); + return handle; + } + + function finish(result: SpeakResult | SpeechError) { + if (current === self) current = undefined; + utterance = undefined; + if (timer !== undefined) clearTimeout(timer); + timer = undefined; + clearStartTimer(); + options.signal?.removeEventListener('abort', stop); + if (result instanceof SpeechError) { + state = 'error'; + reject(result); + } else { + state = result.status; + resolve(result); + } + } + + function live() { + return state === 'speaking' || state === 'paused'; + } + + function clearStartTimer() { + if (startTimer !== undefined) clearTimeout(startTimer); + startTimer = undefined; + } + + function step() { + if (state !== 'speaking') return; + if (next >= steps.length) { + finish({ status: 'ended' }); + return; + } + const s = steps[next++]; + if (s.first) options.onSegment?.(s.segment, segments[s.segment]); + if (s.kind === 'wait') { + waitLeft = s.ms; + startWait(); + return; + } + say(s.text, false); + } + + // Speaks one utterance. A retry is the second try after it failed to start. + function say(text: string, retry: boolean) { + const u = new Utterance!(text); + if (voice) u.voice = voice; + const lang = options.lang ?? voice?.lang; + if (lang) u.lang = lang; + if (options.rate !== undefined) u.rate = options.rate; + if (options.pitch !== undefined) u.pitch = options.pitch; + if (options.volume !== undefined) u.volume = options.volume; + u.onstart = () => { + if (utterance === u) clearStartTimer(); + }; + u.onend = () => { + if (utterance !== u) return; + clearStartTimer(); + utterance = undefined; + step(); + }; + u.onerror = (e: SpeechSynthesisErrorEvent) => { + if (utterance !== u) return; + // This package lets go of the utterance before it cancels one, so a + // cancel that still finds it came from elsewhere: another library or + // tab calling speechSynthesis.cancel(). The speech has been stopped. + if (e.error === 'interrupted' || e.error === 'canceled') { + finish({ status: 'stopped' }); + return; + } + finish(new SpeechError(e.error ?? 'synthesis-failed')); + }; + utterance = u; + synth!.speak(u); + if (startTimeoutMs > 0 && utterance === u) { + clearStartTimer(); + startTimer = setTimeout(() => { + startTimer = undefined; + if (utterance !== u || state !== 'speaking') return; + // Accepted but never started: the stall above, set off after this + // speech began. Clear the queue and try once more. + utterance = undefined; + synth!.cancel(); + if (synth!.paused) synth!.resume(); + if (retry) { + finish( + new SpeechError('start-timeout', 'the browser accepted the speech but never started it') + ); + return; + } + say(text, true); + }, startTimeoutMs); + } + } + + function startWait() { + waitStarted = Date.now(); + timer = setTimeout(() => { + timer = undefined; + waitLeft = 0; + step(); + }, waitLeft); + } + + function stop() { + if (!live()) return; + const speaking = utterance !== undefined; + finish({ status: 'stopped' }); + if (speaking) synth!.cancel(); + } + + // Pausing never uses the browser's pause: speech paused and then abandoned + // stalls Chrome's speech for every page until it is cancelled. Instead the + // utterance is cancelled, and resuming speaks it again. + function pause() { + if (state !== 'speaking') return; + state = 'paused'; + if (timer !== undefined) { + clearTimeout(timer); + timer = undefined; + waitLeft = Math.max(0, waitLeft - (Date.now() - waitStarted)); + } else if (utterance) { + clearStartTimer(); + utterance = undefined; + repeat = true; + synth!.cancel(); + } + } + + function resume() { + if (state !== 'paused') return; + state = 'speaking'; + if (repeat) { + repeat = false; + const s = steps[next - 1]; + if (s.kind === 'say') say(s.text, false); + } else if (waitLeft > 0) startWait(); + else step(); + } + + step(); + return handle; +} + +/** Stops any speech this package started. */ +export function stopSpeaking(): void { + current?.stop(); +} + +function resolveVoice( + synth: SynthesisLike, + options: SpeakOptions +): SpeechSynthesisVoice | undefined { + const { voice, lang } = options; + if (voice && typeof voice !== 'string') return voice; + const voices = synth.getVoices(); + if (typeof voice === 'string') { + const found = voices.find((v) => v.voiceURI === voice) ?? voices.find((v) => v.name === voice); + if (found) return found; + } + return lang ? pickVoice(voices, lang) : undefined; +} + +function normalizeLang(lang: string): string { + return lang.replace(/_/g, '-').toLowerCase(); +} + +/** + * Reports whether a voice language suits a requested language: the same + * language and region, or the same language when either has no region + * (`en` matches `en-GB`). `en-US` does not match `en-GB`. + */ +export function langMatches(voiceLang: string, lang: string): boolean { + const a = normalizeLang(voiceLang); + const b = normalizeLang(lang); + if (a === b) return true; + const [pa, ra] = a.split('-'); + const [pb, rb] = b.split('-'); + return pa === pb && (!ra || !rb); +} + +/** + * Picks the best voice for a language: an exact language and region match + * before a language-only match, and within each the browser's default, then + * a voice on the device (which needs no network), then the first. + */ +export function pickVoice( + voices: readonly SpeechSynthesisVoice[], + lang: string +): SpeechSynthesisVoice | undefined { + const want = normalizeLang(lang); + const exact = voices.filter((v) => normalizeLang(v.lang) === want); + const primary = want.split('-')[0]; + const sameLanguage = voices.filter((v) => normalizeLang(v.lang).split('-')[0] === primary); + for (const group of [exact, sameLanguage]) { + const v = group.find((x) => x.default) ?? group.find((x) => x.localService) ?? group[0]; + if (v) return v; + } + return undefined; +} + +export interface GetVoicesOptions extends SynthesisEnv { + /** Only voices for this language (see langMatches). */ + lang?: string; + /** How long to wait for the browser to load its voices. Default 2000. */ + timeoutMs?: number; +} + +/** + * Lists the browser's voices. Browsers load voices after the page, so this + * waits for them, up to `timeoutMs`. Resolves to an empty list when the + * browser cannot speak or has no voices. + */ +export function getVoices(options: GetVoicesOptions = {}): Promise { + const synth = options.synthesis ?? browserSynthesis(); + const filter = (vs: SpeechSynthesisVoice[]) => + options.lang ? vs.filter((v) => langMatches(v.lang, options.lang!)) : vs; + if (!synth) return Promise.resolve([]); + const now = synth.getVoices(); + if (now.length > 0 || !synth.addEventListener) return Promise.resolve(filter(now)); + + return new Promise((resolve) => { + const finish = () => { + clearTimeout(timer); + synth.removeEventListener?.('voiceschanged', finish); + resolve(filter(synth.getVoices())); + }; + const timer = setTimeout(finish, options.timeoutMs ?? 2000); + synth.addEventListener!('voiceschanged', finish); + }); +} diff --git a/packages/speech/src/text.ts b/packages/speech/src/text.ts new file mode 100644 index 0000000..b49d39b --- /dev/null +++ b/packages/speech/src/text.ts @@ -0,0 +1,65 @@ +/** + * Default longest chunk passed to one utterance. Some browser voices stop + * partway through long utterances (Chrome's network voices after about 15 + * seconds), so long text is spoken as several shorter ones. + */ +export const DEFAULT_MAX_CHUNK_CHARS = 180; + +/** + * Splits text into chunks of at most `maxChars` characters, breaking at + * sentence ends first, then at clause punctuation, then between words. Short + * sentences are joined, so speech is not broken up more than needed. A single + * word longer than `maxChars` is kept whole. Whitespace is collapsed. + */ +export function splitText(text: string, maxChars: number = DEFAULT_MAX_CHUNK_CHARS): string[] { + const t = text.replace(/\s+/g, ' ').trim(); + if (!t) return []; + if (maxChars <= 0 || t.length <= maxChars) return [t]; + + const parts: string[] = []; + for (const sentence of pieces(t, '.!?…')) { + if (sentence.length <= maxChars) { + parts.push(sentence); + continue; + } + for (const clause of pieces(sentence, ',;:')) { + if (clause.length <= maxChars) parts.push(clause); + else parts.push(...pack(clause.split(' '), maxChars)); + } + } + return pack(parts, maxChars); +} + +// Splits after any character in `ends` that is followed by whitespace or the +// end of the text, keeping the punctuation. No lookbehind, which older +// Safari versions cannot parse. +function pieces(text: string, ends: string): string[] { + const out: string[] = []; + let start = 0; + for (let i = 0; i < text.length; i++) { + if (ends.includes(text[i]) && (i + 1 === text.length || text[i + 1] === ' ')) { + out.push(text.slice(start, i + 1).trim()); + start = i + 1; + } + } + const rest = text.slice(start).trim(); + if (rest) out.push(rest); + return out.filter(Boolean); +} + +// Joins parts with spaces into chunks no longer than max. +function pack(parts: string[], max: number): string[] { + const out: string[] = []; + let cur = ''; + for (const p of parts) { + if (!p) continue; + if (!cur) cur = p; + else if (cur.length + 1 + p.length <= max) cur += ' ' + p; + else { + out.push(cur); + cur = p; + } + } + if (cur) out.push(cur); + return out; +} diff --git a/packages/speech/tsconfig.json b/packages/speech/tsconfig.json new file mode 100644 index 0000000..b813d42 --- /dev/null +++ b/packages/speech/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src" + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist"] +} diff --git a/packages/speech/tsup.config.ts b/packages/speech/tsup.config.ts new file mode 100644 index 0000000..4c624e3 --- /dev/null +++ b/packages/speech/tsup.config.ts @@ -0,0 +1,23 @@ +import { defineConfig } from 'tsup'; + +export default defineConfig([ + { + entry: { index: 'src/index.ts', react: 'src/react.ts' }, + format: ['cjs', 'esm'], + dts: true, + splitting: false, + sourcemap: true, + clean: true, + external: ['react'], + }, + { + // One file for pages without a build step: a script tag, or a page + // served by a Go program that embeds it. Exposes window.PlexusSpeech. + entry: { speech: 'src/index.ts' }, + format: ['iife'], + globalName: 'PlexusSpeech', + minify: true, + sourcemap: false, + clean: false, + }, +]); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6fd1296..6f6855a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -296,6 +296,22 @@ importers: specifier: ^5.3.3 version: 5.9.3 + packages/speech: + dependencies: + react: + specifier: ^19.0.0 + version: 19.2.5 + devDependencies: + '@types/react': + specifier: ^19.0.0 + version: 19.2.14 + tsup: + specifier: ^8.0.1 + version: 8.5.1(jiti@2.7.0)(postcss@8.5.9)(typescript@5.9.3)(yaml@2.8.4) + typescript: + specifier: ^5.3.3 + version: 5.9.3 + packages/telemetry: dependencies: react: From 8d3028d97d4b5d8bd94a845a204e931034e261c4 Mon Sep 17 00:00:00 2001 From: John Wang Date: Mon, 2 Nov 2026 07:51:36 -0800 Subject: [PATCH 02/16] test(speech): cover speaking, listening, text splitting and the hooks Refs: RMI-SYSTEMFORGEWEB-031 Co-Authored-By: Claude Opus 5.5 --- packages/speech/src/__tests__/fakes.ts | 152 +++++++++++ packages/speech/src/__tests__/react.test.tsx | 86 ++++++ .../speech/src/__tests__/recognition.test.ts | 86 ++++++ .../speech/src/__tests__/synthesis.test.ts | 252 ++++++++++++++++++ packages/speech/src/__tests__/text.test.ts | 44 +++ vitest.config.ts | 1 + 6 files changed, 621 insertions(+) create mode 100644 packages/speech/src/__tests__/fakes.ts create mode 100644 packages/speech/src/__tests__/react.test.tsx create mode 100644 packages/speech/src/__tests__/recognition.test.ts create mode 100644 packages/speech/src/__tests__/synthesis.test.ts create mode 100644 packages/speech/src/__tests__/text.test.ts diff --git a/packages/speech/src/__tests__/fakes.ts b/packages/speech/src/__tests__/fakes.ts new file mode 100644 index 0000000..513df23 --- /dev/null +++ b/packages/speech/src/__tests__/fakes.ts @@ -0,0 +1,152 @@ +import type { + RecognitionErrorEventLike, + RecognitionLike, + RecognitionResultEventLike, + SynthesisLike, + UtteranceConstructor, +} from '../index'; + +/** Stands in for SpeechSynthesisUtterance. */ +export class FakeUtterance { + text: string; + lang = ''; + voice: SpeechSynthesisVoice | null = null; + rate = 1; + pitch = 1; + volume = 1; + onstart: (() => void) | null = null; + onend: (() => void) | null = null; + onerror: ((e: { error: string }) => void) | null = null; + + constructor(text: string) { + this.text = text; + } +} + +export const Utterance = FakeUtterance as unknown as UtteranceConstructor; + +export function voice(name: string, lang: string, extra: Partial = {}) { + return { + name, + lang, + voiceURI: `uri:${name}`, + default: false, + localService: false, + ...extra, + } as SpeechSynthesisVoice; +} + +/** + * Stands in for speechSynthesis. The test starts each utterance with start() + * and ends it with end(); one never started stands for Chrome's stalled queue. + */ +export class FakeSynthesis implements SynthesisLike { + spoken: FakeUtterance[] = []; + current: FakeUtterance | undefined; + paused = false; + voices: SpeechSynthesisVoice[] = []; + calls: string[] = []; + private listeners: Array<() => void> = []; + + get speaking() { + return this.current !== undefined; + } + get pending() { + return false; + } + + speak(u: SpeechSynthesisUtterance) { + const f = u as unknown as FakeUtterance; + this.calls.push('speak'); + this.spoken.push(f); + this.current = f; + } + /** Starts the current utterance. */ + start() { + this.current?.onstart?.(); + } + /** Finishes the current utterance. */ + end() { + const u = this.current; + this.current = undefined; + u?.onend?.(); + } + fail(error: string) { + const u = this.current; + this.current = undefined; + u?.onerror?.({ error }); + } + cancel() { + this.calls.push('cancel'); + const u = this.current; + this.current = undefined; + u?.onerror?.({ error: 'interrupted' }); + } + pause() { + this.calls.push('pause'); + this.paused = true; + } + resume() { + this.calls.push('resume'); + this.paused = false; + } + getVoices() { + return this.voices; + } + addEventListener(_type: 'voiceschanged', l: () => void) { + this.listeners.push(l); + } + removeEventListener(_type: 'voiceschanged', l: () => void) { + this.listeners = this.listeners.filter((x) => x !== l); + } + loadVoices(vs: SpeechSynthesisVoice[]) { + this.voices = vs; + for (const l of [...this.listeners]) l(); + } + texts() { + return this.spoken.map((u) => u.text); + } +} + +/** Stands in for SpeechRecognition. */ +export class FakeRecognition implements RecognitionLike { + lang = ''; + continuous = false; + interimResults = false; + onresult: ((e: RecognitionResultEventLike) => void) | null = null; + onerror: ((e: RecognitionErrorEventLike) => void) | null = null; + onend: (() => void) | null = null; + onspeechstart: (() => void) | null = null; + starts = 0; + running = false; + private results: Array<{ isFinal: boolean; length: number; 0: { transcript: string } }> = []; + + start() { + if (this.running) throw new Error('already started'); + this.starts++; + this.running = true; + } + stop() { + this.endSession(); + } + abort() { + this.onerror?.({ error: 'aborted' }); + this.endSession(); + } + /** Reports a result, replacing the interim one when there is one. */ + say(transcript: string, isFinal: boolean) { + const last = this.results[this.results.length - 1]; + const index = last && !last.isFinal ? this.results.length - 1 : this.results.length; + this.results[index] = { isFinal, length: 1, 0: { transcript } }; + this.onresult?.({ resultIndex: index, results: this.results }); + } + error(error: string) { + this.onerror?.({ error }); + } + /** The browser ends a session; results start over in the next one. */ + endSession() { + this.running = false; + this.results = []; + this.onend?.(); + } +} diff --git a/packages/speech/src/__tests__/react.test.tsx b/packages/speech/src/__tests__/react.test.tsx new file mode 100644 index 0000000..2033c1f --- /dev/null +++ b/packages/speech/src/__tests__/react.test.tsx @@ -0,0 +1,86 @@ +import { act, renderHook, waitFor } from '@testing-library/react'; +import { describe, expect, it } from 'vitest'; +import { useListen, useSpeech, useVoices } from '../react'; +import { FakeRecognition, FakeSynthesis, Utterance, voice } from './fakes'; + +describe('useSpeech', () => { + it('reports state and the segment being spoken', async () => { + const synthesis = new FakeSynthesis(); + const { result } = renderHook(() => useSpeech({ synthesis, Utterance })); + expect(result.current.state).toBe('idle'); + expect(result.current.supported).toBe(true); + + let done!: Promise; + act(() => { + done = result.current.speak([{ text: 'A.' }, { text: 'B.' }]); + }); + expect(result.current.state).toBe('speaking'); + expect(result.current.segment).toBe(0); + + act(() => synthesis.end()); + expect(result.current.segment).toBe(1); + + await act(async () => { + synthesis.end(); + await done; + }); + expect(result.current.state).toBe('ended'); + expect(result.current.segment).toBe(-1); + }); + + it('reports an error', async () => { + const synthesis = new FakeSynthesis(); + const { result } = renderHook(() => useSpeech({ synthesis, Utterance })); + await act(async () => { + const p = result.current.speak('A.').catch(() => undefined); + synthesis.fail('synthesis-failed'); + await p; + }); + expect(result.current.state).toBe('error'); + expect(result.current.error?.code).toBe('synthesis-failed'); + }); + + it('stops speaking on unmount', () => { + const synthesis = new FakeSynthesis(); + const { result, unmount } = renderHook(() => useSpeech({ synthesis, Utterance })); + act(() => { + void result.current.speak('A long sentence.'); + }); + unmount(); + expect(synthesis.calls).toContain('cancel'); + }); +}); + +describe('useVoices', () => { + it('lists voices for a language once loaded', async () => { + const synthesis = new FakeSynthesis(); + const { result } = renderHook(() => useVoices('en', { synthesis })); + expect(result.current).toEqual([]); + act(() => synthesis.loadVoices([voice('Samantha', 'en-US'), voice('Thomas', 'fr-FR')])); + await waitFor(() => expect(result.current.map((v) => v.name)).toEqual(['Samantha'])); + }); +}); + +describe('useListen', () => { + it('reports recognized text and stops', async () => { + const rec = new FakeRecognition(); + const { result } = renderHook(() => useListen({ recognition: () => rec })); + let done!: Promise; + act(() => { + done = result.current.start(); + }); + expect(result.current.listening).toBe(true); + + act(() => rec.say('hello there', false)); + expect(result.current.text).toBe('hello there'); + expect(result.current.interim).toBe('hello there'); + + act(() => rec.say('hello there', true)); + await act(async () => { + result.current.stop(); + await done; + }); + expect(result.current.listening).toBe(false); + expect(result.current.final).toBe('hello there'); + }); +}); diff --git a/packages/speech/src/__tests__/recognition.test.ts b/packages/speech/src/__tests__/recognition.test.ts new file mode 100644 index 0000000..7acd4c8 --- /dev/null +++ b/packages/speech/src/__tests__/recognition.test.ts @@ -0,0 +1,86 @@ +import { describe, expect, it } from 'vitest'; +import { isRecognitionSupported, listen } from '../index'; +import type { ListenResult } from '../index'; +import { FakeRecognition } from './fakes'; + +function setup(options: Parameters[0] = {}) { + const rec = new FakeRecognition(); + const results: ListenResult[] = []; + const l = listen({ ...options, recognition: () => rec, onResult: (r) => results.push(r) }); + return { rec, results, l }; +} + +describe('listen', () => { + it('configures the recognizer and starts it', () => { + const { rec, l } = setup({ lang: 'fr-FR' }); + expect(rec.lang).toBe('fr-FR'); + expect(rec.continuous).toBe(true); + expect(rec.interimResults).toBe(true); + expect(rec.starts).toBe(1); + expect(l.listening).toBe(true); + }); + + it('reports final and interim text', () => { + const { rec, results } = setup(); + rec.say('hello', false); + rec.say('hello world', true); + rec.say('how', false); + expect(results.at(-1)).toEqual({ + text: 'hello world how', + final: 'hello world', + interim: 'how', + }); + }); + + it('restarts when the browser ends a session, keeping the text', async () => { + const { rec, results, l } = setup(); + rec.say('one', true); + rec.endSession(); + expect(rec.starts).toBe(2); + rec.say('two', true); + expect(results.at(-1)?.final).toBe('one two'); + l.stop(); + await expect(l.done).resolves.toBe('one two'); + }); + + it('ends after several silences in a row', async () => { + const { rec, l } = setup({ maxSilences: 2 }); + rec.error('no-speech'); + rec.endSession(); + expect(rec.starts).toBe(2); + rec.error('no-speech'); + rec.endSession(); + expect(rec.starts).toBe(2); + await expect(l.done).resolves.toBe(''); + expect(l.listening).toBe(false); + }); + + it('rejects on a fatal error', async () => { + const { rec, l } = setup(); + rec.error('not-allowed'); + await expect(l.done).rejects.toMatchObject({ code: 'not-allowed' }); + expect(l.listening).toBe(false); + }); + + it('abort drops the interim text', async () => { + const { rec, l } = setup(); + rec.say('kept', true); + rec.say('dropped', false); + l.abort(); + await expect(l.done).resolves.toBe('kept'); + }); + + it('ends after one session when not continuous', async () => { + const { rec, l } = setup({ continuous: false }); + rec.say('once', true); + rec.endSession(); + expect(rec.starts).toBe(1); + await expect(l.done).resolves.toBe('once'); + }); + + it('rejects as unsupported without speech recognition', async () => { + const l = listen(); + await expect(l.done).rejects.toMatchObject({ code: 'unsupported' }); + expect(isRecognitionSupported()).toBe(false); + }); +}); diff --git a/packages/speech/src/__tests__/synthesis.test.ts b/packages/speech/src/__tests__/synthesis.test.ts new file mode 100644 index 0000000..da81b2e --- /dev/null +++ b/packages/speech/src/__tests__/synthesis.test.ts @@ -0,0 +1,252 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { + getVoices, + isSynthesisSupported, + langMatches, + pickVoice, + speak, + SpeechError, +} from '../index'; +import { FakeSynthesis, Utterance, voice } from './fakes'; + +let synthesis: FakeSynthesis; +const env = () => ({ synthesis, Utterance }); + +beforeEach(() => { + vi.useFakeTimers(); + synthesis = new FakeSynthesis(); +}); + +afterEach(() => { + vi.useRealTimers(); +}); + +describe('speak', () => { + it('speaks segments in order with their pauses', async () => { + const started: number[] = []; + const s = speak( + [{ text: 'First.', pauseMs: 500 }, { text: '', pauseMs: 300 }, { text: 'Third.' }], + { ...env(), onSegment: (i) => started.push(i) } + ); + + expect(synthesis.texts()).toEqual(['First.']); + synthesis.end(); + expect(synthesis.current).toBeUndefined(); + vi.advanceTimersByTime(499); + expect(started).toEqual([0]); + vi.advanceTimersByTime(1); + expect(started).toEqual([0, 1]); + vi.advanceTimersByTime(299); + expect(synthesis.texts()).toEqual(['First.']); + vi.advanceTimersByTime(1); + expect(synthesis.texts()).toEqual(['First.', 'Third.']); + expect(started).toEqual([0, 1, 2]); + synthesis.end(); + + await expect(s.done).resolves.toEqual({ status: 'ended' }); + expect(s.state).toBe('ended'); + }); + + it('reports speech cancelled from elsewhere as stopped', async () => { + const s = speak( + ['One.', 'Two.'].map((text) => ({ text })), + env() + ); + synthesis.start(); + synthesis.fail('interrupted'); // another script called speechSynthesis.cancel() + await expect(s.done).resolves.toEqual({ status: 'stopped' }); + expect(s.state).toBe('stopped'); + expect(synthesis.texts()).toEqual(['One.']); + }); + + it('splits long text into several utterances', async () => { + const s = speak('One two. Three four. Five six seven.', { ...env(), maxChunkChars: 20 }); + synthesis.end(); + synthesis.end(); + await s.done; + expect(synthesis.texts()).toEqual(['One two. Three four.', 'Five six seven.']); + }); + + it('sets the voice, language and rate on each utterance', () => { + const en = voice('Samantha', 'en-US'); + synthesis.voices = [voice('Thomas', 'fr-FR'), en]; + speak('Hi.', { ...env(), voice: 'Samantha', rate: 1.2 }); + const u = synthesis.spoken[0]; + expect(u.voice).toBe(en); + expect(u.lang).toBe('en-US'); + expect(u.rate).toBe(1.2); + }); + + it('picks a voice for the language when none is named', () => { + const fr = voice('Thomas', 'fr-FR'); + synthesis.voices = [voice('Samantha', 'en-US'), fr]; + speak('Bonjour.', { ...env(), lang: 'fr-FR' }); + expect(synthesis.spoken[0].voice).toBe(fr); + }); + + it('stops, cancelling the browser speech', async () => { + const s = speak('Hello.', env()); + s.stop(); + await expect(s.done).resolves.toEqual({ status: 'stopped' }); + expect(synthesis.calls).toContain('cancel'); + expect(s.state).toBe('stopped'); + }); + + it('stops during a pause without speaking the rest', async () => { + const s = speak([{ text: 'A.', pauseMs: 1000 }, { text: 'B.' }], env()); + synthesis.end(); + s.stop(); + vi.advanceTimersByTime(2000); + await s.done; + expect(synthesis.texts()).toEqual(['A.']); + }); + + it('stops earlier speech when new speech starts', async () => { + const first = speak('First.', env()); + const second = speak('Second.', env()); + await expect(first.done).resolves.toEqual({ status: 'stopped' }); + expect(second.state).toBe('speaking'); + expect(synthesis.texts()).toEqual(['First.', 'Second.']); + }); + + it('pauses by cancelling, never with the browser pause, and repeats the utterance on resume', async () => { + const s = speak([{ text: 'A.', pauseMs: 1000 }, { text: 'B.' }], env()); + synthesis.start(); + s.pause(); + expect(s.state).toBe('paused'); + expect(synthesis.current).toBeUndefined(); + expect(synthesis.calls).not.toContain('pause'); + vi.advanceTimersByTime(10000); + s.resume(); + expect(synthesis.texts()).toEqual(['A.', 'A.']); + synthesis.start(); + synthesis.end(); + + vi.advanceTimersByTime(400); + s.pause(); + vi.advanceTimersByTime(5000); + expect(synthesis.texts()).toEqual(['A.', 'A.']); + s.resume(); + vi.advanceTimersByTime(599); + expect(synthesis.texts()).toEqual(['A.', 'A.']); + vi.advanceTimersByTime(1); + expect(synthesis.texts()).toEqual(['A.', 'A.', 'B.']); + synthesis.start(); + synthesis.end(); + await expect(s.done).resolves.toEqual({ status: 'ended' }); + expect(synthesis.calls).not.toContain('pause'); + }); + + it('reports a segment once when an utterance is repeated after a pause', () => { + const started: number[] = []; + const s = speak([{ text: 'A.' }], { ...env(), onSegment: (i) => started.push(i) }); + s.pause(); + s.resume(); + expect(started).toEqual([0]); + }); + + it('clears the queue before speaking, even when the browser reports it idle', () => { + synthesis.paused = true; + speak('Hello.', env()); + expect(synthesis.calls.slice(0, 3)).toEqual(['cancel', 'resume', 'speak']); + }); + + it('retries an utterance that never starts, then rejects with start-timeout', async () => { + const s = speak('Hello.', env()); + vi.advanceTimersByTime(2999); + expect(synthesis.texts()).toEqual(['Hello.']); + vi.advanceTimersByTime(1); + expect(synthesis.texts()).toEqual(['Hello.', 'Hello.']); + expect(synthesis.calls.slice(-2)).toEqual(['cancel', 'speak']); + vi.advanceTimersByTime(3000); + const err = await s.done.catch((e: unknown) => e); + expect((err as SpeechError).code).toBe('start-timeout'); + expect(s.state).toBe('error'); + }); + + it('carries on after a retry starts', async () => { + const s = speak('Hello.', { ...env(), startTimeoutMs: 500 }); + vi.advanceTimersByTime(500); + synthesis.start(); + vi.advanceTimersByTime(5000); + synthesis.end(); + await expect(s.done).resolves.toEqual({ status: 'ended' }); + expect(synthesis.texts()).toEqual(['Hello.', 'Hello.']); + }); + + it('does not watch an utterance that started, or when startTimeoutMs is 0', async () => { + const started = speak('One.', env()); + synthesis.start(); + vi.advanceTimersByTime(10000); + expect(synthesis.texts()).toEqual(['One.']); + synthesis.end(); + await started.done; + + const unwatched = speak('Two.', { ...env(), startTimeoutMs: 0 }); + vi.advanceTimersByTime(10000); + expect(unwatched.state).toBe('speaking'); + expect(synthesis.texts()).toEqual(['One.', 'Two.']); + }); + + it('stops when the page is hidden', async () => { + const s = speak('Hello.', env()); + window.dispatchEvent(new Event('pagehide')); + await expect(s.done).resolves.toEqual({ status: 'stopped' }); + }); + + it('rejects with the browser error code', async () => { + const s = speak('Hello.', env()); + synthesis.fail('not-allowed'); + const err = await s.done.catch((e: unknown) => e); + expect(err).toBeInstanceOf(SpeechError); + expect((err as SpeechError).code).toBe('not-allowed'); + expect(s.state).toBe('error'); + }); + + it('stops when the signal aborts', async () => { + const ac = new AbortController(); + const s = speak('Hello.', { ...env(), signal: ac.signal }); + ac.abort(); + await expect(s.done).resolves.toEqual({ status: 'stopped' }); + }); + + it('rejects as unsupported without speech synthesis', async () => { + const s = speak('Hello.', { synthesis: undefined, Utterance: undefined }); + await expect(s.done).rejects.toMatchObject({ code: 'unsupported' }); + expect(isSynthesisSupported()).toBe(false); + expect(isSynthesisSupported(env())).toBe(true); + }); +}); + +describe('voices', () => { + it('matches languages with and without a region', () => { + expect(langMatches('en-US', 'en-us')).toBe(true); + expect(langMatches('en_GB', 'en')).toBe(true); + expect(langMatches('en', 'en-GB')).toBe(true); + expect(langMatches('en-US', 'en-GB')).toBe(false); + expect(langMatches('fr-FR', 'en')).toBe(false); + }); + + it('prefers an exact match, then the default, then a voice on the device', () => { + const remote = voice('Remote', 'en-US'); + const local = voice('Local', 'en-US', { localService: true }); + const preferred = voice('Default', 'en-US', { default: true }); + const gb = voice('GB', 'en-GB', { default: true }); + expect(pickVoice([gb, remote, local, preferred], 'en-US')).toBe(preferred); + expect(pickVoice([gb, remote, local], 'en-US')).toBe(local); + expect(pickVoice([gb, remote], 'en-AU')).toBe(gb); + expect(pickVoice([gb], 'de-DE')).toBeUndefined(); + }); + + it('waits for the browser to load its voices', async () => { + const pending = getVoices({ synthesis, lang: 'fr' }); + synthesis.loadVoices([voice('Samantha', 'en-US'), voice('Thomas', 'fr-FR')]); + expect((await pending).map((v) => v.name)).toEqual(['Thomas']); + }); + + it('gives up waiting after the timeout', async () => { + const pending = getVoices({ synthesis, timeoutMs: 100 }); + vi.advanceTimersByTime(100); + await expect(pending).resolves.toEqual([]); + }); +}); diff --git a/packages/speech/src/__tests__/text.test.ts b/packages/speech/src/__tests__/text.test.ts new file mode 100644 index 0000000..36884a3 --- /dev/null +++ b/packages/speech/src/__tests__/text.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, it } from 'vitest'; +import { splitText } from '../index'; + +describe('splitText', () => { + it('keeps short text whole and collapses whitespace', () => { + expect(splitText(' Hello\n there. ')).toEqual(['Hello there.']); + }); + + it('returns nothing for empty text', () => { + expect(splitText(' ')).toEqual([]); + }); + + it('breaks at sentence ends and joins short sentences', () => { + expect(splitText('One two. Three four. Five six seven.', 20)).toEqual([ + 'One two. Three four.', + 'Five six seven.', + ]); + }); + + it('does not break inside a number', () => { + expect(splitText('It grew 3.5 times. Then it stopped.', 20)).toEqual([ + 'It grew 3.5 times.', + 'Then it stopped.', + ]); + }); + + it('breaks a long sentence at clauses, then between words', () => { + expect(splitText('alpha beta, gamma delta epsilon zeta eta theta', 16)).toEqual([ + 'alpha beta,', + 'gamma delta', + 'epsilon zeta eta', + 'theta', + ]); + }); + + it('keeps a word longer than the limit whole', () => { + expect(splitText('a supercalifragilistic b', 5)).toEqual(['a', 'supercalifragilistic', 'b']); + }); + + it('does not split when the limit is zero', () => { + const long = 'word '.repeat(100).trim(); + expect(splitText(long, 0)).toEqual([long]); + }); +}); diff --git a/vitest.config.ts b/vitest.config.ts index 2e68c31..7c11557 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -9,6 +9,7 @@ const packagesDir = fileURLToPath(new URL('./packages', import.meta.url)); // its own entry instead of being appended to the package's index. const alias = [ { find: /^@plexusone\/host\/testing$/, replacement: `${packagesDir}/host/src/testing.tsx` }, + { find: /^@plexusone\/speech\/react$/, replacement: `${packagesDir}/speech/src/react.ts` }, ...readdirSync(packagesDir).map((name) => ({ find: new RegExp(`^@plexusone/${name}$`), replacement: `${packagesDir}/${name}/src/index.ts`, From cd7f540ff3e32c2a01f2225c35f31f81a71b2048 Mon Sep 17 00:00:00 2001 From: John Wang Date: Mon, 2 Nov 2026 07:51:37 -0800 Subject: [PATCH 03/16] docs(speech): document the speech package Refs: RMI-SYSTEMFORGEWEB-031 Co-Authored-By: Claude Opus 5.5 --- README.md | 1 + docs/packages/index.md | 7 +- docs/packages/speech.md | 167 ++++++++++++++++++++++++++++++++++++++ mkdocs.yml | 1 + packages/speech/README.md | 24 ++++++ 5 files changed, 198 insertions(+), 2 deletions(-) create mode 100644 docs/packages/speech.md create mode 100644 packages/speech/README.md diff --git a/README.md b/README.md index 391b673..6d34054 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ Alphabetical; see [Packages](docs/packages/index.md) for how they depend on each | `@plexusone/invites` | Invite-gated sign-up pages | [![npm](https://img.shields.io/npm/v/@plexusone/invites)](https://www.npmjs.com/package/@plexusone/invites) | | `@plexusone/pages` | Pre-built pages | [![npm](https://img.shields.io/npm/v/@plexusone/pages)](https://www.npmjs.com/package/@plexusone/pages) | | `@plexusone/shell` | Application shell | [![npm](https://img.shields.io/npm/v/@plexusone/shell)](https://www.npmjs.com/package/@plexusone/shell) | +| `@plexusone/speech` | Browser speech in and out | [![npm](https://img.shields.io/npm/v/@plexusone/speech)](https://www.npmjs.com/package/@plexusone/speech) | | `@plexusone/telemetry` | Event instrumentation | [![npm](https://img.shields.io/npm/v/@plexusone/telemetry)](https://www.npmjs.com/package/@plexusone/telemetry) | | `@plexusone/tenant` | Multi-tenant context | [![npm](https://img.shields.io/npm/v/@plexusone/tenant)](https://www.npmjs.com/package/@plexusone/tenant) | | `@plexusone/ui` | UI components | [![npm](https://img.shields.io/npm/v/@plexusone/ui)](https://www.npmjs.com/package/@plexusone/ui) | diff --git a/docs/packages/index.md b/docs/packages/index.md index e201bc9..006596b 100644 --- a/docs/packages/index.md +++ b/docs/packages/index.md @@ -16,11 +16,13 @@ SystemForge Web is a monorepo of focused packages, published to npm under @plexusone/pages ─────▶ @plexusone/auth Standalone: @plexusone/api-client, @plexusone/chat, - @plexusone/design-tokens, @plexusone/telemetry + @plexusone/design-tokens, @plexusone/speech, + @plexusone/telemetry ``` An arrow is a peer dependency: the application installs both, and they -share one copy. Every package also needs React 19. +share one copy. Every package also needs React 19, except +`@plexusone/design-tokens` and the main entry point of `@plexusone/speech`. ## Package summary @@ -35,6 +37,7 @@ share one copy. Every package also needs React 19. | [@plexusone/invites](invites.md) | Invite-gated sign-up pages | `InviteAdminPage`, `InvitePeoplePage`, `createInvitesApi` | | [@plexusone/pages](pages.md) | Pre-built pages | `LoginPage`, `ErrorPage`, `OrganizationSettingsPage` | | [@plexusone/shell](shell.md) | Application chrome | `AppShell`, `ProductSwitcher`, `OrganizationSwitcher`, `ProfileMenu` | +| [@plexusone/speech](speech.md) | Browser speech in and out | `speak`, `listen`, `getVoices`, `useSpeech`, `useListen` | | [@plexusone/telemetry](telemetry.md) | Instrumentation | `TelemetryProvider`, `ErrorBoundary` | | [@plexusone/tenant](tenant.md) | Multi-tenancy | `TenantProvider`, `RequireRole`, `useOrganization` | | [@plexusone/ui](ui.md) | UI components | `Button`, `Card`, `DropdownMenu`, `Tooltip` | diff --git a/docs/packages/speech.md b/docs/packages/speech.md new file mode 100644 index 0000000..13f959f --- /dev/null +++ b/docs/packages/speech.md @@ -0,0 +1,167 @@ +# @plexusone/speech + +Text-to-speech and speech-to-text with the browser's own speech (the Web +Speech API). No service, no key, no cost: useful for previews, read-aloud, +and voice input. + +The main entry point has no dependencies and works without React. React +hooks are in `@plexusone/speech/react`. A standalone build, +`@plexusone/speech/browser`, is one file for pages without a build step. + +## Installation + +```bash +pnpm add @plexusone/speech +``` + +React is needed only for `@plexusone/speech/react`. + +## Speaking + +`speak` takes text, or segments with a pause after each, and returns the +speech in progress: + +```ts +import { speak } from '@plexusone/speech'; + +const speech = speak( + [{ text: 'Welcome to the course.', pauseMs: 600 }, { text: 'Let us start with the funnel.' }], + { lang: 'en-US', rate: 1.1, onSegment: (i) => highlight(i) } +); + +await speech.done; // { status: 'ended' } or { status: 'stopped' } +``` + +| Option | Purpose | +| ------------------------- | ----------------------------------------------------------------- | +| `lang` | BCP 47 language. Picks a voice when `voice` is not set | +| `voice` | A voice, or a voice's `voiceURI` or `name` | +| `rate`, `pitch`, `volume` | As on `SpeechSynthesisUtterance` | +| `onSegment` | Called as each segment starts, including one that is only a pause | +| `signal` | An `AbortSignal` that stops the speech | +| `maxChunkChars` | Longest text per utterance (default 180) | +| `startTimeoutMs` | How long an utterance may take to start (default 3000; 0 is off) | + +The returned object has `stop()`, `pause()`, `resume()` and `state` +(`speaking`, `paused`, `ended`, `stopped` or `error`). `done` rejects with a +`SpeechError` whose `code` is the browser's error, `unsupported`, or +`start-timeout` when the browser never started the speech. On +`start-timeout`, offer another way to hear the text, such as a recording, or +show it to read. + +The browser speaks one thing at a time, so `speak` stops speech already in +progress. `stopSpeaking()` stops it from anywhere. + +### What it handles + +- **Long text.** Some voices stop partway through a long utterance (Chrome's + online voices after about 15 seconds). Text is split at sentence ends, + then clauses, then words, into utterances of at most `maxChunkChars`. +- **Lost end events.** Chrome can drop an utterance's end event when the + utterance is garbage-collected; the one being spoken is kept referenced. +- **Stalled speech.** In Chrome, speech that was paused and then abandoned, + by leaving the page or in another tab, stalls speech in every page: the + browser reports nothing speaking, accepts new speech and never starts it, + with no error. Only cancelling clears it, so `speak` always clears the + queue first. If an utterance still hasn't started after `startTimeoutMs`, + it is cancelled and tried once more, then `done` rejects with + `start-timeout`. +- **Pausing without the browser's pause.** Because a paused, abandoned + utterance causes that stall, `pause()` cancels the current utterance and + `resume()` speaks it again from its start: at most one sentence, or + `maxChunkChars`, is repeated. A segment's `pauseMs` is a timer, which + `pause()` and `resume()` stop and continue. +- **Cancelled from elsewhere.** Another script or library calling + `speechSynthesis.cancel()` ends this speech too; `done` then resolves + with `stopped` rather than waiting for an end that never comes. +- **Leaving the page.** Speech is stopped when the page is hidden for good + (`pagehide`), so it can't stall the next page. +- **User gestures.** Browsers may refuse speech that is not started from a + click or key press; `done` then rejects with code `not-allowed`. + +## Voices + +Browsers load voices after the page. `getVoices` waits for them: + +```ts +import { getVoices, pickVoice } from '@plexusone/speech'; + +const voices = await getVoices({ lang: 'fr' }); // fr-FR, fr-CA, ... +const best = pickVoice(voices, 'fr-FR'); +``` + +`pickVoice` prefers the exact language and region, then the same language; +within each, the browser's default voice, then a voice on the device (which +needs no network). `langMatches('en', 'en-GB')` is true; +`langMatches('en-US', 'en-GB')` is not. + +## Listening + +`listen` reports text as it is recognized: + +```ts +import { listen } from '@plexusone/speech'; + +const listening = listen({ + lang: 'en-US', + onResult: ({ text, final, interim }) => (input.value = text), +}); + +// later +listening.stop(); +const text = await listening.done; +``` + +In continuous mode (the default) the browser ends recognition by itself +from time to time; `listen` restarts it until `stop()`, a fatal error (no +permission, no microphone, no service), or `maxSilences` silences in a row +(default 3). `abort()` stops at once and drops text still being recognized. +Speech recognition is available in Chrome, Edge and Safari; check +`isRecognitionSupported()`. + +## React + +```tsx +import { useListen, useSpeech, useVoices } from '@plexusone/speech/react'; + +function ReadAloud({ text }: { text: string }) { + const { speak, stop, state, supported } = useSpeech({ lang: 'en-US' }); + if (!supported) return null; + return state === 'speaking' ? ( + + ) : ( + + ); +} +``` + +| Hook | Returns | +| -------------------- | --------------------------------------------------------------------------------------- | +| `useSpeech(options)` | `speak`, `stop`, `pause`, `resume`, `state`, `segment`, `error`, `supported` | +| `useListen(options)` | `start`, `stop`, `abort`, `listening`, `text`, `final`, `interim`, `error`, `supported` | +| `useVoices(lang)` | The voices, empty until loaded | + +Speech and listening stop when the component unmounts. The promises from +`speak` and `start` reject on errors, which the hooks also report as +`error`. + +## Pages without a build step + +`dist/speech.global.js` (the `@plexusone/speech/browser` export) defines +`window.PlexusSpeech` with the same functions as the main entry point: + +```html + + +``` + +A program that serves its own pages, such as a Go binary that embeds them, +can copy the file in at a pinned version instead of loading it from a CDN. + +## Testing + +Every function takes the browser objects as options (`synthesis` and +`Utterance` for speaking, `recognition` for listening), so tests can pass +fakes where jsdom has no speech. diff --git a/mkdocs.yml b/mkdocs.yml index f13a884..72d199a 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -63,6 +63,7 @@ nav: - '@plexusone/invites': packages/invites.md - '@plexusone/pages': packages/pages.md - '@plexusone/shell': packages/shell.md + - '@plexusone/speech': packages/speech.md - '@plexusone/telemetry': packages/telemetry.md - '@plexusone/tenant': packages/tenant.md - '@plexusone/ui': packages/ui.md diff --git a/packages/speech/README.md b/packages/speech/README.md new file mode 100644 index 0000000..5271a60 --- /dev/null +++ b/packages/speech/README.md @@ -0,0 +1,24 @@ +# @plexusone/speech + +Text-to-speech and speech-to-text with the browser's own speech, with React +hooks and a standalone build for pages without a build step. + +Part of [SystemForge Web](https://github.com/plexusone/systemforge-web), web +packages for applications built on the +[SystemForge](https://github.com/plexusone/systemforge) platform. + +## Install + +```bash +pnpm add @plexusone/speech +``` + +React is needed only for `@plexusone/speech/react`. + +## Documentation + +See the [documentation](https://plexusone.github.io/systemforge-web/packages/speech/) for usage and the API reference. + +## License + +MIT From c160f1768d427cc3c85a840f922dd6274b31e4e6 Mon Sep 17 00:00:00 2001 From: John Wang Date: Mon, 2 Nov 2026 07:57:35 -0800 Subject: [PATCH 04/16] feat(speech): report spoken words for highlighting speak takes onWord, called with each word's segment and its start and end offsets in the text passed, mapped back through chunking and whitespace collapsing. A boundary without a length runs to the next space or punctuation. useSpeech keeps the current word in word when trackWords is set; it is off by default because it re-renders on every word. Refs: RMI-SYSTEMFORGEWEB-031 Co-Authored-By: Claude Opus 5.5 --- packages/speech/src/index.ts | 1 + packages/speech/src/react.ts | 38 +++++++++++++++--- packages/speech/src/synthesis.ts | 66 ++++++++++++++++++++++++++++---- packages/speech/src/text.ts | 51 +++++++++++++++++++++++- 4 files changed, 142 insertions(+), 14 deletions(-) diff --git a/packages/speech/src/index.ts b/packages/speech/src/index.ts index 8383038..0060397 100644 --- a/packages/speech/src/index.ts +++ b/packages/speech/src/index.ts @@ -14,6 +14,7 @@ export type { SpeakOptions, SpeakResult, SpeakSegment, + SpeakWord, Speaking, SpeechState, SynthesisEnv, diff --git a/packages/speech/src/react.ts b/packages/speech/src/react.ts index 5b7ff47..569c17a 100644 --- a/packages/speech/src/react.ts +++ b/packages/speech/src/react.ts @@ -7,6 +7,7 @@ import type { SpeakInput, SpeakOptions, SpeakResult, + SpeakWord, Speaking, SpeechState, SynthesisEnv, @@ -28,9 +29,17 @@ export function useVoices(lang?: string, env: SynthesisEnv = {}): SpeechSynthesi return voices; } +export interface UseSpeechOptions extends SpeakOptions { + /** + * Track the word being spoken in `word`, for highlighting. Off by default: + * it re-renders the component on every word. + */ + trackWords?: boolean; +} + export interface UseSpeechResult { /** Speaks, stopping anything already speaking. Options add to the hook's. */ - speak: (input: SpeakInput, options?: SpeakOptions) => Promise; + speak: (input: SpeakInput, options?: UseSpeechOptions) => Promise; stop: () => void; pause: () => void; resume: () => void; @@ -38,14 +47,20 @@ export interface UseSpeechResult { state: SpeechState | 'idle'; /** The segment being spoken, or -1. */ segment: number; + /** + * The word being spoken, with `trackWords`; undefined between segments, + * after speech, and with voices that do not report words. + */ + word: SpeakWord | undefined; error: SpeechError | undefined; supported: boolean; } /** Text-to-speech in a component. Speech stops when the component unmounts. */ -export function useSpeech(defaults: SpeakOptions = {}): UseSpeechResult { +export function useSpeech(defaults: UseSpeechOptions = {}): UseSpeechResult { const [state, setState] = useState('idle'); const [segment, setSegment] = useState(-1); + const [word, setWord] = useState(); const [error, setError] = useState(); const handle = useRef(undefined); // Identifies the latest speech, so callbacks from earlier speech are ignored. @@ -57,18 +72,28 @@ export function useSpeech(defaults: SpeakOptions = {}): UseSpeechResult { useEffect(() => () => handle.current?.stop(), []); - const run = useCallback((input: SpeakInput, options: SpeakOptions = {}) => { - const merged = { ...defaultsRef.current, ...options }; + const run = useCallback((input: SpeakInput, options: UseSpeechOptions = {}) => { + const { trackWords, ...merged } = { ...defaultsRef.current, ...options }; const id = ++seq.current; const latest = () => seq.current === id; setError(undefined); setSegment(-1); + setWord(undefined); const h = speak(input, { ...merged, onSegment: (i, seg) => { - if (latest()) setSegment(i); + if (latest()) { + setSegment(i); + setWord(undefined); + } merged.onSegment?.(i, seg); }, + onWord: trackWords + ? (w) => { + if (latest()) setWord(w); + merged.onWord?.(w); + } + : merged.onWord, }); handle.current = h; setState(h.state); @@ -77,6 +102,7 @@ export function useSpeech(defaults: SpeakOptions = {}): UseSpeechResult { if (latest()) { setState(r.status); setSegment(-1); + setWord(undefined); } return r; }, @@ -84,6 +110,7 @@ export function useSpeech(defaults: SpeakOptions = {}): UseSpeechResult { if (latest()) { setState('error'); setSegment(-1); + setWord(undefined); setError(e); } throw e; @@ -108,6 +135,7 @@ export function useSpeech(defaults: SpeakOptions = {}): UseSpeechResult { resume, state, segment, + word, error, supported: isSynthesisSupported(defaults), }; diff --git a/packages/speech/src/synthesis.ts b/packages/speech/src/synthesis.ts index 308d7be..19bcd86 100644 --- a/packages/speech/src/synthesis.ts +++ b/packages/speech/src/synthesis.ts @@ -1,5 +1,5 @@ import { SpeechError } from './errors'; -import { DEFAULT_MAX_CHUNK_CHARS, splitText } from './text'; +import { chunkText, DEFAULT_MAX_CHUNK_CHARS } from './text'; /** A piece of text to speak, and the silence after it. */ export interface SpeakSegment { @@ -34,6 +34,17 @@ export interface SynthesisEnv { Utterance?: UtteranceConstructor; } +/** + * A word being spoken: `start` and `end` are UTF-16 offsets into the text of + * segment `segment`, as passed to speak, so `text.slice(start, end)` is the + * word. + */ +export interface SpeakWord { + segment: number; + start: number; + end: number; +} + export interface SpeakOptions extends SynthesisEnv { /** BCP 47 language, such as `en-US`. Picks a voice when `voice` is not set. */ lang?: string; @@ -56,6 +67,11 @@ export interface SpeakOptions extends SynthesisEnv { startTimeoutMs?: number; /** Called as each segment starts, including one that is only a pause. */ onSegment?: (index: number, segment: SpeakSegment) => void; + /** + * Called as each word is spoken, for highlighting. Only some voices report + * words; with others this is never called, and speech is the same. + */ + onWord?: (word: SpeakWord) => void; } export type SpeechState = 'speaking' | 'paused' | 'ended' | 'stopped' | 'error'; @@ -80,7 +96,7 @@ export interface Speaking { export const DEFAULT_START_TIMEOUT_MS = 3000; type Step = - | { kind: 'say'; text: string; segment: number; first: boolean } + | { kind: 'say'; text: string; source: number[]; segment: number; first: boolean } | { kind: 'wait'; ms: number; segment: number; first: boolean }; // The browser speaks one thing at a time, so starting new speech stops the @@ -121,8 +137,8 @@ function plan(segments: readonly SpeakSegment[], maxChars: number): Step[] { const steps: Step[] = []; segments.forEach((seg, segment) => { let first = true; - for (const text of splitText(seg.text ?? '', maxChars)) { - steps.push({ kind: 'say', text, segment, first }); + for (const { text, source } of chunkText(seg.text ?? '', maxChars)) { + steps.push({ kind: 'say', text, source, segment, first }); first = false; } const ms = Math.max(0, seg.pauseMs ?? 0); @@ -237,11 +253,12 @@ export function speak(input: SpeakInput, options: SpeakOptions = {}): Speaking { startWait(); return; } - say(s.text, false); + say(s, false); } // Speaks one utterance. A retry is the second try after it failed to start. - function say(text: string, retry: boolean) { + function say(s: Extract, retry: boolean) { + const { text } = s; const u = new Utterance!(text); if (voice) u.voice = voice; const lang = options.lang ?? voice?.lang; @@ -252,6 +269,14 @@ export function speak(input: SpeakInput, options: SpeakOptions = {}): Speaking { u.onstart = () => { if (utterance === u) clearStartTimer(); }; + const onWord = options.onWord; + if (onWord) { + u.onboundary = (e: SpeechSynthesisEvent) => { + if (utterance !== u || e.name === 'sentence') return; + const word = locateWord(s, e.charIndex, e.charLength); + if (word) onWord(word); + }; + } u.onend = () => { if (utterance !== u) return; clearStartTimer(); @@ -287,7 +312,7 @@ export function speak(input: SpeakInput, options: SpeakOptions = {}): Speaking { ); return; } - say(text, true); + say(s, true); }, startTimeoutMs); } } @@ -332,7 +357,7 @@ export function speak(input: SpeakInput, options: SpeakOptions = {}): Speaking { if (repeat) { repeat = false; const s = steps[next - 1]; - if (s.kind === 'say') say(s.text, false); + if (s.kind === 'say') say(s, false); } else if (waitLeft > 0) startWait(); else step(); } @@ -341,6 +366,31 @@ export function speak(input: SpeakInput, options: SpeakOptions = {}): Speaking { return handle; } +// Characters that end a word when the browser gives no word length. +const WORD_END = /[\s.,;:!?…"')\]]/; + +// Maps a word the browser reports in an utterance back to the segment's +// text. Some browsers (Safari) report no length; the word then runs to the +// next space or punctuation. +function locateWord( + s: { text: string; source: number[]; segment: number }, + charIndex: number, + charLength: number | undefined +): SpeakWord | undefined { + if (!(charIndex >= 0 && charIndex < s.text.length)) return undefined; + if (s.text[charIndex] === ' ') { + charLength = charLength ? charLength - 1 : charLength; + charIndex++; + if (charIndex >= s.text.length) return undefined; + } + let end = charLength && charLength > 0 ? Math.min(charIndex + charLength, s.text.length) : 0; + if (!end) { + end = charIndex + 1; + while (end < s.text.length && !WORD_END.test(s.text[end])) end++; + } + return { segment: s.segment, start: s.source[charIndex], end: s.source[end - 1] + 1 }; +} + /** Stops any speech this package started. */ export function stopSpeaking(): void { current?.stop(); diff --git a/packages/speech/src/text.ts b/packages/speech/src/text.ts index b49d39b..00022b3 100644 --- a/packages/speech/src/text.ts +++ b/packages/speech/src/text.ts @@ -12,8 +12,57 @@ export const DEFAULT_MAX_CHUNK_CHARS = 180; * word longer than `maxChars` is kept whole. Whitespace is collapsed. */ export function splitText(text: string, maxChars: number = DEFAULT_MAX_CHUNK_CHARS): string[] { - const t = text.replace(/\s+/g, ' ').trim(); + return chunkText(text, maxChars).map((c) => c.text); +} + +/** A chunk of text to speak, and where each of its characters came from. */ +export interface TextChunk { + text: string; + /** For each character of `text`, its index in the original text. */ + source: number[]; +} + +/** + * Splits text like splitText, keeping for each chunk the position of every + * character in the original text, so a position the browser reports in a + * chunk can be mapped back to the text the caller passed. + */ +export function chunkText(text: string, maxChars: number = DEFAULT_MAX_CHUNK_CHARS): TextChunk[] { + const { t, source } = collapse(text); if (!t) return []; + // Each chunk is a run of t: its parts are adjacent in t and were joined + // with the single space that separated them there. + let from = 0; + return split(t, maxChars).map((chunk) => { + const at = t.indexOf(chunk, from); + from = at + chunk.length; + return { text: chunk, source: source.slice(at, at + chunk.length) }; + }); +} + +// Collapses runs of whitespace to one space and trims, recording the +// original index of every character kept. +function collapse(text: string): { t: string; source: number[] } { + let t = ''; + const source: number[] = []; + let space = false; + for (let i = 0; i < text.length; i++) { + if (/\s/.test(text[i])) { + space = t.length > 0; + continue; + } + if (space) { + t += ' '; + source.push(i - 1); + space = false; + } + t += text[i]; + source.push(i); + } + return { t, source }; +} + +function split(t: string, maxChars: number): string[] { if (maxChars <= 0 || t.length <= maxChars) return [t]; const parts: string[] = []; From 170ba5c845d58bd1220c2213ba5502d8ac95769f Mon Sep 17 00:00:00 2001 From: John Wang Date: Mon, 2 Nov 2026 07:57:35 -0800 Subject: [PATCH 05/16] test(speech): cover word offsets, missing lengths and trackWords Refs: RMI-SYSTEMFORGEWEB-031 Co-Authored-By: Claude Opus 5.5 --- packages/speech/src/__tests__/fakes.ts | 5 ++ packages/speech/src/__tests__/react.test.tsx | 42 +++++++++++++++ .../speech/src/__tests__/synthesis.test.ts | 51 +++++++++++++++++++ packages/speech/src/__tests__/text.test.ts | 15 ++++++ 4 files changed, 113 insertions(+) diff --git a/packages/speech/src/__tests__/fakes.ts b/packages/speech/src/__tests__/fakes.ts index 513df23..9ae87ba 100644 --- a/packages/speech/src/__tests__/fakes.ts +++ b/packages/speech/src/__tests__/fakes.ts @@ -17,6 +17,7 @@ export class FakeUtterance { onstart: (() => void) | null = null; onend: (() => void) | null = null; onerror: ((e: { error: string }) => void) | null = null; + onboundary: ((e: { name: string; charIndex: number; charLength?: number }) => void) | null = null; constructor(text: string) { this.text = text; @@ -71,6 +72,10 @@ export class FakeSynthesis implements SynthesisLike { this.current = undefined; u?.onend?.(); } + /** Reports a boundary in the current utterance, as the browser does. */ + boundary(charIndex: number, charLength?: number, name = 'word') { + this.current?.onboundary?.({ name, charIndex, charLength }); + } fail(error: string) { const u = this.current; this.current = undefined; diff --git a/packages/speech/src/__tests__/react.test.tsx b/packages/speech/src/__tests__/react.test.tsx index 2033c1f..47b0fec 100644 --- a/packages/speech/src/__tests__/react.test.tsx +++ b/packages/speech/src/__tests__/react.test.tsx @@ -28,6 +28,48 @@ describe('useSpeech', () => { expect(result.current.segment).toBe(-1); }); + it('tracks the word being spoken with trackWords', async () => { + const synthesis = new FakeSynthesis(); + const { result } = renderHook(() => useSpeech({ synthesis, Utterance, trackWords: true })); + let done!: Promise; + act(() => { + done = result.current.speak([{ text: 'Hello there.' }, { text: 'Bye.' }]); + }); + expect(result.current.word).toBeUndefined(); + act(() => synthesis.boundary(6, 5)); + expect(result.current.word).toEqual({ segment: 0, start: 6, end: 11 }); + + act(() => synthesis.end()); + expect(result.current.segment).toBe(1); + expect(result.current.word).toBeUndefined(); + act(() => synthesis.boundary(0, 3)); + expect(result.current.word).toEqual({ segment: 1, start: 0, end: 3 }); + + await act(async () => { + synthesis.end(); + await done; + }); + expect(result.current.word).toBeUndefined(); + }); + + it('leaves word unset without trackWords, but still calls onWord', () => { + const synthesis = new FakeSynthesis(); + const seen: number[] = []; + let renders = 0; + const { result } = renderHook(() => { + renders++; + return useSpeech({ synthesis, Utterance, onWord: (w) => seen.push(w.start) }); + }); + act(() => { + void result.current.speak('Hello there.'); + }); + const before = renders; + act(() => synthesis.boundary(6, 5)); + expect(seen).toEqual([6]); + expect(result.current.word).toBeUndefined(); + expect(renders).toBe(before); + }); + it('reports an error', async () => { const synthesis = new FakeSynthesis(); const { result } = renderHook(() => useSpeech({ synthesis, Utterance })); diff --git a/packages/speech/src/__tests__/synthesis.test.ts b/packages/speech/src/__tests__/synthesis.test.ts index da81b2e..692d825 100644 --- a/packages/speech/src/__tests__/synthesis.test.ts +++ b/packages/speech/src/__tests__/synthesis.test.ts @@ -59,6 +59,57 @@ describe('speak', () => { expect(synthesis.texts()).toEqual(['One.']); }); + it('reports words as offsets into the text passed, across chunks', () => { + const words: string[] = []; + const text = 'One two. Three four.'; + speak([{ text: 'Intro.' }, { text }], { + ...env(), + maxChunkChars: 10, + onWord: (w) => words.push(`${w.segment}:${text.slice(w.start, w.end)}`), + }); + synthesis.end(); // Intro. + expect(synthesis.current?.text).toBe('One two.'); + synthesis.boundary(4, 3); // "two" in the collapsed chunk + synthesis.boundary(0, 8, 'sentence'); // sentence boundaries are not words + synthesis.end(); + expect(synthesis.current?.text).toBe('Three'); + synthesis.end(); + synthesis.boundary(0, 4); // "four" + expect(words).toEqual(['1:two', '1:four']); + }); + + it('finds the word end when the browser reports no length', () => { + const words: Array<[number, number]> = []; + speak('Hello, wide world.', { ...env(), onWord: (w) => words.push([w.start, w.end]) }); + synthesis.boundary(0); + synthesis.boundary(6); // the space before "wide" + synthesis.boundary(12, 0); + synthesis.boundary(99); + expect(words).toEqual([ + [0, 5], + [7, 11], + [12, 17], + ]); + }); + + it('ignores words from an utterance that is no longer spoken', () => { + const words: number[] = []; + const s = speak('One. Two.', { + ...env(), + maxChunkChars: 4, + onWord: (w) => words.push(w.start), + }); + const first = synthesis.current!; + s.stop(); + first.onboundary?.({ name: 'word', charIndex: 0, charLength: 3 }); + expect(words).toEqual([]); + }); + + it('listens for words only when asked', () => { + speak('One.', env()); + expect(synthesis.current?.onboundary).toBeNull(); + }); + it('splits long text into several utterances', async () => { const s = speak('One two. Three four. Five six seven.', { ...env(), maxChunkChars: 20 }); synthesis.end(); diff --git a/packages/speech/src/__tests__/text.test.ts b/packages/speech/src/__tests__/text.test.ts index 36884a3..f57746e 100644 --- a/packages/speech/src/__tests__/text.test.ts +++ b/packages/speech/src/__tests__/text.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from 'vitest'; import { splitText } from '../index'; +import { chunkText } from '../text'; describe('splitText', () => { it('keeps short text whole and collapses whitespace', () => { @@ -42,3 +43,17 @@ describe('splitText', () => { expect(splitText(long, 0)).toEqual([long]); }); }); + +describe('chunkText', () => { + it('maps every character of a chunk back to the original text', () => { + const text = ' One two.\n\nThree four. '; + const chunks = chunkText(text, 10); + expect(chunks.map((c) => c.text)).toEqual(['One two.', 'Three', 'four.']); + for (const c of chunks) { + [...c.text].forEach((ch, i) => { + if (ch !== ' ') expect(text[c.source[i]]).toBe(ch); + }); + } + expect(chunks[1].source[0]).toBe(text.indexOf('Three')); + }); +}); From 5d862d15da7f6a04ff47b6dbb5ed948bddbe643c Mon Sep 17 00:00:00 2001 From: John Wang Date: Mon, 2 Nov 2026 07:57:36 -0800 Subject: [PATCH 06/16] docs(speech): describe highlighting words Refs: RMI-SYSTEMFORGEWEB-031 Co-Authored-By: Claude Opus 5.5 --- docs/packages/speech.md | 46 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/docs/packages/speech.md b/docs/packages/speech.md index 13f959f..5d23aab 100644 --- a/docs/packages/speech.md +++ b/docs/packages/speech.md @@ -38,6 +38,7 @@ await speech.done; // { status: 'ended' } or { status: 'stopped' } | `voice` | A voice, or a voice's `voiceURI` or `name` | | `rate`, `pitch`, `volume` | As on `SpeechSynthesisUtterance` | | `onSegment` | Called as each segment starts, including one that is only a pause | +| `onWord` | Called as each word is spoken (see Highlighting words) | | `signal` | An `AbortSignal` that stops the speech | | `maxChunkChars` | Longest text per utterance (default 180) | | `startTimeoutMs` | How long an utterance may take to start (default 3000; 0 is off) | @@ -79,6 +80,25 @@ progress. `stopSpeaking()` stops it from anywhere. - **User gestures.** Browsers may refuse speech that is not started from a click or key press; `done` then rejects with code `not-allowed`. +### Highlighting words + +Pass `onWord` to hear about each word as it is spoken. `start` and `end` +are offsets into the text of segment `segment`, exactly as passed to +`speak`, even when the text was split into several utterances or had its +whitespace collapsed: + +```ts +const text = 'Welcome to the course.'; +speak(text, { + onWord: ({ start, end }) => mark(text.slice(start, end)), +}); +``` + +Only some voices report words. Chrome's online voices often do not, so +`onWord` may never be called; speech is the same either way. When the +browser reports where a word starts but not its length, as Safari does, +the word runs to the next space or punctuation. + ## Voices Browsers load voices after the page. `getVoices` waits for them: @@ -137,10 +157,34 @@ function ReadAloud({ text }: { text: string }) { | Hook | Returns | | -------------------- | --------------------------------------------------------------------------------------- | -| `useSpeech(options)` | `speak`, `stop`, `pause`, `resume`, `state`, `segment`, `error`, `supported` | +| `useSpeech(options)` | `speak`, `stop`, `pause`, `resume`, `state`, `segment`, `word`, `error`, `supported` | | `useListen(options)` | `start`, `stop`, `abort`, `listening`, `text`, `final`, `interim`, `error`, `supported` | | `useVoices(lang)` | The voices, empty until loaded | +With `trackWords: true`, `useSpeech` keeps the word being spoken in +`word` (`{ segment, start, end }`), for highlighting; it is undefined +between segments and after speech. It is off by default because it +re-renders the component on every word; `onWord` alone does not: + +```tsx +function Highlighted({ text }: { text: string }) { + const { speak, word } = useSpeech({ trackWords: true }); + return ( +

speak(text)}> + {word ? ( + <> + {text.slice(0, word.start)} + {text.slice(word.start, word.end)} + {text.slice(word.end)} + + ) : ( + text + )} +

+ ); +} +``` + Speech and listening stop when the component unmounts. The promises from `speak` and `start` reject on errors, which the hooks also report as `error`. From bf98fd076317eb3f9957fccd2f8a46c805fadfc4 Mon Sep 17 00:00:00 2001 From: John Wang Date: Mon, 2 Nov 2026 08:18:21 -0800 Subject: [PATCH 07/16] fix(speech): leave punctuation out of spoken words Chrome includes trailing punctuation in a word's length while Safari gives no length, so the two marked different spans. Punctuation is now trimmed from both ends of every word, a closing apostrophe is kept, and a piece that is only punctuation is not reported. Refs: RMI-SYSTEMFORGEWEB-031 Co-Authored-By: Claude Opus 5.5 --- .../speech/src/__tests__/synthesis.test.ts | 13 ++++++++ packages/speech/src/synthesis.ts | 32 +++++++++++-------- 2 files changed, 32 insertions(+), 13 deletions(-) diff --git a/packages/speech/src/__tests__/synthesis.test.ts b/packages/speech/src/__tests__/synthesis.test.ts index 692d825..90be9b1 100644 --- a/packages/speech/src/__tests__/synthesis.test.ts +++ b/packages/speech/src/__tests__/synthesis.test.ts @@ -92,6 +92,19 @@ describe('speak', () => { ]); }); + it('leaves punctuation out of words', () => { + const text = 'Promote, (twice!). It grew 3.5%, "users\' guess."'; + const words: string[] = []; + speak(text, { ...env(), onWord: (w) => words.push(text.slice(w.start, w.end)) }); + synthesis.boundary(0, 8); // "Promote," as Chrome reports it + synthesis.boundary(9, 9); // "(twice!)." + synthesis.boundary(27, 5); // "3.5%," + synthesis.boundary(33, 7); // the quote and "users'" + synthesis.boundary(41, 7); // "guess." and the quote + synthesis.boundary(17, 1); // "." alone, as some voices report it + expect(words).toEqual(['Promote', 'twice', '3.5%', "users'", 'guess']); + }); + it('ignores words from an utterance that is no longer spoken', () => { const words: number[] = []; const s = speak('One. Two.', { diff --git a/packages/speech/src/synthesis.ts b/packages/speech/src/synthesis.ts index 19bcd86..7fff5c4 100644 --- a/packages/speech/src/synthesis.ts +++ b/packages/speech/src/synthesis.ts @@ -368,27 +368,33 @@ export function speak(input: SpeakInput, options: SpeakOptions = {}): Speaking { // Characters that end a word when the browser gives no word length. const WORD_END = /[\s.,;:!?…"')\]]/; +// Punctuation trimmed from either end of a word, so every browser marks the +// same span: Chrome includes it in the word's length, Safari gives no length. +// An apostrophe is kept at the end, as in "users'". +const LEADING = /[([{"'“‘¿¡]/; +const TRAILING = /[.,;:!?…)\]}"”]/; // Maps a word the browser reports in an utterance back to the segment's -// text. Some browsers (Safari) report no length; the word then runs to the -// next space or punctuation. +// text, without surrounding punctuation. A word that is only punctuation, +// as some voices report, is no word. function locateWord( s: { text: string; source: number[]; segment: number }, charIndex: number, charLength: number | undefined ): SpeakWord | undefined { - if (!(charIndex >= 0 && charIndex < s.text.length)) return undefined; - if (s.text[charIndex] === ' ') { - charLength = charLength ? charLength - 1 : charLength; - charIndex++; - if (charIndex >= s.text.length) return undefined; + const { text } = s; + if (!(charIndex >= 0 && charIndex < text.length)) return undefined; + let start = charIndex; + let end = charLength && charLength > 0 ? Math.min(charIndex + charLength, text.length) : 0; + while (start < text.length && text[start] === ' ') start++; + if (!end || end <= start) { + end = start + 1; + while (end < text.length && !WORD_END.test(text[end])) end++; } - let end = charLength && charLength > 0 ? Math.min(charIndex + charLength, s.text.length) : 0; - if (!end) { - end = charIndex + 1; - while (end < s.text.length && !WORD_END.test(s.text[end])) end++; - } - return { segment: s.segment, start: s.source[charIndex], end: s.source[end - 1] + 1 }; + while (start < end && LEADING.test(text[start])) start++; + while (end > start && TRAILING.test(text[end - 1])) end--; + if (start >= end) return undefined; + return { segment: s.segment, start: s.source[start], end: s.source[end - 1] + 1 }; } /** Stops any speech this package started. */ From 515b673aae670c399a6cf8a4a5cb356cc23fa2ee Mon Sep 17 00:00:00 2001 From: John Wang Date: Mon, 2 Nov 2026 08:18:21 -0800 Subject: [PATCH 08/16] docs(speech): describe punctuation and voice differences in highlighting Refs: RMI-SYSTEMFORGEWEB-031 Co-Authored-By: Claude Opus 5.5 --- docs/packages/speech.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/packages/speech.md b/docs/packages/speech.md index 5d23aab..0cdfaee 100644 --- a/docs/packages/speech.md +++ b/docs/packages/speech.md @@ -94,10 +94,16 @@ speak(text, { }); ``` +A word is marked without the punctuation around it ("promote," marks +"promote"); a piece that is only punctuation is not reported. Numbers and +contractions such as "3.5%" and "didn't" are usually one word. + Only some voices report words. Chrome's online voices often do not, so `onWord` may never be called; speech is the same either way. When the browser reports where a word starts but not its length, as Safari does, -the word runs to the next space or punctuation. +the word runs to the next space or punctuation. Voices differ in what they +call a word: some report pieces of a word, join two short words, or skip a +word they spell out, so the mark can lag for a word without drifting. ## Voices From c636e767de97d54aa19ecb3f50952bb8fc302dd5 Mon Sep 17 00:00:00 2001 From: John Wang Date: Mon, 2 Nov 2026 08:20:14 -0800 Subject: [PATCH 09/16] fix(speech): report only pieces with a letter or digit as words A dash, an ellipsis or a symbol spoken on its own was reported as a word. A word now needs a letter or digit, and dashes are trimmed from its ends like other punctuation. Refs: RMI-SYSTEMFORGEWEB-031 Co-Authored-By: Claude Opus 5.5 --- docs/packages/speech.md | 5 +++-- packages/speech/src/__tests__/synthesis.test.ts | 10 ++++++++++ packages/speech/src/synthesis.ts | 11 +++++++---- 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/docs/packages/speech.md b/docs/packages/speech.md index 0cdfaee..71f562b 100644 --- a/docs/packages/speech.md +++ b/docs/packages/speech.md @@ -95,8 +95,9 @@ speak(text, { ``` A word is marked without the punctuation around it ("promote," marks -"promote"); a piece that is only punctuation is not reported. Numbers and -contractions such as "3.5%" and "didn't" are usually one word. +"promote"), and a piece with no letter or digit, such as a dash, is not +reported. Numbers and contractions such as "3.5%" and "didn't" are usually +one word. Only some voices report words. Chrome's online voices often do not, so `onWord` may never be called; speech is the same either way. When the diff --git a/packages/speech/src/__tests__/synthesis.test.ts b/packages/speech/src/__tests__/synthesis.test.ts index 90be9b1..6c2fc15 100644 --- a/packages/speech/src/__tests__/synthesis.test.ts +++ b/packages/speech/src/__tests__/synthesis.test.ts @@ -105,6 +105,16 @@ describe('speak', () => { expect(words).toEqual(['Promote', 'twice', '3.5%', "users'", 'guess']); }); + it('does not report dashes, ellipses or symbols as words', () => { + const text = 'Plan: 3.5% — fine – well… & so-called'; + const words: string[] = []; + speak(text, { ...env(), onWord: (w) => words.push(text.slice(w.start, w.end)) }); + for (const w of ['3.5%', '—', 'fine', '–', 'well…', '&', 'so-called']) { + synthesis.boundary(text.indexOf(w), w.length); + } + expect(words).toEqual(['3.5%', 'fine', 'well', 'so-called']); + }); + it('ignores words from an utterance that is no longer spoken', () => { const words: number[] = []; const s = speak('One. Two.', { diff --git a/packages/speech/src/synthesis.ts b/packages/speech/src/synthesis.ts index 7fff5c4..206ea93 100644 --- a/packages/speech/src/synthesis.ts +++ b/packages/speech/src/synthesis.ts @@ -371,11 +371,14 @@ const WORD_END = /[\s.,;:!?…"')\]]/; // Punctuation trimmed from either end of a word, so every browser marks the // same span: Chrome includes it in the word's length, Safari gives no length. // An apostrophe is kept at the end, as in "users'". -const LEADING = /[([{"'“‘¿¡]/; -const TRAILING = /[.,;:!?…)\]}"”]/; +const LEADING = /[([{"'“‘¿¡–—-]/; +const TRAILING = /[.,;:!?…)\]}"”–—-]/; +// A word has at least one letter or digit; anything else, such as a dash +// or an ellipsis spoken as a pause, is not reported. +const WORDLIKE = /[\p{L}\p{N}]/u; // Maps a word the browser reports in an utterance back to the segment's -// text, without surrounding punctuation. A word that is only punctuation, +// text, without surrounding punctuation. A piece with no letter or digit, // as some voices report, is no word. function locateWord( s: { text: string; source: number[]; segment: number }, @@ -393,7 +396,7 @@ function locateWord( } while (start < end && LEADING.test(text[start])) start++; while (end > start && TRAILING.test(text[end - 1])) end--; - if (start >= end) return undefined; + if (start >= end || !WORDLIKE.test(text.slice(start, end))) return undefined; return { segment: s.segment, start: s.source[start], end: s.source[end - 1] + 1 }; } From 59525dabf3e19a589f99a5797ea17fcb9c54255a Mon Sep 17 00:00:00 2001 From: John Wang Date: Mon, 2 Nov 2026 08:43:42 -0800 Subject: [PATCH 10/16] docs(guides): add a guide to adding read-aloud and voice input Refs: RMI-SYSTEMFORGEWEB-031 Co-Authored-By: Claude Opus 5.5 --- docs/guides/read-aloud.md | 206 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 206 insertions(+) create mode 100644 docs/guides/read-aloud.md diff --git a/docs/guides/read-aloud.md b/docs/guides/read-aloud.md new file mode 100644 index 0000000..c1f10f7 --- /dev/null +++ b/docs/guides/read-aloud.md @@ -0,0 +1,206 @@ +# Adding read-aloud and voice input + +`@plexusone/speech` lets an application read text aloud and take dictation +with the browser's own speech: no service, no key and no cost. This guide +adds a read-aloud control with word highlighting, a voice picker and voice +input to an application. The [package page](../packages/speech.md) is the +reference for every option. + +## 1. Choose an entry point + +| You have | Use | +| ------------------------------------------ | ---------------------------------------------------------------- | +| A React application | `@plexusone/speech/react`: `useSpeech`, `useListen`, `useVoices` | +| Code without React | `@plexusone/speech`: `speak`, `listen`, `getVoices` | +| A page with no build step, or a Go program | `@plexusone/speech/browser`: one file defining `PlexusSpeech` | + +```bash +pnpm add @plexusone/speech +``` + +The main entry point has no dependencies. React is needed only for the +hooks. + +## 2. Read text aloud + +Pass the text as segments when it has natural parts, such as the sentences +of a lesson or the paragraphs of an article. Each segment can have a pause +after it, and `segment` says which one is being read: + +```tsx +import { useSpeech } from '@plexusone/speech/react'; + +function ReadAloud({ paragraphs }: { paragraphs: string[] }) { + const { speak, stop, pause, resume, state, segment, supported } = useSpeech({ + lang: 'en-US', + }); + if (!supported) return null; + + const segments = paragraphs.map((text) => ({ text, pauseMs: 400 })); + return ( + <> + {state === 'speaking' ? ( + + ) : state === 'paused' ? ( + + ) : ( + + )} + {(state === 'speaking' || state === 'paused') && } + {paragraphs.map((p, i) => ( +

+ {p} +

+ ))} + + ); +} +``` + +Start speech from a click or a key press. Browsers may refuse speech that a +page starts on its own, and report `not-allowed`. + +Long text is split into short utterances for you, because some voices stop +partway through a long one. Pausing repeats at most the sentence that was +interrupted when speech resumes. + +## 3. Highlight the word being read + +Set `trackWords` and mark `word` in the segment being read. `start` and `end` +are offsets into that segment's text exactly as you passed it: + +```tsx +import type { SpeakWord } from '@plexusone/speech'; + +// In ReadAloud above: useSpeech({ lang: 'en-US', trackWords: true }), and +// render each paragraph with renderParagraph(p, i, segment, word). +function renderParagraph(text: string, index: number, segment: number, word?: SpeakWord) { + if (index !== segment) return

{text}

; + if (!word) + return ( +

+ {text} +

+ ); + return ( +

+ {text.slice(0, word.start)} + {text.slice(word.start, word.end)} + {text.slice(word.end)} +

+ ); +} +``` + +Pick the colours together: a light background on the segment being read +and a stronger one on the word reads well. Two things keep the segment's +mark visible: + +- **No transition on the segment's background.** A short sentence is + spoken in under half a second, so a fade can still be running when the + next one starts. +- **Mark the word with more than a shade,** such as an underline as well as + a background. In dark mode two backgrounds of one hue are hard to tell + apart. + +A one-word segment is covered by the word's mark, so only the word shows. + +Words are marked without the punctuation around them. Not every voice +reports words: many of Chrome's online voices report none, and the word +mark never appears while the segment's still does. Some voices report +pieces of a word or join two short words, so treat the mark as a guide to +where the voice is, not an exact transcript. + +`trackWords` re-renders the component on every word. To highlight without +re-rendering, for example by setting a class on a DOM node, pass `onWord` +instead and update the DOM yourself. + +## 4. When speech does not start + +`speak` rejects with a `SpeechError`. Show the text, or offer a recording, +rather than a silent button: + +| `error.code` | What happened | What to show | +| --------------- | ------------------------------------------------ | ---------------------------------------------- | +| `unsupported` | The browser cannot speak (`supported` is false) | Hide the control | +| `not-allowed` | Speech was not started from a click or key press | Start it from a button | +| `start-timeout` | The browser accepted speech but never started it | The text to read, or a recording | +| anything else | The voice failed | The text, and let the person try another voice | + +```tsx +const { error } = useSpeech(); +{ + error?.code === 'start-timeout' && ( +

Your browser's voice isn't responding. Here is the text.

+ ); +} +``` + +## 5. Let people choose a voice + +`useVoices` lists the voices for a language once the browser has loaded +them. Store the chosen voice's `voiceURI`; `voice` accepts it as a string: + +```tsx +import { useVoices } from '@plexusone/speech/react'; + +function VoicePicker({ value, onChange }: { value: string; onChange: (uri: string) => void }) { + const voices = useVoices('en'); + return ( + + ); +} + +// useSpeech({ lang: 'en-US', voice: savedVoiceURI || undefined }) +``` + +Voices differ by browser and device, so a saved voice may be missing on +another computer. Without it, `speak` picks a voice for `lang`: the +browser's default for that language, then a voice on the device. + +## 6. Add voice input to a text field + +`useListen` reports text as it is recognized, final and interim together: + +```tsx +import { useListen } from '@plexusone/speech/react'; + +function Dictation({ onText }: { onText: (text: string) => void }) { + const { start, stop, listening, text, error, supported } = useListen({ lang: 'en-US' }); + if (!supported) return null; + return ( + <> +