From 25ff14e8ec345b00963540c96c69392521c5dd78 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Thu, 16 Jul 2026 12:40:09 +0200 Subject: [PATCH 01/19] feat(speech): wire VAD to Whisper STT and rewrite example app screen --- apps/speech/app/_layout.tsx | 7 + apps/speech/app/index.tsx | 3 + apps/speech/app/stt/index.tsx | 733 ++++++++++++++++++ apps/speech/package.json | 1 + .../speech/tasks/whisperSpeechToText.ts | 260 +++++++ .../src/hooks/useSpeechToText.ts | 44 ++ packages/react-native-executorch/src/index.ts | 2 + .../react-native-executorch/src/models.ts | 111 +++ yarn.lock | 1 + 9 files changed, 1162 insertions(+) create mode 100644 apps/speech/app/stt/index.tsx create mode 100644 packages/react-native-executorch/src/extensions/speech/tasks/whisperSpeechToText.ts create mode 100644 packages/react-native-executorch/src/hooks/useSpeechToText.ts diff --git a/apps/speech/app/_layout.tsx b/apps/speech/app/_layout.tsx index bd4a75cc71..177edf6a0d 100644 --- a/apps/speech/app/_layout.tsx +++ b/apps/speech/app/_layout.tsx @@ -27,6 +27,13 @@ export default function Layout() { title: 'Voice Activity Detection', }} /> + ); } diff --git a/apps/speech/app/index.tsx b/apps/speech/app/index.tsx index 453b8e3863..032768267f 100644 --- a/apps/speech/app/index.tsx +++ b/apps/speech/app/index.tsx @@ -14,6 +14,9 @@ export default function Home() { router.navigate('vad/')}> Voice Activity Detection + router.navigate('stt/')}> + Speech-to-Text (ASR) + ); diff --git a/apps/speech/app/stt/index.tsx b/apps/speech/app/stt/index.tsx new file mode 100644 index 0000000000..f44361d9a4 --- /dev/null +++ b/apps/speech/app/stt/index.tsx @@ -0,0 +1,733 @@ +import React, { useEffect, useRef, useState } from 'react'; +import { + View, + Text, + StyleSheet, + ScrollView, + Platform, + TouchableOpacity, + Switch, +} from 'react-native'; +import { + useSpeechToText, + useVoiceActivityDetection, + models, + WHISPER_SAMPLE_RATE_HZ, + type WhisperLanguage, + type WhisperSttModel, +} from 'react-native-executorch'; +import { AudioContext, AudioManager, AudioRecorder } from 'react-native-audio-api'; +import RNFS from 'react-native-fs'; +import DeviceInfo from 'react-native-device-info'; + +import ScreenWrapper from '../../components/ScreenWrapper'; +import { ModelStatus } from '../../components/ModelStatus'; +import { Button } from '../../components/Button'; +import { theme } from '../../theme'; + +const MODELS: { name: string; config: WhisperSttModel }[] = [ + { name: 'Tiny (EN) CPU', config: models.speechToText.WHISPER.EN.TINY.XNNPACK_FP32 }, + { name: 'Tiny CPU', config: models.speechToText.WHISPER.TINY.XNNPACK_FP32 }, + ...(Platform.OS === 'ios' + ? [ + { name: 'Tiny (EN) CoreML', config: models.speechToText.WHISPER.EN.TINY.COREML_FP16 }, + { name: 'Tiny CoreML', config: models.speechToText.WHISPER.TINY.COREML_FP16 }, + ] + : []), +]; + +const SAMPLE_AUDIOS = [ + { + name: 'English (US)', + url: 'https://www.voiptroubleshooter.com/open_speech/american/OSR_us_000_0010_8k.wav', + language: 'en' as WhisperLanguage, + }, + { + name: 'English (UK)', + url: 'https://www.voiptroubleshooter.com/open_speech/british/OSR_uk_000_0020_8k.wav', + language: 'en' as WhisperLanguage, + }, + { + name: 'French', + url: 'https://www.voiptroubleshooter.com/open_speech/french/OSR_fr_000_0041_8k.wav', + language: 'fr' as WhisperLanguage, + }, + { + name: 'Mandarin (Chinese)', + url: 'https://www.voiptroubleshooter.com/open_speech/chinese/OSR_cn_000_0072_8k.wav', + language: 'zh' as WhisperLanguage, + }, +]; + +const isSimulator = DeviceInfo.isEmulatorSync(); + +function STTContent() { + const [selectedModel, setSelectedModel] = useState(MODELS[0]!); + const [useVadGating, setUseVadGating] = useState(false); + const [activeTab, setActiveTab] = useState<'file' | 'mic'>('file'); + + // Whisper STT Hook + const { + isReady: isSttReady, + downloadProgress: sttProgress, + error: sttError, + transcribe, + stream, + streamInsert, + streamStop, + } = useSpeechToText(selectedModel.config); + + // VAD Hook: only download/load VAD if VAD Gating toggle is enabled + const vad = useVoiceActivityDetection(models.vad.FSMN_VAD, { preventLoad: !useVadGating }); + + // UI State + const [status, setStatus] = useState('Idle'); + const [fileTokens, setFileTokens] = useState([]); + const [committedText, setCommittedText] = useState(''); + const [nonCommittedText, setNonCommittedText] = useState(''); + const [isRecording, setIsRecording] = useState(false); + const [isPlaying, setIsPlaying] = useState(false); + const [audioDownloadProgress, setAudioDownloadProgress] = useState(null); + const [currentAudioBuffer, setCurrentAudioBuffer] = useState(null); + const [hasMicPermission, setHasMicPermission] = useState(false); + const [runError, setRunError] = useState(null); + + const recorderRef = useRef(null); + const activeSourceRef = useRef(null); + + useEffect(() => { + AudioManager.setAudioSessionOptions({ + iosCategory: 'playAndRecord', + iosMode: 'spokenAudio', + iosOptions: ['allowBluetoothHFP', 'defaultToSpeaker'], + }); + AudioManager.requestRecordingPermissions().then((permStatus) => + setHasMicPermission(permStatus === 'Granted') + ); + }, []); + + const handleTabChange = (tab: 'file' | 'mic') => { + if (status.includes('...')) return; + setActiveTab(tab); + setFileTokens([]); + setCommittedText(''); + setNonCommittedText(''); + setStatus('Idle'); + setCurrentAudioBuffer(null); + setRunError(null); + }; + + const handleTranscribeUrl = async (url: string, language: WhisperLanguage) => { + if (activeSourceRef.current) { + try { + activeSourceRef.current.stop(); + } catch (e) {} + activeSourceRef.current = null; + setIsPlaying(false); + } + setCurrentAudioBuffer(null); + setRunError(null); + + if (!isSttReady || !transcribe) return; + + setStatus('Processing...'); + setFileTokens([]); + + const ext = url.split('?')[0]!.split('.').pop()?.toLowerCase() ?? 'wav'; + const localDest = `${RNFS.CachesDirectoryPath}/test_sample.${ext}`; + + try { + // 1. Download audio file + setStatus('Downloading audio...'); + setAudioDownloadProgress(0); + const downloadRes = await RNFS.downloadFile({ + fromUrl: url, + toFile: localDest, + progressInterval: 100, + begin: () => setAudioDownloadProgress(0), + progress: ({ bytesWritten, contentLength }: any) => { + if (contentLength > 0) { + setAudioDownloadProgress(Math.round((bytesWritten / contentLength) * 100)); + } + }, + }).promise; + setAudioDownloadProgress(100); + + if (downloadRes.statusCode !== 200) { + throw new Error(`Download failed with HTTP status ${downloadRes.statusCode}`); + } + + // 2. Decode Audio + setAudioDownloadProgress(null); + setStatus('Decoding audio...'); + const audioContext = new AudioContext({ sampleRate: WHISPER_SAMPLE_RATE_HZ }); + const decodedData = await audioContext.decodeAudioData(localDest); + + setCurrentAudioBuffer(decodedData); + const waveform = decodedData.getChannelData(0); + + await RNFS.unlink(localDest).catch(() => {}); + + // 3. Transcribe + setStatus('Transcribing...'); + const result = await transcribe(waveform, { language }, (token: string) => { + setFileTokens((prev) => [...prev, token]); + }); + + setStatus('Done'); + console.log(`Transcribed: "${result}"`); + } catch (err) { + setStatus('Error'); + setRunError(err instanceof Error ? err.message : String(err)); + setAudioDownloadProgress(null); + await RNFS.unlink(localDest).catch(() => {}); + } + }; + + const togglePlayback = () => { + if (isPlaying && activeSourceRef.current) { + try { + activeSourceRef.current.stop(); + } catch (e) {} + activeSourceRef.current = null; + setIsPlaying(false); + return; + } + if (!currentAudioBuffer) return; + try { + const audioContext = new AudioContext({ sampleRate: WHISPER_SAMPLE_RATE_HZ }); + const source = audioContext.createBufferSource(); + source.buffer = currentAudioBuffer; + source.connect(audioContext.destination); + source.onEnded = () => { + setIsPlaying(false); + activeSourceRef.current = null; + }; + source.start(); + activeSourceRef.current = source; + setIsPlaying(true); + } catch (err) { + setRunError(err instanceof Error ? err.message : String(err)); + } + }; + + const startRecording = async () => { + if (!isSttReady || !stream || !streamInsert || isRecording) return; + + if (!hasMicPermission) { + setRunError('Microphone permission denied. Please enable it in Settings.'); + return; + } + + setRunError(null); + setCommittedText(''); + setNonCommittedText(''); + setStatus('Streaming...'); + setIsRecording(true); + + // Run Whisper STT Streaming Async Loop + (async () => { + try { + console.log('[App] starting STT stream, useVadGating =', useVadGating); + const textStream = stream({ + language: 'en', + vad: useVadGating && vad.detectWorklet ? { detectWorklet: vad.detectWorklet } : undefined, + vadOptions: { speechThreshold: 0.5 }, + }); + for await (const result of textStream) { + setCommittedText(result.committed); + setNonCommittedText(result.nonCommitted); + } + } catch (err) { + console.error(`Streaming error: ${err}`); + setRunError(err instanceof Error ? err.message : String(err)); + } + })(); + + const recorder = new AudioRecorder(); + recorderRef.current = recorder; + + recorder.onAudioReady( + { sampleRate: WHISPER_SAMPLE_RATE_HZ, bufferLength: 4096, channelCount: 1 }, + (event: any) => { + const samples = event.buffer.getChannelData(0); + streamInsert(new Float32Array(samples)); + } + ); + + try { + await AudioManager.setAudioSessionActivity(true); + const result = await recorder.start(); + if (result.status === 'error') { + throw new Error(result.message); + } + } catch (e) { + setRunError(e instanceof Error ? e.message : String(e)); + setIsRecording(false); + recorderRef.current = null; + if (streamStop) streamStop(); + setStatus('Error'); + } + }; + + const stopRecording = async () => { + if (recorderRef.current) { + await recorderRef.current.stop(); + recorderRef.current = null; + } + if (streamStop) streamStop(); + setIsRecording(false); + setStatus('Done'); + }; + + useEffect(() => { + return () => { + recorderRef.current?.stop().catch(() => {}); + if (activeSourceRef.current) { + try { + activeSourceRef.current.stop(); + } catch (e) {} + } + }; + }, []); + + const isModelBusy = status.includes('...'); + const isMicDisabled = isSimulator || !isSttReady || (useVadGating && !vad.isReady); + + return ( + + {/* Model Selector */} + + Whisper Model Selector + + {MODELS.map((item) => { + const isSelected = item.config === selectedModel.config; + return ( + setSelectedModel(item)} + disabled={isModelBusy} + > + + {item.name} + + + ); + })} + + + + {/* Model Status Card */} + + Model Status + + Whisper STT: + + {isSttReady ? 'Ready' : 'Not Loaded'} + + + + + {/* VAD Gating Toggle */} + + + Enable VAD Gating + + Use the FSMN-VAD model to suppress transcription of silences and noise + + + { + if (isModelBusy || isRecording) return; + setUseVadGating(val); + }} + trackColor={{ false: '#d1d1d6', true: theme.colors.accent }} + /> + + + {useVadGating && ( + <> + + FSMN VAD: + + {vad.isReady ? 'Ready' : 'Not Loaded'} + + + + + )} + + + {runError && ( + + {runError} + + )} + + {/* Tab Selector */} + + handleTabChange('file')} + > + + Transcribe File + + + handleTabChange('mic')} + > + + Live Microphone + + + + + {/* File Tab Content */} + {activeTab === 'file' && ( + + Test Audio Files + {SAMPLE_AUDIOS.map((item, idx) => ( + handleTranscribeUrl(item.url, item.language)} + disabled={!isSttReady || isModelBusy} + > + + {item.name} + + {item.url} + + + → + + ))} + + {status !== 'Idle' && ( + + Status: {status} + {audioDownloadProgress !== null && ( + + + + + + Downloading audio... {audioDownloadProgress}% + + + )} + + )} + + {currentAudioBuffer && ( +