From 95b5825275760fc5fbc16e20c326f8cb4290cb26 Mon Sep 17 00:00:00 2001 From: IgorSwat Date: Wed, 15 Jul 2026 14:47:47 +0200 Subject: [PATCH] Add MLX urls & entries for whisper models --- .cspell-wordlist.txt | 1 + apps/speech/screens/SpeechToTextScreen.tsx | 2 +- apps/speech/screens/TextToSpeechScreen.tsx | 165 ++++++++++-------- .../src/constants/modelRegistry.ts | 32 +++- .../src/constants/modelUrls.ts | 6 + 5 files changed, 122 insertions(+), 84 deletions(-) diff --git a/.cspell-wordlist.txt b/.cspell-wordlist.txt index 8d801d3850..564e97ff88 100644 --- a/.cspell-wordlist.txt +++ b/.cspell-wordlist.txt @@ -216,3 +216,4 @@ BLAZEFACE Blazeface blazeface nums +Pressable \ No newline at end of file diff --git a/apps/speech/screens/SpeechToTextScreen.tsx b/apps/speech/screens/SpeechToTextScreen.tsx index 909c2da57f..3a82037040 100644 --- a/apps/speech/screens/SpeechToTextScreen.tsx +++ b/apps/speech/screens/SpeechToTextScreen.tsx @@ -23,7 +23,7 @@ const speechToText = models.speech_to_text; const vad = models.vad; const isSimulator = DeviceInfo.isEmulatorSync(); -const backend = Platform.OS === 'ios' && !isSimulator ? 'coreml' : 'xnnpack'; +const backend = Platform.OS === 'ios' && !isSimulator ? 'mlx' : 'xnnpack'; type STTModelSources = SpeechToTextProps['model']; diff --git a/apps/speech/screens/TextToSpeechScreen.tsx b/apps/speech/screens/TextToSpeechScreen.tsx index 1bdeb9f542..ed0a3fd5df 100644 --- a/apps/speech/screens/TextToSpeechScreen.tsx +++ b/apps/speech/screens/TextToSpeechScreen.tsx @@ -8,6 +8,7 @@ import { KeyboardAvoidingView, Platform, Keyboard, + Pressable, } from 'react-native'; import { SafeAreaProvider, SafeAreaView } from 'react-native-safe-area-context'; import { @@ -276,86 +277,93 @@ export const TextToSpeechScreen = ({ onBack }: { onBack: () => void }) => { style={styles.keyboardAvoidingView} behavior={Platform.OS === 'ios' ? 'padding' : undefined} > - - - - - - React Native ExecuTorch - Text to Speech - - - - Status: {getModelStatus()} - - setError(null)} /> - - - - setSelectedSpeaker(m)} - /> - - {selectedTtsModel === 'supertonic' && ( - <> - setSelectedLang(l)} - /> + + + + + + + React Native ExecuTorch + Text to Speech + + + + Status: {getModelStatus()} + + setError(null)} /> + + - setTotalSteps(s)} - /> - - )} - - - Enter text to synthesize - setSelectedSpeaker(m)} /> - - - - - + setSelectedLang(l)} + /> + + setTotalSteps(s)} + /> + + )} + + + Enter text to synthesize + - - {isPlaying ? 'Playing...' : 'Generate & Play'} - - - + + + + + + + {isPlaying ? 'Playing...' : 'Generate & Play'} + + + + @@ -373,6 +381,9 @@ const styles = StyleSheet.create({ flex: 1, width: '100%', }, + keyboardDismissPressable: { + flex: 1, + }, header: { alignItems: 'center', position: 'relative', diff --git a/packages/react-native-executorch/src/constants/modelRegistry.ts b/packages/react-native-executorch/src/constants/modelRegistry.ts index 270ddf0ecf..7c7c7ef63b 100644 --- a/packages/react-native-executorch/src/constants/modelRegistry.ts +++ b/packages/react-native-executorch/src/constants/modelRegistry.ts @@ -581,6 +581,7 @@ function whisperVariants( isMultilingual: IsML, xnnpackUrl: string, coremlUrl: string, + mlxUrl: string, tokenizerUrl: string ) { return { @@ -610,6 +611,19 @@ function whisperVariants( tokenizerSource: string; }, }, + mlx: { + base: { + modelName, + isMultilingual, + modelSource: mlxUrl, + tokenizerSource: tokenizerUrl, + } as { + modelName: N; + isMultilingual: IsML; + modelSource: string; + tokenizerSource: string; + }, + }, }; } @@ -618,6 +632,7 @@ const WHISPER_TINY_EN_VARIANTS = whisperVariants( false, M.WHISPER_TINY_EN_MODEL_XNNPACK, M.WHISPER_TINY_EN_MODEL_COREML, + M.WHISPER_TINY_EN_MODEL_MLX, M.WHISPER_TINY_EN_TOKENIZER ); const WHISPER_BASE_EN_VARIANTS = whisperVariants( @@ -625,6 +640,7 @@ const WHISPER_BASE_EN_VARIANTS = whisperVariants( false, M.WHISPER_BASE_EN_MODEL_XNNPACK, M.WHISPER_BASE_EN_MODEL_COREML, + M.WHISPER_BASE_EN_MODEL_MLX, M.WHISPER_BASE_EN_TOKENIZER ); const WHISPER_SMALL_EN_VARIANTS = whisperVariants( @@ -632,6 +648,7 @@ const WHISPER_SMALL_EN_VARIANTS = whisperVariants( false, M.WHISPER_SMALL_EN_MODEL_XNNPACK, M.WHISPER_SMALL_EN_MODEL_COREML, + M.WHISPER_SMALL_EN_MODEL_MLX, M.WHISPER_SMALL_EN_TOKENIZER ); const WHISPER_TINY_VARIANTS = whisperVariants( @@ -639,6 +656,7 @@ const WHISPER_TINY_VARIANTS = whisperVariants( true, M.WHISPER_TINY_MODEL_XNNPACK, M.WHISPER_TINY_MODEL_COREML, + M.WHISPER_TINY_MODEL_MLX, M.WHISPER_TINY_TOKENIZER ); const WHISPER_BASE_VARIANTS = whisperVariants( @@ -646,6 +664,7 @@ const WHISPER_BASE_VARIANTS = whisperVariants( true, M.WHISPER_BASE_MODEL_XNNPACK, M.WHISPER_BASE_MODEL_COREML, + M.WHISPER_BASE_MODEL_MLX, M.WHISPER_BASE_TOKENIZER ); const WHISPER_SMALL_VARIANTS = whisperVariants( @@ -653,6 +672,7 @@ const WHISPER_SMALL_VARIANTS = whisperVariants( true, M.WHISPER_SMALL_MODEL_XNNPACK, M.WHISPER_SMALL_MODEL_COREML, + M.WHISPER_SMALL_MODEL_MLX, M.WHISPER_SMALL_TOKENIZER ); @@ -820,12 +840,12 @@ export const models = { udnie: variant(STYLE_TRANSFER_UDNIE_VARIANTS), }, speech_to_text: { - whisper_tiny_en: variant(WHISPER_TINY_EN_VARIANTS), - whisper_base_en: variant(WHISPER_BASE_EN_VARIANTS), - whisper_small_en: variant(WHISPER_SMALL_EN_VARIANTS), - whisper_tiny: variant(WHISPER_TINY_VARIANTS), - whisper_base: variant(WHISPER_BASE_VARIANTS), - whisper_small: variant(WHISPER_SMALL_VARIANTS), + whisper_tiny_en: variant(WHISPER_TINY_EN_VARIANTS, { ios: 'mlx' }), + whisper_base_en: variant(WHISPER_BASE_EN_VARIANTS, { ios: 'mlx' }), + whisper_small_en: variant(WHISPER_SMALL_EN_VARIANTS, { ios: 'mlx' }), + whisper_tiny: variant(WHISPER_TINY_VARIANTS, { ios: 'mlx' }), + whisper_base: variant(WHISPER_BASE_VARIANTS, { ios: 'mlx' }), + whisper_small: variant(WHISPER_SMALL_VARIANTS, { ios: 'mlx' }), }, // Kokoro presets bundle model + voice + phonemizer per language. They go // through `useTextToSpeech` directly — pick one and pass it as the `model`. diff --git a/packages/react-native-executorch/src/constants/modelUrls.ts b/packages/react-native-executorch/src/constants/modelUrls.ts index 8429155dd2..a7ddf9c31a 100644 --- a/packages/react-native-executorch/src/constants/modelUrls.ts +++ b/packages/react-native-executorch/src/constants/modelUrls.ts @@ -855,26 +855,32 @@ export const STYLE_TRANSFER_UDNIE_QUANTIZED = { export const WHISPER_TINY_EN_TOKENIZER = `${URL_PREFIX}-whisper-tiny.en/${VERSION_TAG}/tokenizer.json`; export const WHISPER_TINY_EN_MODEL_XNNPACK = `${URL_PREFIX}-whisper-tiny.en/${VERSION_TAG}/xnnpack/whisper_tiny_en_xnnpack_fp32.pte`; export const WHISPER_TINY_EN_MODEL_COREML = `${URL_PREFIX}-whisper-tiny.en/${VERSION_TAG}/coreml/whisper_tiny_en_coreml_fp16.pte`; +export const WHISPER_TINY_EN_MODEL_MLX = `${URL_PREFIX}-whisper-tiny.en/${VERSION_TAG}/mlx/whisper_tiny_en_mlx_bf16.pte`; export const WHISPER_BASE_EN_TOKENIZER = `${URL_PREFIX}-whisper-base.en/${VERSION_TAG}/tokenizer.json`; export const WHISPER_BASE_EN_MODEL_XNNPACK = `${URL_PREFIX}-whisper-base.en/${VERSION_TAG}/xnnpack/whisper_base_en_xnnpack_fp32.pte`; export const WHISPER_BASE_EN_MODEL_COREML = `${URL_PREFIX}-whisper-base.en/${VERSION_TAG}/coreml/whisper_base_en_coreml_fp16.pte`; +export const WHISPER_BASE_EN_MODEL_MLX = `${URL_PREFIX}-whisper-base.en/${VERSION_TAG}/mlx/whisper_base_en_mlx_bf16.pte`; export const WHISPER_SMALL_EN_TOKENIZER = `${URL_PREFIX}-whisper-small.en/${VERSION_TAG}/tokenizer.json`; export const WHISPER_SMALL_EN_MODEL_XNNPACK = `${URL_PREFIX}-whisper-small.en/${VERSION_TAG}/xnnpack/whisper_small_en_xnnpack_fp32.pte`; export const WHISPER_SMALL_EN_MODEL_COREML = `${URL_PREFIX}-whisper-small.en/${VERSION_TAG}/coreml/whisper_small_en_coreml_fp16.pte`; +export const WHISPER_SMALL_EN_MODEL_MLX = `${URL_PREFIX}-whisper-small.en/${VERSION_TAG}/mlx/whisper_small_en_mlx_bf16.pte`; export const WHISPER_TINY_TOKENIZER = `${URL_PREFIX}-whisper-tiny/${VERSION_TAG}/tokenizer.json`; export const WHISPER_TINY_MODEL_XNNPACK = `${URL_PREFIX}-whisper-tiny/${VERSION_TAG}/xnnpack/whisper_tiny_xnnpack_fp32.pte`; export const WHISPER_TINY_MODEL_COREML = `${URL_PREFIX}-whisper-tiny/${VERSION_TAG}/coreml/whisper_tiny_coreml_fp16.pte`; +export const WHISPER_TINY_MODEL_MLX = `${URL_PREFIX}-whisper-tiny/${VERSION_TAG}/mlx/whisper_tiny_mlx_bf16.pte`; export const WHISPER_BASE_TOKENIZER = `${URL_PREFIX}-whisper-base/${VERSION_TAG}/tokenizer.json`; export const WHISPER_BASE_MODEL_XNNPACK = `${URL_PREFIX}-whisper-base/${VERSION_TAG}/xnnpack/whisper_base_xnnpack_fp32.pte`; export const WHISPER_BASE_MODEL_COREML = `${URL_PREFIX}-whisper-base/${VERSION_TAG}/coreml/whisper_base_coreml_fp16.pte`; +export const WHISPER_BASE_MODEL_MLX = `${URL_PREFIX}-whisper-base/${VERSION_TAG}/mlx/whisper_base_mlx_bf16.pte`; export const WHISPER_SMALL_TOKENIZER = `${URL_PREFIX}-whisper-small/${VERSION_TAG}/tokenizer.json`; export const WHISPER_SMALL_MODEL_XNNPACK = `${URL_PREFIX}-whisper-small/${VERSION_TAG}/xnnpack/whisper_small_xnnpack_fp32.pte`; export const WHISPER_SMALL_MODEL_COREML = `${URL_PREFIX}-whisper-small/${VERSION_TAG}/coreml/whisper_small_coreml_fp16.pte`; +export const WHISPER_SMALL_MODEL_MLX = `${URL_PREFIX}-whisper-small/${VERSION_TAG}/mlx/whisper_small_mlx_bf16.pte`; /** * @category Models - Speech To Text