Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .cspell-wordlist.txt
Original file line number Diff line number Diff line change
Expand Up @@ -216,3 +216,4 @@ BLAZEFACE
Blazeface
blazeface
nums
Pressable
2 changes: 1 addition & 1 deletion apps/speech/screens/SpeechToTextScreen.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ const speechToText = models.speech_to_text;
const vad = models.vad;

const isSimulator = DeviceInfo.isEmulatorSync();
const backend = Platform.OS === 'ios' && !isSimulator ? 'coreml' : 'xnnpack';
const backend = Platform.OS === 'ios' && !isSimulator ? 'mlx' : 'xnnpack';

type STTModelSources = SpeechToTextProps['model'];

Expand Down
165 changes: 88 additions & 77 deletions apps/speech/screens/TextToSpeechScreen.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import {
KeyboardAvoidingView,
Platform,
Keyboard,
Pressable,
} from 'react-native';
import { SafeAreaProvider, SafeAreaView } from 'react-native-safe-area-context';
import {
Expand Down Expand Up @@ -276,86 +277,93 @@ export const TextToSpeechScreen = ({ onBack }: { onBack: () => void }) => {
style={styles.keyboardAvoidingView}
behavior={Platform.OS === 'ios' ? 'padding' : undefined}
>
<View style={styles.header}>
<TouchableOpacity style={styles.backButton} onPress={onBack}>
<FontAwesome name="chevron-left" size={20} color="#0f186e" />
</TouchableOpacity>
<SWMIcon width={60} height={60} />
<Text style={styles.headerText}>React Native ExecuTorch</Text>
<Text style={styles.headerText}>Text to Speech</Text>
</View>

<View style={styles.statusContainer}>
<Text>Status: {getModelStatus()}</Text>
</View>
<ErrorBanner message={error} onDismiss={() => setError(null)} />

<ModelPicker
label="Model"
models={TTS_MODEL_OPTIONS}
selectedModel={selectedTtsModel}
disabled={model.isGenerating || isPlaying}
onSelect={handleSelectTtsModel}
/>

<ModelPicker
label="Voice"
models={selectedTtsModel === 'supertonic' ? VOICES : KOKORO_VOICES}
selectedModel={selectedSpeaker}
disabled={model.isGenerating}
onSelect={(m) => setSelectedSpeaker(m)}
/>

{selectedTtsModel === 'supertonic' && (
<>
<ModelPicker
label="Language"
models={LANGUAGES}
selectedModel={selectedLang}
disabled={isPlaying}
onSelect={(l) => setSelectedLang(l)}
/>
<Pressable
style={styles.keyboardDismissPressable}
onPress={Keyboard.dismiss}
>
<View style={styles.header}>
<TouchableOpacity style={styles.backButton} onPress={onBack}>
<FontAwesome name="chevron-left" size={20} color="#0f186e" />
</TouchableOpacity>
<SWMIcon width={60} height={60} />
<Text style={styles.headerText}>React Native ExecuTorch</Text>
<Text style={styles.headerText}>Text to Speech</Text>
</View>

<View style={styles.statusContainer}>
<Text>Status: {getModelStatus()}</Text>
</View>
<ErrorBanner message={error} onDismiss={() => setError(null)} />

<ModelPicker
label="Model"
models={TTS_MODEL_OPTIONS}
selectedModel={selectedTtsModel}
disabled={model.isGenerating || isPlaying}
onSelect={handleSelectTtsModel}
/>

<ModelPicker
label="Steps"
models={STEPS}
selectedModel={totalSteps}
disabled={isPlaying}
onSelect={(s) => setTotalSteps(s)}
/>
</>
)}

<View style={styles.inputContainer}>
<Text style={styles.inputLabel}>Enter text to synthesize</Text>
<TextInput
placeholder="Type something..."
placeholderTextColor="#aaa"
style={styles.textInput}
value={inputText}
onChangeText={setInputText}
multiline
numberOfLines={4}
textAlignVertical="top"
<ModelPicker
label="Voice"
models={
selectedTtsModel === 'supertonic' ? VOICES : KOKORO_VOICES
}
selectedModel={selectedSpeaker}
disabled={model.isGenerating}
onSelect={(m) => setSelectedSpeaker(m)}
/>
</View>

<View style={styles.buttonContainer}>
<TouchableOpacity
disabled={!readyToGenerate}
onPress={handlePlayAudio}
style={[styles.playButton, !readyToGenerate && styles.disabled]}
>
<FontAwesome
name={isPlaying ? 'volume-up' : 'play'}
size={20}
color="white"

{selectedTtsModel === 'supertonic' && (
<>
<ModelPicker
label="Language"
models={LANGUAGES}
selectedModel={selectedLang}
disabled={isPlaying}
onSelect={(l) => setSelectedLang(l)}
/>

<ModelPicker
label="Steps"
models={STEPS}
selectedModel={totalSteps}
disabled={isPlaying}
onSelect={(s) => setTotalSteps(s)}
/>
</>
)}

<View style={styles.inputContainer}>
<Text style={styles.inputLabel}>Enter text to synthesize</Text>
<TextInput
placeholder="Type something..."
placeholderTextColor="#aaa"
style={styles.textInput}
value={inputText}
onChangeText={setInputText}
multiline
numberOfLines={4}
textAlignVertical="top"
/>
<Text style={styles.buttonText}>
{isPlaying ? 'Playing...' : 'Generate & Play'}
</Text>
</TouchableOpacity>
</View>
</View>

<View style={styles.buttonContainer}>
<TouchableOpacity
disabled={!readyToGenerate}
onPress={handlePlayAudio}
style={[styles.playButton, !readyToGenerate && styles.disabled]}
>
<FontAwesome
name={isPlaying ? 'volume-up' : 'play'}
size={20}
color="white"
/>
<Text style={styles.buttonText}>
{isPlaying ? 'Playing...' : 'Generate & Play'}
</Text>
</TouchableOpacity>
</View>
</Pressable>
</KeyboardAvoidingView>
</SafeAreaView>
</SafeAreaProvider>
Expand All @@ -373,6 +381,9 @@ const styles = StyleSheet.create({
flex: 1,
width: '100%',
},
keyboardDismissPressable: {
flex: 1,
},
header: {
alignItems: 'center',
position: 'relative',
Expand Down
32 changes: 26 additions & 6 deletions packages/react-native-executorch/src/constants/modelRegistry.ts
Original file line number Diff line number Diff line change
Expand Up @@ -581,6 +581,7 @@ function whisperVariants<const N extends string, const IsML extends boolean>(
isMultilingual: IsML,
xnnpackUrl: string,
coremlUrl: string,
mlxUrl: string,
tokenizerUrl: string
) {
return {
Expand Down Expand Up @@ -610,6 +611,19 @@ function whisperVariants<const N extends string, const IsML extends boolean>(
tokenizerSource: string;
},
},
mlx: {
base: {
modelName,
isMultilingual,
modelSource: mlxUrl,
tokenizerSource: tokenizerUrl,
} as {
modelName: N;
isMultilingual: IsML;
modelSource: string;
tokenizerSource: string;
},
},
};
}

Expand All @@ -618,41 +632,47 @@ const WHISPER_TINY_EN_VARIANTS = whisperVariants(
false,
M.WHISPER_TINY_EN_MODEL_XNNPACK,
M.WHISPER_TINY_EN_MODEL_COREML,
M.WHISPER_TINY_EN_MODEL_MLX,
M.WHISPER_TINY_EN_TOKENIZER
);
const WHISPER_BASE_EN_VARIANTS = whisperVariants(
'whisper-base-en',
false,
M.WHISPER_BASE_EN_MODEL_XNNPACK,
M.WHISPER_BASE_EN_MODEL_COREML,
M.WHISPER_BASE_EN_MODEL_MLX,
M.WHISPER_BASE_EN_TOKENIZER
);
const WHISPER_SMALL_EN_VARIANTS = whisperVariants(
'whisper-small-en',
false,
M.WHISPER_SMALL_EN_MODEL_XNNPACK,
M.WHISPER_SMALL_EN_MODEL_COREML,
M.WHISPER_SMALL_EN_MODEL_MLX,
M.WHISPER_SMALL_EN_TOKENIZER
);
const WHISPER_TINY_VARIANTS = whisperVariants(
'whisper-tiny',
true,
M.WHISPER_TINY_MODEL_XNNPACK,
M.WHISPER_TINY_MODEL_COREML,
M.WHISPER_TINY_MODEL_MLX,
M.WHISPER_TINY_TOKENIZER
);
const WHISPER_BASE_VARIANTS = whisperVariants(
'whisper-base',
true,
M.WHISPER_BASE_MODEL_XNNPACK,
M.WHISPER_BASE_MODEL_COREML,
M.WHISPER_BASE_MODEL_MLX,
M.WHISPER_BASE_TOKENIZER
);
const WHISPER_SMALL_VARIANTS = whisperVariants(
'whisper-small',
true,
M.WHISPER_SMALL_MODEL_XNNPACK,
M.WHISPER_SMALL_MODEL_COREML,
M.WHISPER_SMALL_MODEL_MLX,
M.WHISPER_SMALL_TOKENIZER
);

Expand Down Expand Up @@ -820,12 +840,12 @@ export const models = {
udnie: variant(STYLE_TRANSFER_UDNIE_VARIANTS),
},
speech_to_text: {
whisper_tiny_en: variant(WHISPER_TINY_EN_VARIANTS),
whisper_base_en: variant(WHISPER_BASE_EN_VARIANTS),
whisper_small_en: variant(WHISPER_SMALL_EN_VARIANTS),
whisper_tiny: variant(WHISPER_TINY_VARIANTS),
whisper_base: variant(WHISPER_BASE_VARIANTS),
whisper_small: variant(WHISPER_SMALL_VARIANTS),
whisper_tiny_en: variant(WHISPER_TINY_EN_VARIANTS, { ios: 'mlx' }),
whisper_base_en: variant(WHISPER_BASE_EN_VARIANTS, { ios: 'mlx' }),
whisper_small_en: variant(WHISPER_SMALL_EN_VARIANTS, { ios: 'mlx' }),
whisper_tiny: variant(WHISPER_TINY_VARIANTS, { ios: 'mlx' }),
whisper_base: variant(WHISPER_BASE_VARIANTS, { ios: 'mlx' }),
whisper_small: variant(WHISPER_SMALL_VARIANTS, { ios: 'mlx' }),
},
// Kokoro presets bundle model + voice + phonemizer per language. They go
// through `useTextToSpeech` directly — pick one and pass it as the `model`.
Expand Down
6 changes: 6 additions & 0 deletions packages/react-native-executorch/src/constants/modelUrls.ts
Original file line number Diff line number Diff line change
Expand Up @@ -855,26 +855,32 @@ export const STYLE_TRANSFER_UDNIE_QUANTIZED = {
export const WHISPER_TINY_EN_TOKENIZER = `${URL_PREFIX}-whisper-tiny.en/${VERSION_TAG}/tokenizer.json`;
export const WHISPER_TINY_EN_MODEL_XNNPACK = `${URL_PREFIX}-whisper-tiny.en/${VERSION_TAG}/xnnpack/whisper_tiny_en_xnnpack_fp32.pte`;
export const WHISPER_TINY_EN_MODEL_COREML = `${URL_PREFIX}-whisper-tiny.en/${VERSION_TAG}/coreml/whisper_tiny_en_coreml_fp16.pte`;
export const WHISPER_TINY_EN_MODEL_MLX = `${URL_PREFIX}-whisper-tiny.en/${VERSION_TAG}/mlx/whisper_tiny_en_mlx_bf16.pte`;

export const WHISPER_BASE_EN_TOKENIZER = `${URL_PREFIX}-whisper-base.en/${VERSION_TAG}/tokenizer.json`;
export const WHISPER_BASE_EN_MODEL_XNNPACK = `${URL_PREFIX}-whisper-base.en/${VERSION_TAG}/xnnpack/whisper_base_en_xnnpack_fp32.pte`;
export const WHISPER_BASE_EN_MODEL_COREML = `${URL_PREFIX}-whisper-base.en/${VERSION_TAG}/coreml/whisper_base_en_coreml_fp16.pte`;
export const WHISPER_BASE_EN_MODEL_MLX = `${URL_PREFIX}-whisper-base.en/${VERSION_TAG}/mlx/whisper_base_en_mlx_bf16.pte`;

export const WHISPER_SMALL_EN_TOKENIZER = `${URL_PREFIX}-whisper-small.en/${VERSION_TAG}/tokenizer.json`;
export const WHISPER_SMALL_EN_MODEL_XNNPACK = `${URL_PREFIX}-whisper-small.en/${VERSION_TAG}/xnnpack/whisper_small_en_xnnpack_fp32.pte`;
export const WHISPER_SMALL_EN_MODEL_COREML = `${URL_PREFIX}-whisper-small.en/${VERSION_TAG}/coreml/whisper_small_en_coreml_fp16.pte`;
export const WHISPER_SMALL_EN_MODEL_MLX = `${URL_PREFIX}-whisper-small.en/${VERSION_TAG}/mlx/whisper_small_en_mlx_bf16.pte`;

export const WHISPER_TINY_TOKENIZER = `${URL_PREFIX}-whisper-tiny/${VERSION_TAG}/tokenizer.json`;
export const WHISPER_TINY_MODEL_XNNPACK = `${URL_PREFIX}-whisper-tiny/${VERSION_TAG}/xnnpack/whisper_tiny_xnnpack_fp32.pte`;
export const WHISPER_TINY_MODEL_COREML = `${URL_PREFIX}-whisper-tiny/${VERSION_TAG}/coreml/whisper_tiny_coreml_fp16.pte`;
export const WHISPER_TINY_MODEL_MLX = `${URL_PREFIX}-whisper-tiny/${VERSION_TAG}/mlx/whisper_tiny_mlx_bf16.pte`;

export const WHISPER_BASE_TOKENIZER = `${URL_PREFIX}-whisper-base/${VERSION_TAG}/tokenizer.json`;
export const WHISPER_BASE_MODEL_XNNPACK = `${URL_PREFIX}-whisper-base/${VERSION_TAG}/xnnpack/whisper_base_xnnpack_fp32.pte`;
export const WHISPER_BASE_MODEL_COREML = `${URL_PREFIX}-whisper-base/${VERSION_TAG}/coreml/whisper_base_coreml_fp16.pte`;
export const WHISPER_BASE_MODEL_MLX = `${URL_PREFIX}-whisper-base/${VERSION_TAG}/mlx/whisper_base_mlx_bf16.pte`;

export const WHISPER_SMALL_TOKENIZER = `${URL_PREFIX}-whisper-small/${VERSION_TAG}/tokenizer.json`;
export const WHISPER_SMALL_MODEL_XNNPACK = `${URL_PREFIX}-whisper-small/${VERSION_TAG}/xnnpack/whisper_small_xnnpack_fp32.pte`;
export const WHISPER_SMALL_MODEL_COREML = `${URL_PREFIX}-whisper-small/${VERSION_TAG}/coreml/whisper_small_coreml_fp16.pte`;
export const WHISPER_SMALL_MODEL_MLX = `${URL_PREFIX}-whisper-small/${VERSION_TAG}/mlx/whisper_small_mlx_bf16.pte`;

/**
* @category Models - Speech To Text
Expand Down
Loading