From c76f7e44c36442d2c06b435590c93cf907a4c520 Mon Sep 17 00:00:00 2001 From: IgorSwat Date: Mon, 16 Nov 2026 09:34:43 +0100 Subject: [PATCH 1/8] build: bump binaries to v0.11.2 --- packages/react-native-executorch/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/react-native-executorch/package.json b/packages/react-native-executorch/package.json index ccacaad0f4..a5d08fc610 100644 --- a/packages/react-native-executorch/package.json +++ b/packages/react-native-executorch/package.json @@ -1,7 +1,7 @@ { "name": "react-native-executorch", "version": "1.0.0", - "nativeLibsVersion": "0.11.0", + "nativeLibsVersion": "0.11.2", "description": "An easy way to run AI models in React Native with ExecuTorch", "main": "./lib/module/index.js", "module": "./lib/module/index.js", From 457beb285a77143c65a63a45208dda1b64c5b7ef Mon Sep 17 00:00:00 2001 From: IgorSwat <114943112+IgorSwat@users.noreply.github.com> Date: Mon, 16 Nov 2026 16:47:37 +0100 Subject: [PATCH 2/8] Change nativeLibsVersion to 0.11.1 --- packages/react-native-executorch/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/react-native-executorch/package.json b/packages/react-native-executorch/package.json index a5d08fc610..76f7b8bee9 100644 --- a/packages/react-native-executorch/package.json +++ b/packages/react-native-executorch/package.json @@ -1,7 +1,7 @@ { "name": "react-native-executorch", "version": "1.0.0", - "nativeLibsVersion": "0.11.2", + "nativeLibsVersion": "0.11.1", "description": "An easy way to run AI models in React Native with ExecuTorch", "main": "./lib/module/index.js", "module": "./lib/module/index.js", From 41da34d28d6ab8b7b0b05173f2a44bce9ef63f44 Mon Sep 17 00:00:00 2001 From: IgorSwat Date: Thu, 12 Nov 2026 09:20:49 +0100 Subject: [PATCH 3/8] Add Reverb-TTS task & utils code --- .cspell-wordlist.txt | 7 + .../api/__snapshots__/apiSurface.test.ts.snap | 8 + .../extensions/reverbPartitioner.test.ts | 117 +++ .../__tests__/extensions/reverbUtils.test.ts | 216 ++++++ .../tasks/reverbTextToSpeech.test.ts | 514 +++++++++++++ .../react-native-executorch/jest.config.js | 6 + packages/react-native-executorch/package.json | 3 +- .../src/extensions/speech/index.ts | 2 + .../speech/tasks/reverbTextToSpeech.ts | 689 ++++++++++++++++++ .../speech/utils/reverbPartitioner.ts | 92 +++ .../extensions/speech/utils/reverbUtils.ts | 192 +++++ .../speech/utils/textPartitioner.ts | 14 +- .../src/hooks/useTextToSpeech.ts | 48 +- packages/react-native-executorch/src/index.ts | 1 + .../tsconfig.build.json | 2 +- .../react-native-executorch/tsconfig.json | 3 +- yarn.lock | 8 + 17 files changed, 1913 insertions(+), 9 deletions(-) create mode 100644 packages/react-native-executorch/__tests__/extensions/reverbPartitioner.test.ts create mode 100644 packages/react-native-executorch/__tests__/extensions/reverbUtils.test.ts create mode 100644 packages/react-native-executorch/__tests__/tasks/reverbTextToSpeech.test.ts create mode 100644 packages/react-native-executorch/src/extensions/speech/tasks/reverbTextToSpeech.ts create mode 100644 packages/react-native-executorch/src/extensions/speech/utils/reverbPartitioner.ts create mode 100644 packages/react-native-executorch/src/extensions/speech/utils/reverbUtils.ts diff --git a/.cspell-wordlist.txt b/.cspell-wordlist.txt index defed32659..cf5470a0d2 100644 --- a/.cspell-wordlist.txt +++ b/.cspell-wordlist.txt @@ -450,3 +450,10 @@ eurollm doclayout netp Uyghur +codebook +codebooks +remask +higgs +verbalizers +workletized +partitioners diff --git a/packages/react-native-executorch/__tests__/api/__snapshots__/apiSurface.test.ts.snap b/packages/react-native-executorch/__tests__/api/__snapshots__/apiSurface.test.ts.snap index b5603d08d1..e01b92bb3d 100644 --- a/packages/react-native-executorch/__tests__/api/__snapshots__/apiSurface.test.ts.snap +++ b/packages/react-native-executorch/__tests__/api/__snapshots__/apiSurface.test.ts.snap @@ -98,18 +98,22 @@ exports[`public API surface matches the recorded contents of each namespace 1`] "validateSpec", ], "speech": [ + "EOS_PATTERN", "KOKORO_PAUSE_MS", "KOKORO_TICKS_PER_DURATION", "KOKORO_VOICE_REF_SIZE", + "PAUSE_PATTERN", "SUPERTONIC_SUPPORTED_LANGUAGES", "cleanText", "createPhonemizer", "encodeText", + "estimateTargetFrames", "extractFrames", "formatChunk", "parseVoice", "parseVoiceStyle", "partition", + "partitionReverbText", "preprocessText", "scaleDurations", "stripAudio", @@ -137,6 +141,9 @@ exports[`public API surface matches the recorded export list 1`] = ` "PRIVACY_FILTER_OPENAI_LABELS", "QWEN3_ASR_LANGUAGES", "QWEN3_ASR_SAMPLE_RATE_HZ", + "REVERB_FRAME_RATE", + "REVERB_LANGUAGES", + "REVERB_SAMPLE_RATE", "RnExecuTorchError", "SUPERTONIC_DEFAULT_VOICE_NAMES", "SUPERTONIC_SAMPLE_RATE", @@ -160,6 +167,7 @@ exports[`public API surface matches the recorded export list 1`] = ` "createPrivacyFilter", "createQwen3AsrSpeechToText", "createResourceScope", + "createReverbTextToSpeech", "createSdxsTextToImage", "createSemanticSegmenter", "createStyleTransfer", diff --git a/packages/react-native-executorch/__tests__/extensions/reverbPartitioner.test.ts b/packages/react-native-executorch/__tests__/extensions/reverbPartitioner.test.ts new file mode 100644 index 0000000000..c6aae9706e --- /dev/null +++ b/packages/react-native-executorch/__tests__/extensions/reverbPartitioner.test.ts @@ -0,0 +1,117 @@ +/** + * Reverb's partitioner: chunks within a duration limit, cut at the coarsest + * boundary that allows it. Durations come from the estimate at a pace of one + * letter per frame. + */ +import { partitionReverbText } from '../../src/extensions/speech/utils/reverbPartitioner'; +import { estimateTargetFrames } from '../../src/extensions/speech/utils/reverbUtils'; + +const seconds = (chunk: string) => estimateTargetFrames(chunk, 'abcdefghij', 10) / 25; + +/** `count` words of four letters, then `end`. About 1.7 s per ten words. */ +const run = (count: number, end = '.') => `${new Array(count).fill('word').join(' ')}${end}`; + +const LONG_TEXT = [ + run(10), + run(25, ','), + run(12, '!'), + run(60), + run(8, '?'), + run(15, ';'), + run(5), +].join(' '); + +describe('partitionReverbText', () => { + it('keeps a text that fits as one chunk', () => { + expect(partitionReverbText(run(10), seconds)).toEqual([run(10)]); + }); + + it('returns no chunks for an empty text', () => { + expect(partitionReverbText('', seconds)).toEqual([]); + expect(partitionReverbText(' ', seconds)).toEqual([]); + }); + + it('keeps every chunk within the duration limit', () => { + const chunks = partitionReverbText(LONG_TEXT, seconds, { maxDuration: 4 }); + + expect(chunks.length).toBeGreaterThan(3); + for (const chunk of chunks) expect(seconds(chunk)).toBeLessThanOrEqual(4); + }); + + it('gives back the text, its whitespace collapsed, when the chunks are joined', () => { + const text = ` ${run(30)}\n\n${run(40, ',')} ${run(50)}\t${run(3)} `; + + const chunks = partitionReverbText(text, seconds, { maxDuration: 3 }); + + expect(chunks.join(' ')).toBe(text.split(/\s+/).filter(Boolean).join(' ')); + }); + + it('packs whole sentences into a chunk while they fit', () => { + const [a, b, c] = [run(10), run(10, '!'), run(10, '?')]; + + expect(partitionReverbText(`${a} ${b} ${c}`, seconds, { maxDuration: 4 })).toEqual([ + `${a} ${b}`, + c, + ]); + }); + + it('splits a sentence too long on its own at other punctuation', () => { + const sentence = `${run(10, ',')} ${run(10, ':')} ${run(10)}`; + + const chunks = partitionReverbText(sentence, seconds, { maxDuration: 4 }); + + expect(chunks).toEqual([`${run(10, ',')} ${run(10, ':')}`, run(10)]); + }); + + it('splits a piece still too long between words', () => { + const chunks = partitionReverbText(run(60), seconds, { maxDuration: 4 }); + + expect(chunks.length).toBeGreaterThan(1); + for (const chunk of chunks) expect(seconds(chunk)).toBeLessThanOrEqual(4); + // Packed as far as it goes: one more word would not have fit. + for (const chunk of chunks.slice(0, -1)) expect(seconds(`${chunk} word`)).toBeGreaterThan(4); + }); + + it('starts a sentence after closing quotes, not before them', () => { + const text = `He said "${run(8)}" ${run(10)}`; + + expect(partitionReverbText(text, seconds, { maxDuration: 3 })[0]).toBe(`He said "${run(8)}"`); + }); + + it('keeps a single word that does not fit as a chunk of its own', () => { + expect(partitionReverbText('a extraordinarily b', seconds, { maxDuration: 0.5 })).toEqual([ + 'a', + 'extraordinarily', + 'b', + ]); + }); + + it('fits more text per chunk when chunks are estimated shorter', () => { + const faster = (chunk: string) => seconds(chunk) / 1.4; + + const normal = partitionReverbText(LONG_TEXT, seconds, { maxDuration: 4 }); + const fast = partitionReverbText(LONG_TEXT, faster, { maxDuration: 4 }); + + expect(fast.length).toBeLessThan(normal.length); + for (const chunk of fast) expect(faster(chunk)).toBeLessThanOrEqual(4); + }); + + it('meets the extra condition on every chunk', () => { + const fits = (chunk: string) => chunk.length <= 40; + + const chunks = partitionReverbText(LONG_TEXT, seconds, { fits }); + + expect(chunks.every(fits)).toBe(true); + expect(chunks.join(' ')).toBe(LONG_TEXT); + }); + + it.each([ + ['zero', 0], + ['negative', -1], + ['NaN', NaN], + ])('rejects a %s maxDuration', (_label, maxDuration) => { + expect(() => partitionReverbText('Hello.', seconds, { maxDuration })).toThrow( + /must be positive/ + ); + }); +}); diff --git a/packages/react-native-executorch/__tests__/extensions/reverbUtils.test.ts b/packages/react-native-executorch/__tests__/extensions/reverbUtils.test.ts new file mode 100644 index 0000000000..32441a1421 --- /dev/null +++ b/packages/react-native-executorch/__tests__/extensions/reverbUtils.test.ts @@ -0,0 +1,216 @@ +/** + * Reverb's utilities. The expected normalization, estimates, and trims come + * from the Python they port (Echo's and Voice Studio's), run on the same inputs. + */ +import { + buildPrompt, + estimateTargetFrames, + parseVoice, + pauseAfter, + prepareText, + roundHalfEven, + trimSilence, +} from '../../src/extensions/speech/utils/reverbUtils'; + +describe('prepareText', () => { + it.each([ + ['Wait… what?', 'Wait... what?'], + ['«Bonjour», dit-il.', '"Bonjour", dit-il.'], + ['It’s ‘fine’', "It's 'fine'"], + ['Szczurek–Żelazko', 'Szczurek-Żelazko'], + ['yes — no', 'yes - no'], + ['one -- two', 'one - two'], + ['a – — b', 'a - b'], + ['non breaking thin', 'non breaking thin'], + ['zero​width', 'zerowidth'], + ['too many spaces', 'too many spaces'], + ['Hello , world !', 'Hello, world!'], + ['Hmm ... ok .', 'Hmm ... ok.'], + [' edges ', 'edges'], + ['end—', 'end -'], + ['--start', '- start'], + ['x--y--z', 'x-y-z'], + ['tab\there', 'tab here'], + ['quote„test“', 'quote"test"'], + ['´acute` and ʼmodifier', "'acute' and 'modifier"], + ])('normalizes %j as Echo did', (input, expected) => { + expect(prepareText(input, 'en')).toBe(expected); + }); + + it('writes numbers and symbols out as words in the given language', () => { + expect(prepareText('I have 3 cats & 2 dogs.', 'en')).toBe('I have three cats and two dogs.'); + expect(prepareText('Mam 2 koty.', 'pl')).toBe('Mam dwa koty.'); + }); +}); + +describe('estimateTargetFrames', () => { + it.each([ + ['Hello world.', 'This is a reference transcript.', 250, 97], + ['Mam 2 koty i psa.', 'To jest nagranie referencyjne mojego głosu.', 300, 123], + ['Hi', 'A long transcript of the reference clip that sets the speaking rate.', 400, 32], + ['Café naïve - ok!', 'Reference.', 100, 132], + ['Zażółć gęślą jaźń. To jest test, który ma sprawdzić długość.', 'Krótki tekst.', 150, 657], + ['Some text', '', 100, 819], + ])('estimates %j against %j over %i frames as Voice Studio does', (...args) => { + const [text, referenceText, referenceFrames, expected] = args; + expect(estimateTargetFrames(text, referenceText, referenceFrames)).toBe(expected); + }); +}); + +describe('roundHalfEven', () => { + it.each([ + [0.5, 0], + [1.5, 2], + [2.5, 2], + [2.4, 2], + [2.6, 3], + [7, 7], + ])('rounds %d to %d like Python', (value, expected) => { + expect(roundHalfEven(value)).toBe(expected); + }); +}); + +describe('trimSilence', () => { + /** Concatenated constant segments: [seconds, amplitude] each. */ + const segments = (...parts: [number, number][]): Float32Array => { + const lengths = parts.map(([seconds]) => Math.round(seconds * 24000)); + const audio = new Float32Array(lengths.reduce((a, b) => a + b, 0)); + let offset = 0; + parts.forEach(([, amplitude], i) => { + audio.fill(amplitude, offset, offset + lengths[i]!); + offset += lengths[i]!; + }); + return audio; + }; + + it.each<[string, [number, number][], number, number]>([ + [ + 'speech', + [ + [0.5, 0], + [1.0, 0.1], + [0.5, 0], + ], + 9600, + 38400, + ], + [ + 'a leading blip', + [ + [0.2, 0], + [0.1, 0.1], + [0.3, 0], + [1.0, 0.1], + [0.4, 0], + ], + 12000, + 40800, + ], + [ + 'a trailing blip', + [ + [0.3, 0], + [1.0, 0.1], + [0.25, 0], + [0.12, 0.1], + [0.3, 0], + ], + 4800, + 33600, + ], + [ + 'a leading sound', + [ + [0.2, 0], + [0.2, 0.1], + [0.3, 0], + [1.0, 0.1], + [0.4, 0], + ], + 2400, + 43200, + ], + [ + 'a blip close to the speech', + [ + [0.2, 0], + [0.1, 0.1], + [0.15, 0], + [1.0, 0.1], + [0.4, 0], + ], + 2400, + 37200, + ], + [ + 'quiet speech', + [ + [0.3, 0], + [0.5, 0.004], + [0.3, 0.003], + ], + 4800, + 21600, + ], + ['silence', [[0.5, 0]], 0, 12000], + ['less than a window', [[0.005, 0.5]], 0, 120], + [ + 'no room for the margin', + [ + [0.05, 0], + [1.0, 0.1], + [0.02, 0], + ], + 0, + 25680, + ], + ])('trims %s as Voice Studio does', (_label, parts, start, end) => { + const trimmed = trimSilence(segments(...parts), 24000, -50, 0.1); + + expect(trimmed.byteOffset / 4).toBe(start); + expect(trimmed.byteOffset / 4 + trimmed.length).toBe(end); + }); +}); + +describe('pauseAfter', () => { + it.each([ + ['A sentence.', 0.375], + ['A question?', 0.5], + ['A clause,', 0.13], + ['He said "yes."', 0.375], + ['Wait...', 1], + ['No mark', 0], + ])('pauses after %j for %d s', (chunk, seconds) => { + expect(pauseAfter(chunk)).toBeCloseTo(seconds); + }); +}); + +describe('buildPrompt', () => { + it('puts the language, the transcript and the text in the template', () => { + expect(buildPrompt('pl', 'Dzień dobry.', 'Cześć!')).toBe( + '<|denoise|><|lang_start|>pl<|lang_end|><|instruct_start|>None<|instruct_end|>' + + '<|text_start|>Dzień dobry. Cześć!<|text_end|>' + ); + }); +}); + +describe('parseVoice', () => { + it('parses codes, transcript and language', () => { + const codes = Array.from({ length: 16 }, (_, i) => i * 10); + + expect(parseVoice({ codes, transcript: 'Hello.', language: 'en' })).toEqual({ + codes: BigInt64Array.from(codes, BigInt), + transcript: 'Hello.', + language: 'en', + }); + }); + + it.each([ + ['no codes', { transcript: 'Hi.', language: 'en' }], + ['a fractional code', { codes: [0.5], transcript: 'Hi.', language: 'en' }], + ['no transcript', { codes: [0], language: 'en' }], + ['no language', { codes: [0], transcript: 'Hi.' }], + ])('rejects a voice with %s', (_label, json) => { + expect(() => parseVoice(json)).toThrow(/Invalid voice JSON/); + }); +}); diff --git a/packages/react-native-executorch/__tests__/tasks/reverbTextToSpeech.test.ts b/packages/react-native-executorch/__tests__/tasks/reverbTextToSpeech.test.ts new file mode 100644 index 0000000000..0878bed73c --- /dev/null +++ b/packages/react-native-executorch/__tests__/tasks/reverbTextToSpeech.test.ts @@ -0,0 +1,514 @@ +/** + * The Reverb pipeline on the fake runtime: the programs' contract, the calls + * and shapes that make a chunk, the sampler, chunking, validation, the lazy + * encoder, and disposal. The speech itself needs real weights. + */ +import { RangeDim, constraint, f16, f32, i64, method } from '../../src/core/schema'; +import { + REVERB_SAMPLE_RATE, + createReverbTextToSpeech, + type ReverbVoice, +} from '../../src/extensions/speech/tasks/reverbTextToSpeech'; +import { + estimateTargetFrames, + prepareText, + roundHalfEven, +} from '../../src/extensions/speech/utils/reverbUtils'; +import { fakeJsi } from '../support/fakeJsi'; +import { fakeFs } from '../support/blobUtilMock'; +import { exported } from '../support/fixtures'; +import { tracked } from '../support/lifetime'; +import type { FakeTensor } from '../support/fakeTensor'; + +const BACKBONE_PATH = '/models/echo_mlx_fp16.pte'; +const DECODER_PATH = '/models/codec_decoder_coreml_fp16.pte'; +const ENCODER_PATH = '/models/codec_encoder_coreml_int4.pte'; +const TOKENIZER_PATH = '/models/tokenizer.json'; +const VOICE_PATH = '/models/voices/anna.json'; + +const FRAME_SAMPLES = 960; +const MAX_PROMPT = 64; // tokens; the fake tokenizer makes one per whitespace-separated piece +const PREDICTED = 5; // every step predicts this code +const POLISHED = 9; // polish fills codebooks 2-7 with this code +const ENCODED = 17; // the encoder's codes + +const frames = (min: number, max: number) => RangeDim(min, max); +const tDim = (paramSide: 'input' | 'output', tensorIdx: number, dimIdx = 1) => ({ + paramSide, + tensorIdx, + dimIdx, +}); + +const backboneSchema = (cacheType = f16) => { + const cache = () => cacheType(2, 24, 6, frames(4, 1024), 128); + return exported({ + ...method('prefill', [i64(frames(2, MAX_PROMPT)), i64(8, frames(2, 512))], [cache()]), + ...method( + 'step', + [i64(8, frames(4, 1024)), cache()], + [f32(2, frames(4, 1024)), i64(2, frames(4, 1024)), f32(2, frames(4, 1024))], + [ + constraint.equality( + tDim('input', 0), + tDim('output', 0), + tDim('output', 1), + tDim('output', 2) + ), + ] + ), + ...method( + 'polish', + [i64(8, frames(4, 1024)), i64(1), cache()], + [i64(6, frames(4, 1024))], + [constraint.equality(tDim('input', 0), tDim('output', 0))] + ), + }); +}; + +const decoderSchema = (withConstraint = true) => + exported( + method( + 'forward', + [i64(8, frames(4, 1024))], + [f32(RangeDim(4 * FRAME_SAMPLES, 1024 * FRAME_SAMPLES, FRAME_SAMPLES))], + withConstraint + ? [constraint.linear(tDim('output', 0, 0), tDim('input', 0), FRAME_SAMPLES)] + : [] + ) + ); + +const encoderSchema = (width = FRAME_SAMPLES) => + exported( + method( + 'forward', + [f32(frames(25, 500), width)], + [i64(8, frames(25, 500))], + [constraint.equality(tDim('input', 0, 0), tDim('output', 0))] + ) + ); + +const fill = (t: FakeTensor, value: number) => { + for (let i = 0; i < t.numel; i++) t.setElement(i, value); +}; + +/** Every call's method, input shapes, and inputs. */ +let calls: { methodName: string; shapes: (readonly number[])[]; inputs: FakeTensor[] }[] = []; +/** Masked slots of codebooks 0-1 entering each step and the polish. */ +let masked: number[] = []; + +const registerModels = (backbone = backboneSchema(), decoder = decoderSchema()): void => { + calls = []; + masked = []; + const record = (methodName: string, inputs: readonly unknown[]) => { + const tensors = inputs as FakeTensor[]; + calls.push({ methodName, shapes: tensors.map((t) => t.shape), inputs: tensors }); + }; + fakeJsi.registerModel(BACKBONE_PATH, { + schema: backbone, + execute: (methodName, inputs, outputs) => { + record(methodName, inputs); + const tokens = inputs[0] as FakeTensor; + if (methodName !== 'prefill') { + const slots = Array.from({ length: 2 * tokens.shape[1]! }, (_, i) => tokens.getElement(i)); + masked.push(slots.filter((code) => code === 1024).length); + } + if (methodName === 'step') { + fill(outputs[0]!, -1); + fill(outputs[1]!, PREDICTED); + outputs[2]!.data.forEach((_, i) => outputs[2]!.setElement(i, -i)); + } + if (methodName === 'polish') fill(outputs[0]!, POLISHED); + }, + }); + fakeJsi.registerModel(DECODER_PATH, { + schema: decoder, + execute: (methodName, inputs, outputs) => { + record(`decoder.${methodName}`, inputs); + fill(outputs[0]!, 0.5); // a steady tone: nothing for the trim to cut + }, + }); + fakeJsi.registerModel(ENCODER_PATH, { + schema: encoderSchema(), + execute: (methodName, inputs, outputs) => { + record(`encoder.${methodName}`, inputs); + fill(outputs[0]!, ENCODED); + }, + }); + fakeJsi.registerTokenizer(TOKENIZER_PATH, { tokens: [], open: true }); +}; + +const PRESET = { + codes: new Array(8 * 30).fill(3), + transcript: 'This is my voice.', + language: 'en', +}; +const preset: ReverbVoice = { + codes: BigInt64Array.from(PRESET.codes, BigInt), + transcript: PRESET.transcript, + language: 'en', +}; + +const config = { + name: 'reverb', + modelPaths: { backbone: BACKBONE_PATH, codecDecoder: DECODER_PATH, codecEncoder: ENCODER_PATH }, + tokenizerPath: TOKENIZER_PATH, + voices: { anna: VOICE_PATH }, +} as const; + +const collect = async (stream: AsyncGenerator): Promise => { + const chunks: T[] = []; + for await (const chunk of stream) chunks.push(chunk); + return chunks; +}; + +const methodsCalled = () => calls.map(({ methodName }) => methodName); + +beforeEach(() => { + fakeFs.write(VOICE_PATH, JSON.stringify(PRESET)); + registerModels(); +}); + +describe('createReverbTextToSpeech — the model contract', () => { + it('accepts the exported programs and exposes the synthesis API', async () => { + const tts = tracked(await createReverbTextToSpeech(config)); + + expect(tts.encodeVoice).toBeInstanceOf(Function); + expect(tts.synthesize).toBeInstanceOf(Function); + expect(tts.synthesizeStop).toBeInstanceOf(Function); + }); + + it('rejects a backbone whose cache is not float16', async () => { + registerModels(backboneSchema(f32)); + + await expect(createReverbTextToSpeech(config)).rejects.toThrow(/DType mismatch/); + }); + + it('rejects a decoder that does not tie its audio length to the frames', async () => { + registerModels(backboneSchema(), decoderSchema(false)); + + await expect(createReverbTextToSpeech(config)).rejects.toThrow(/constraint/i); + }); + + it('rejects a malformed preset voice', async () => { + fakeFs.write(VOICE_PATH, JSON.stringify({ ...PRESET, codes: [1, 2, 3] })); + await expect(createReverbTextToSpeech(config)).rejects.toThrow(/Voice 'anna': A voice holds/); + + fakeFs.write(VOICE_PATH, JSON.stringify({ ...PRESET, language: 'xx' })); + await expect(createReverbTextToSpeech(config)).rejects.toThrow(/Voice 'anna': A voice holds/); + }); + + it('releases the programs, the tokenizer and every tensor when construction fails', async () => { + registerModels(backboneSchema(), decoderSchema(false)); + + await expect(createReverbTextToSpeech(config)).rejects.toThrow(); + + expect(fakeJsi.liveModels()).toEqual([]); + expect(fakeJsi.liveTokenizers()).toEqual([]); + expect(fakeJsi.liveTensors()).toBe(0); + }); + + it('loads the encoder only when a voice is first encoded', async () => { + const tts = tracked(await createReverbTextToSpeech(config)); + expect(fakeJsi.liveModels()).toEqual([BACKBONE_PATH, DECODER_PATH].sort()); + + await tts.encodeVoice(new Float32Array(2 * REVERB_SAMPLE_RATE), 'Hello there.', 'en'); + + expect(fakeJsi.liveModels()).toEqual([BACKBONE_PATH, DECODER_PATH, ENCODER_PATH].sort()); + }); + + it('releases every program, the tokenizer and the tensors on dispose', async () => { + const tts = await createReverbTextToSpeech(config); + await tts.encodeVoice(new Float32Array(2 * REVERB_SAMPLE_RATE), 'Hello there.', 'en'); + + tts.dispose(); + + expect(fakeJsi.liveModels()).toEqual([]); + expect(fakeJsi.liveTokenizers()).toEqual([]); + expect(fakeJsi.liveTensors()).toBe(0); + }); +}); + +describe('createReverbTextToSpeech — synthesis', () => { + const TEXT = 'Hello world, this is a test.'; + const estimate = (text: string) => + estimateTargetFrames(prepareText(text, 'en'), PRESET.transcript, 30); + const targetFrames = (text: string) => Math.max(10, estimate(text)); + + it('makes a chunk from a prefill, 8 steps, the polish and the decoder, in that order', async () => { + const tts = tracked(await createReverbTextToSpeech(config)); + + await collect(tts.synthesize(TEXT, { voice: 'anna', language: 'en' })); + + expect(methodsCalled()).toEqual([ + 'prefill', + ...new Array(8).fill('step'), + 'polish', + 'decoder.forward', + ]); + }); + + it('sizes every tensor from the prompt, the voice and the estimated length', async () => { + const tts = tracked(await createReverbTextToSpeech(config)); + + await collect(tts.synthesize(TEXT, { voice: 'anna', language: 'en' })); + + const [prefill, step] = calls; + const polish = calls.find(({ methodName }) => methodName === 'polish')!; + const decode = calls.find(({ methodName }) => methodName === 'decoder.forward')!; + const promptTokens = prefill!.shapes[0]![0]!; + const target = targetFrames(TEXT); + expect(prefill!.shapes[1]).toEqual([8, 30]); + expect(step!.shapes).toEqual([ + [8, target], + [2, 24, 6, promptTokens + 30, 128], + ]); + expect(polish.inputs[1]!.getElement(0)).toBe(30); + expect(decode.shapes).toEqual([[8, target]]); + }); + + it('decodes the committed codes of codebooks 0-1 and the polished codebooks 2-7', async () => { + const tts = tracked(await createReverbTextToSpeech(config)); + + await collect(tts.synthesize(TEXT, { voice: 'anna', language: 'en' })); + + const codes = calls.find(({ methodName }) => methodName === 'decoder.forward')!.inputs[0]!; + const target = targetFrames(TEXT); + const rows = Array.from( + { length: 8 }, + (_, row) => + new Set(Array.from({ length: target }, (__, i) => codes.getElement(row * target + i))) + ); + expect(rows.map((values) => [...values])).toEqual([ + [PREDICTED], + [PREDICTED], + ...new Array(6).fill([POLISHED]), + ]); + }); + + it('cuts 0.2 s off the end and appends the pause its punctuation calls for', async () => { + const tts = tracked(await createReverbTextToSpeech(config)); + + const [chunk] = await collect(tts.synthesize(TEXT, { voice: 'anna', language: 'en' })); + + const pause = 0.375 * REVERB_SAMPLE_RATE; // after '.' + expect(chunk!.audio.length).toBe(targetFrames(TEXT) * FRAME_SAMPLES - 4800 + pause); + expect(chunk).toMatchObject({ sampleRate: REVERB_SAMPLE_RATE, chunkIndex: 0, totalChunks: 1 }); + expect(chunk!.duration).toBeCloseTo(chunk!.audio.length / REVERB_SAMPLE_RATE); + }); + + it('returns the raw decoder output, plus the pause, without post-processing', async () => { + const tts = tracked(await createReverbTextToSpeech(config)); + + const [chunk] = await collect( + tts.synthesize('No mark here', { voice: 'anna', language: 'en', postprocess: false }) + ); + + expect(chunk!.audio.length).toBe(targetFrames('No mark here') * FRAME_SAMPLES); + }); + + it('runs as many main passes as asked', async () => { + const tts = tracked(await createReverbTextToSpeech(config)); + + await collect(tts.synthesize(TEXT, { voice: 'anna', language: 'en', sampling: { steps: 3 } })); + + expect(methodsCalled().filter((name) => name === 'step')).toHaveLength(3); + }); + + it('stretches the target as the speed drops', async () => { + const tts = tracked(await createReverbTextToSpeech(config)); + + await collect(tts.synthesize(TEXT, { voice: 'anna', language: 'en', speed: 0.8 })); + + expect(calls[1]!.shapes[0]).toEqual([8, roundHalfEven(estimate(TEXT) / 0.8)]); + }); + + it('prompts with the language, the voice transcript and the prepared text', async () => { + const vocabulary: string[] = []; + fakeJsi.registerTokenizer(TOKENIZER_PATH, { tokens: vocabulary, open: true }); + const tts = tracked(await createReverbTextToSpeech(config)); + + await collect(tts.synthesize('I have 2 cats.', { voice: 'anna', language: 'en' })); + + const ids = calls[0]!.inputs[0]!; + const words = Array.from({ length: ids.numel }, (_, i) => vocabulary[ids.getElement(i)]); + expect(words.join(' ')).toBe( + '<|denoise|><|lang_start|>en<|lang_end|><|instruct_start|>None<|instruct_end|>' + + '<|text_start|>This is my voice. I have two cats.<|text_end|>' + ); + }); + + it("commits and re-masks codebooks 0-1 on Voice Studio's schedule", async () => { + const tts = tracked(await createReverbTextToSpeech(config)); + + await collect(tts.synthesize(TEXT, { voice: 'anna', language: 'en' })); + + // 8 passes: each commits its share of the slots, then all but the last + // re-mask a quarter as many held codes. The polish gets none masked. + const slots = 2 * targetFrames(TEXT); + const share = (i: number) => (0.1 * (i / 8)) / (1 - (1 - 0.1) * (i / 8)); + const expected = [slots]; + for (let pass = 0; pass < 8; pass++) { + const open = expected[pass]!; + const last = pass === 7; + const commits = last + ? open + : Math.min(open, Math.ceil(slots * (share(pass + 1) - share(pass)))); + const withdrawn = last ? 0 : Math.min(roundHalfEven(0.25 * commits), slots - open); + expected.push(open - commits + withdrawn); + } + expect(masked).toEqual(expected); + expect(masked.at(-1)).toBe(0); + }); + + it('speaks a text too long for one chunk in several, each from its own prefill', async () => { + const tts = tracked(await createReverbTextToSpeech(config)); + const text = 'First sentence here. Second sentence follows it. Third one ends the text.'; + + const chunks = await collect( + tts.synthesize(text, { voice: 'anna', language: 'en', maxChunkDuration: 1.5 }) + ); + + expect(chunks.length).toBeGreaterThan(1); + expect(chunks.map(({ chunkIndex }) => chunkIndex)).toEqual(chunks.map((_, i) => i)); + expect(new Set(chunks.map(({ totalChunks }) => totalChunks))).toEqual(new Set([chunks.length])); + expect(methodsCalled().filter((name) => name === 'prefill')).toHaveLength(chunks.length); + }); + + it('keeps every prompt within the backbone token limit', async () => { + const tts = tracked(await createReverbTextToSpeech(config)); + const text = new Array(150).fill('word').join(' ') + '.'; + + await collect(tts.synthesize(text, { voice: 'anna', language: 'en' })); + + const prompts = calls.filter(({ methodName }) => methodName === 'prefill'); + expect(prompts.length).toBeGreaterThan(1); + for (const { shapes } of prompts) expect(shapes[0]![0]).toBeLessThanOrEqual(MAX_PROMPT); + }); + + it('synthesizes in a voice it encoded', async () => { + const tts = tracked(await createReverbTextToSpeech(config)); + const audio = new Float32Array(Math.round(2.5 * REVERB_SAMPLE_RATE)); + + const voice = await tts.encodeVoice(audio, 'I have 2 cats.', 'en'); + await collect(tts.synthesize(TEXT, { voice, language: 'en' })); + + expect(voice.transcript).toBe('I have two cats.'); + expect(voice.codes).toEqual(new BigInt64Array(8 * 62).fill(BigInt(ENCODED))); + expect(calls.find(({ methodName }) => methodName === 'encoder.forward')!.shapes).toEqual([ + [62, FRAME_SAMPLES], + ]); + expect(calls.find(({ methodName }) => methodName === 'prefill')!.shapes[1]).toEqual([8, 62]); + }); + + it('stops streaming when synthesizeStop is called mid-stream', async () => { + const tts = tracked(await createReverbTextToSpeech(config)); + const text = 'First sentence here. Second sentence follows it. Third one ends the text.'; + + const stream = tts.synthesize(text, { voice: 'anna', language: 'en', maxChunkDuration: 1.5 }); + const first = await stream.next(); + tts.synthesizeStop(); + const after = await stream.next(); + + expect(first.value!.totalChunks).toBeGreaterThan(1); + expect(after.done).toBe(true); + }); + + it('leaves no tensor behind once a stream is drained or abandoned', async () => { + const tts = tracked(await createReverbTextToSpeech(config)); + const text = 'First sentence here. Second sentence follows it.'; + + await collect(tts.synthesize(text, { voice: 'anna', language: 'en', maxChunkDuration: 1.5 })); + const stream = tts.synthesize(text, { voice: 'anna', language: 'en', maxChunkDuration: 1.5 }); + await stream.next(); + await stream.return(undefined as never); + + expect(fakeJsi.liveTensors()).toBe(0); + }); +}); + +describe('createReverbTextToSpeech — argument validation', () => { + const synthesizeWith = async (options: object, text = 'Hello.') => { + const tts = tracked(await createReverbTextToSpeech(config)); + return collect(tts.synthesize(text, { voice: 'anna', language: 'en', ...options } as never)); + }; + + it.each([ + ['an empty text', {}, '', /cannot be empty/], + ['an unsupported language', { language: 'xx' }, 'Hello.', /Unsupported language/], + ['an unknown voice', { voice: 'nobody' }, 'Hello.', /Unknown voice/], + ['a speed that is too slow', { speed: 0.4 }, 'Hello.', /speed must be between/], + ['a speed that is too fast', { speed: 2.1 }, 'Hello.', /speed must be between/], + ['a chunk longer than the target window', { maxChunkDuration: 60 }, 'Hello.', /at most 40.96/], + ['a fractional step count', { sampling: { steps: 2.5 } }, 'Hello.', /positive integer/], + ['a remask ratio above 1', { sampling: { remaskRatio: 1.5 } }, 'Hello.', /\[0, 1\]/], + ['a fractional seed', { sampling: { seed: 0.5 } }, 'Hello.', /seed must be an integer/], + ])('rejects %s', async (_label, options, text, error) => { + await expect(synthesizeWith(options, text)).rejects.toThrow(error); + }); + + it.each([ + ['codes not in rows of 8', { ...preset, codes: new BigInt64Array(9) }], + ['a reference shorter than 2 frames', { ...preset, codes: new BigInt64Array(8) }], + ['a code out of range', { ...preset, codes: new BigInt64Array(16).fill(1024n) }], + ['no transcript', { ...preset, transcript: ' ' }], + ])('rejects a voice with %s', async (_label, voice) => { + await expect(synthesizeWith({ voice })).rejects.toThrow(/A voice holds/); + }); + + it('rejects a voice whose transcript fills the prompt', async () => { + const voice = { ...preset, transcript: new Array(MAX_PROMPT).fill('word').join(' ') }; + + await expect(synthesizeWith({ voice })).rejects.toThrow(/leaves no room/); + }); + + it('refuses a second synthesis while one is still running, then runs it afterwards', async () => { + const tts = tracked(await createReverbTextToSpeech(config)); + const first = tts.synthesize('One. Two. Three.', { voice: 'anna', language: 'en' }); + await first.next(); + + await expect( + collect(tts.synthesize('Again.', { voice: 'anna', language: 'en' })) + ).rejects.toThrow(/already in progress/); + await collect(first); + await expect( + collect(tts.synthesize('Again.', { voice: 'anna', language: 'en' })) + ).resolves.toHaveLength(1); + }); + + it.each([ + ['shorter than 1 s', 0.9], + ['longer than 20 s', 20.5], + ])('rejects a recording %s', async (_label, seconds) => { + const tts = tracked(await createReverbTextToSpeech(config)); + const audio = new Float32Array(Math.round(seconds * REVERB_SAMPLE_RATE)); + + await expect(tts.encodeVoice(audio, 'Hello.', 'en')).rejects.toThrow(/must last 1 to 20 s/); + }); + + it('rejects encoding without a transcript or in an unsupported language', async () => { + const tts = tracked(await createReverbTextToSpeech(config)); + const audio = new Float32Array(2 * REVERB_SAMPLE_RATE); + + await expect(tts.encodeVoice(audio, ' ', 'en')).rejects.toThrow(/transcript cannot be empty/); + await expect(tts.encodeVoice(audio, 'Hi.', 'xx' as never)).rejects.toThrow(/Unsupported/); + }); + + it('refuses to encode when the config names no encoder', async () => { + const modelPaths = { backbone: BACKBONE_PATH, codecDecoder: DECODER_PATH }; + const tts = tracked(await createReverbTextToSpeech({ ...config, modelPaths })); + + await expect( + tts.encodeVoice(new Float32Array(2 * REVERB_SAMPLE_RATE), 'Hello.', 'en') + ).rejects.toThrow(/names no codecEncoder/); + }); + + it('rejects an encoder with the wrong frame width, and releases it', async () => { + fakeJsi.registerModel(ENCODER_PATH, { schema: encoderSchema(480) }); + const tts = tracked(await createReverbTextToSpeech(config)); + + await expect( + tts.encodeVoice(new Float32Array(2 * REVERB_SAMPLE_RATE), 'Hello.', 'en') + ).rejects.toThrow(/Constant dimension mismatch/); + expect(fakeJsi.liveModels()).not.toContain(ENCODER_PATH); + }); +}); diff --git a/packages/react-native-executorch/jest.config.js b/packages/react-native-executorch/jest.config.js index 983f84f3bd..8b69ce99e1 100644 --- a/packages/react-native-executorch/jest.config.js +++ b/packages/react-native-executorch/jest.config.js @@ -24,6 +24,12 @@ module.exports = { '^react-native-worklets$': '/__tests__/support/workletsMock.ts', }, + // Verbalis ships ES modules only, so it is compiled like the React Native + // packages the preset already lets through. + transformIgnorePatterns: [ + 'node_modules/(?!((jest-)?react-native|@react-native(-community)?|verbalis)/)', + ], + // Snapshot of the public export surface — see `api/apiSurface.test.ts`. snapshotFormat: { escapeString: false, printBasicPrototype: false }, diff --git a/packages/react-native-executorch/package.json b/packages/react-native-executorch/package.json index 76f7b8bee9..ece3cdd653 100644 --- a/packages/react-native-executorch/package.json +++ b/packages/react-native-executorch/package.json @@ -119,7 +119,8 @@ "registry": "https://registry.npmjs.org/" }, "dependencies": { - "@huggingface/jinja": "^0.5.9" + "@huggingface/jinja": "^0.5.9", + "verbalis": "^0.1.1" }, "peerDependencies": { "@kesha-antonov/react-native-background-downloader": ">=4.4.0", diff --git a/packages/react-native-executorch/src/extensions/speech/index.ts b/packages/react-native-executorch/src/extensions/speech/index.ts index 2ada7825ea..2a31436574 100644 --- a/packages/react-native-executorch/src/extensions/speech/index.ts +++ b/packages/react-native-executorch/src/extensions/speech/index.ts @@ -8,3 +8,5 @@ export * from './utils/supertonicUtils'; export * from './utils/kokoroUtils'; export * from './utils/phonemizer'; export * from './utils/textPartitioner'; +export { estimateTargetFrames } from './utils/reverbUtils'; +export * from './utils/reverbPartitioner'; diff --git a/packages/react-native-executorch/src/extensions/speech/tasks/reverbTextToSpeech.ts b/packages/react-native-executorch/src/extensions/speech/tasks/reverbTextToSpeech.ts new file mode 100644 index 0000000000..0483249cbd --- /dev/null +++ b/packages/react-native-executorch/src/extensions/speech/tasks/reverbTextToSpeech.ts @@ -0,0 +1,689 @@ +/** + * Reverb Text-to-Speech (TTS) synthesis task pipeline. + */ + +import type { WorkletRuntime } from 'react-native-worklets'; + +import RNBlobUtil from 'react-native-blob-util'; + +import { tensor, type Tensor } from '../../../core/tensor'; +import { loadModel } from '../../../core/model'; +import { + validateSpec, + method, + i64, + f16, + f32, + DynamicDim as Dyn, + constraint, + type DimRef, + type Range, +} from '../../../core/schema'; +import { wrapAsync } from '../../../core/runtime'; +import { createResourceScope } from '../../../core/lifetime'; +import { RnExecuTorchError } from '../../../core/error'; +import { mulberry32 } from '../../math'; +import { loadTokenizer } from '../../nlp/tokenizer'; +import { partitionReverbText } from '../utils/reverbPartitioner'; +import { + buildPrompt, + estimateTargetFrames, + parseVoice, + pauseAfter, + prepareText, + roundHalfEven, + trimSilence, +} from '../utils/reverbUtils'; + +/** + * Reverb audio sampling rate in Hz (24000). + * @category Speech / Constants + */ +export const REVERB_SAMPLE_RATE = 24000; + +/** + * Codec frames per second (25). + * @category Speech / Constants + */ +export const REVERB_FRAME_RATE = 25; + +/** + * Languages Reverb speaks. + * @category Speech / Constants + */ +export const REVERB_LANGUAGES = ['de', 'en', 'es', 'fr', 'it', 'pl', 'pt'] as const; + +const FRAME_SAMPLES = REVERB_SAMPLE_RATE / REVERB_FRAME_RATE; // 960 +const CODEBOOKS = 8; +const CODES = 1024; // per codebook; the next id marks a masked slot +const MASK_CODE = BigInt(CODES); +// K/V cache: [keys and values, layers, KV heads, prefix, head dim] +const CACHE_LAYERS = 24; +const CACHE_HEADS = 6; +const CACHE_HEAD_DIM = 128; + +// Voice Studio's sampler +const T_SHIFT = 0.1; // few commits early, most in the last passes +const CODEBOOK_PENALTY = 5; // per codebook index: codebook 0 commits before 1 +const POSITION_TEMPERATURE = 5; // scales confidence and penalty against the Gumbel noise +const DEFAULT_STEPS = 8; +const DEFAULT_REMASK_RATIO = 0.25; + +const MIN_FRAMES = 10; +const DEFAULT_SPEED = 1.0; +const MIN_SPEED = 0.5; +const MAX_SPEED = 2.0; +const DEFAULT_MAX_CHUNK_DURATION = 20; +const DEFAULT_POSTPROCESS = { endCut: 0.2, silenceThreshold: -50, keep: 0.1 }; + +/** + * A language Reverb speaks. + * @category Speech / Types + */ +export type ReverbLanguage = (typeof REVERB_LANGUAGES)[number]; + +/** + * A voice to speak in: a reference clip's codec codes and transcript. + * @category Speech / Types + */ +export type ReverbVoice = { + /** Codec codes, `[8, frames]` row-major. */ + readonly codes: BigInt64Array; + /** What the clip says, verbalized and normalized. */ + readonly transcript: string; + /** The clip's language. */ + readonly language: ReverbLanguage; +}; + +/** + * Model configuration of the Reverb Text-to-Speech pipeline. + * @category Speech / Types + * @typeParam K Preset voice keys. + */ +export type ReverbTtsModel = { + /** Discriminates this config from the other Text-to-Speech pipelines. */ + readonly name: 'reverb'; + /** Local or remote paths to the `.pte` programs. */ + readonly modelPaths: { + /** The backbone: `prefill`, `step`, `polish`. */ + readonly backbone: string; + /** The codec decoder: codes to audio. */ + readonly codecDecoder: string; + /** The codec encoder: audio to codes. Only `encodeVoice` loads it. */ + readonly codecEncoder?: string; + }; + /** Local or remote path to `tokenizer.json`. */ + readonly tokenizerPath: string; + /** Preset voices: name to `{ codes, transcript, language }` JSON file. */ + readonly voices?: Record; +}; + +/** + * Sampler settings; the defaults are the measured ones. + * @category Speech / Types + */ +export type ReverbSamplingOptions = { + /** Main passes. Defaults to 8. + * NOTE: total number of steps (including polish pass) is always N + 2 */ + readonly steps?: number; + /** Held codes re-masked per new commit, in `[0, 1]`. Defaults to 0.25. */ + readonly remaskRatio?: number; + /** Integer seed for reproducible takes. Random by default. */ + readonly seed?: number; +}; + +/** + * Per-chunk post-processing; the defaults are Voice Studio's. + * @category Speech / Types + */ +export type ReverbPostprocessOptions = { + /** Seconds cut from the end. Defaults to 0.2. */ + readonly endCut?: number; + /** Level in dBFS below which an edge window is silent. Defaults to -50. */ + readonly silenceThreshold?: number; + /** Seconds of silence kept at each edge. Defaults to 0.1. */ + readonly keep?: number; +}; + +/** + * Per-call options of Reverb synthesis. + * @category Speech / Types + * @typeParam K Preset voice keys. + */ +export type ReverbTtsOptions = { + /** A preset voice key, or a {@link ReverbVoice} from `encodeVoice`. */ + readonly voice: K | ReverbVoice; + /** Language of the text. */ + readonly language: ReverbLanguage; + /** Speech speed factor, 0.5 to 2.0. Defaults to 1.0. */ + readonly speed?: number; + /** Max seconds of speech per chunk. Defaults to 20. */ + readonly maxChunkDuration?: number; + /** Post-processing, or `false` for the raw decoder output. */ + readonly postprocess?: ReverbPostprocessOptions | false; + /** Sampler settings. */ + readonly sampling?: ReverbSamplingOptions; +}; + +/** + * Audio chunk yielded by {@link ReverbTextToSpeech.synthesize}. + * @category Speech / Types + */ +export type ReverbTtsChunk = { + /** Float32 PCM samples in `[-1, 1]`. */ + readonly audio: Float32Array; + /** Sampling rate in Hz. */ + readonly sampleRate: number; + /** Duration in seconds. */ + readonly duration: number; + /** Zero-based index of this chunk. */ + readonly chunkIndex: number; + /** Number of chunks the text was split into. */ + readonly totalChunks: number; +}; + +/** + * Reverb text-to-speech task runner. + * @category Speech / Types + * @typeParam K Preset voice keys. + */ +export type ReverbTextToSpeech = { + /** Releases all native resources. */ + readonly dispose: () => void; + + /** + * Encodes a recording into a voice, loading the codec encoder on first use. + * @param audio 24 kHz mono PCM, 1 to 20 s. + * @param transcript What the recording says. + * @param language The recording's language. + * @returns The voice. + * @throws {RnExecuTorchError} With code `INVALID_ARGUMENT` on a bad argument, + * `INVALID_STATE` without a codec encoder, or `RESOURCE_DISPOSED` if + * disposed. + */ + readonly encodeVoice: ( + audio: Float32Array, + transcript: string, + language: ReverbLanguage + ) => Promise; + + /** + * Streams synthesized audio, one chunk of text at a time. + * @param text The text to speak. + * @param options See {@link ReverbTtsOptions}. + * @returns An AsyncGenerator of {@link ReverbTtsChunk}s. + * @throws {RnExecuTorchError} With code `INVALID_ARGUMENT` on a bad argument, + * `INVALID_STATE` if a synthesis is running, `RESOURCE_BUSY` if the model is + * in use, or `RESOURCE_DISPOSED` if disposed. + */ + readonly synthesize: ( + text: string, + options: ReverbTtsOptions + ) => AsyncGenerator; + + /** Cancels an in-flight synthesis. */ + readonly synthesizeStop: () => void; +}; + +const frameDim = (paramSide: 'input' | 'output', tensorIdx: number, dimIdx = 1): DimRef => ({ + paramSide, + tensorIdx, + dimIdx, +}); +const cache = () => f16(2, CACHE_LAYERS, CACHE_HEADS, Dyn('P'), CACHE_HEAD_DIM); + +// prettier-ignore +const BACKBONE_SPEC = { + ...method( + 'prefill', + [i64(Dyn('L')), i64(CODEBOOKS, Dyn('R'))], // prompt tokens, reference codes + [cache()] + ), + ...method( + 'step', + [i64(CODEBOOKS, Dyn('T')), cache()], // tokens + [f32(2, Dyn('T')), i64(2, Dyn('T')), f32(2, Dyn('T'))], // confidence, prediction, held + [constraint.equality(frameDim('input', 0), frameDim('output', 0), frameDim('output', 1), frameDim('output', 2))] + ), + ...method( + 'polish', + [i64(CODEBOOKS, Dyn('T')), i64(1), cache()], // tokens, reference frames + [i64(CODEBOOKS - 2, Dyn('T'))], // codebooks 2-7 + [constraint.equality(frameDim('input', 0), frameDim('output', 0))] + ), +}; + +const DECODER_SPEC = method( + 'forward', + [i64(CODEBOOKS, Dyn('T'))], // codes + [f32(Dyn('Samples'))], // audio + [constraint.linear(frameDim('output', 0, 0), frameDim('input', 0), FRAME_SAMPLES)] +); + +const ENCODER_SPEC = method( + 'forward', + [f32(Dyn('F'), FRAME_SAMPLES)], // audio, one row per frame + [i64(CODEBOOKS, Dyn('F'))], // codes + [constraint.equality(frameDim('input', 0, 0), frameDim('output', 0))] +); + +// Cumulative share of the slots committed after each main pass, from 0 to 1. +function commitSchedule(steps: number): number[] { + 'worklet'; + const schedule: number[] = []; + for (let i = 0; i <= steps; i++) { + const t = i / steps; + schedule.push((T_SHIFT * t) / (1 - (1 - T_SHIFT) * t)); + } + return schedule; +} + +// One main pass over codebooks 0-1 of `tokens`, in place: commits the pass's share of the masked slots by +// Gumbel-perturbed confidence, then, except on the last pass, re-masks the least likely codes held before it. +function samplePass( + tokens: BigInt64Array, + frames: number, + pass: { confidence: Float32Array; prediction: BigInt64Array; held: Float32Array }, + step: number, + schedule: readonly number[], + remaskRatio: number, + random: () => number +): void { + 'worklet'; + const slots = 2 * frames; + const last = step === schedule.length - 2; + const score = new Float64Array(slots); + const masked: number[] = []; + const held: number[] = []; + for (let i = 0; i < slots; i++) { + const gumbel = -Math.log(-Math.log(random() + 1e-10) + 1e-10); + if (tokens[i] !== MASK_CODE) { + held.push(i); + continue; + } + const penalty = CODEBOOK_PENALTY * Math.floor(i / frames); + score[i] = (pass.confidence[i]! - penalty) / POSITION_TEMPERATURE + gumbel; + masked.push(i); + } + + const share = Math.ceil(slots * (schedule[step + 1]! - schedule[step]!)); + const count = last ? masked.length : Math.min(masked.length, share); + masked.sort((a, b) => score[b]! - score[a]!); + for (const i of masked.slice(0, count)) tokens[i] = pass.prediction[i]!; + + if (last || remaskRatio <= 0) return; + held.sort((a, b) => pass.held[a]! - pass.held[b]!); + for (const i of held.slice(0, roundHalfEven(remaskRatio * count))) tokens[i] = MASK_CODE; +} + +/** + * Creates a Reverb Text-to-Speech pipeline: speech in the voice of a short + * recording, in 7 languages. + * @category Speech / Tasks + * @typeParam K Preset voice keys (inferred from `config.voices`). + * @param config Paths to the programs, tokenizer, and voices. See + * {@link ReverbTtsModel}. + * @param runtime Optional worklet runtime to run inference on. + * @returns The {@link ReverbTextToSpeech} runner. + * @throws {RnExecuTorchError} With code `LOAD_FAILED` if an asset fails to + * load, or `SCHEMA_MISMATCH` if a program does not match Reverb's. + */ +export async function createReverbTextToSpeech( + config: ReverbTtsModel, + runtime?: WorkletRuntime +): Promise> { + const scope = createResourceScope(); + let disposed = false; + const dispose = (): void => { + disposed = true; + scope.dispose(); + }; + + try { + const load = wrapAsync(loadModel, runtime); + const backbone = scope.track(await load(config.modelPaths.backbone)); + const decoder = scope.track(await load(config.modelPaths.codecDecoder)); + const tokenizer = scope.track(await wrapAsync(loadTokenizer, runtime)(config.tokenizerPath)); + + const backboneSpec = validateSpec(backbone.schema, { default: BACKBONE_SPEC }); + const [promptTokens, referenceFrames, targetFrames] = backboneSpec.dims.range('L', 'R', 'T'); + const [decodedFrames] = validateSpec(decoder.schema, { default: DECODER_SPEC }).dims.range('T'); + const minFrames = Math.max(MIN_FRAMES, targetFrames.min, decodedFrames.min); + const maxFrames = Math.min(targetFrames.max, decodedFrames.max); + + // A voice the programs take: [8, frames] codes in range, a transcript, and a supported language. + const validateVoice = ( + voice: ReverbVoice, + code: 'INVALID_ARGUMENT' | 'LOAD_FAILED', + ctx: string + ): ReverbVoice => { + const frames = voice.codes.length / CODEBOOKS; + const valid = + Number.isInteger(frames) && + frames >= referenceFrames.min && + frames <= referenceFrames.max && + voice.codes.every((c) => c >= 0n && c < MASK_CODE) && + voice.transcript.trim().length > 0 && + REVERB_LANGUAGES.includes(voice.language); + if (!valid) { + throw RnExecuTorchError( + code, + `${ctx}: A voice holds [${CODEBOOKS}, frames] codes below ${CODES}, with ` + + `${referenceFrames.min} to ${referenceFrames.max} frames, a transcript, and a ` + + `supported language.` + ); + } + return voice; + }; + + const voices = {} as Record; + for (const [key, path] of Object.entries(config.voices ?? {}) as [K, string][]) { + const json = JSON.parse(await RNBlobUtil.fs.readFile(path, 'utf8')); + voices[key] = validateVoice(parseVoice(json), 'LOAD_FAILED', `Voice '${String(key)}'`); + } + + const synthesizeChunkWorklet = ( + ids: Int32Array, + voice: ReverbVoice, + frames: number, + chunkOpts: { + steps: number; + remaskRatio: number; + seed: number; + postprocess: Required | false; + pause: number; + } + ): Float32Array => { + 'worklet'; + + const { steps, remaskRatio, seed, postprocess, pause } = chunkOpts; + const references = voice.codes.length / CODEBOOKS; + const slots = 2 * frames; + const tokens = new BigInt64Array(CODEBOOKS * frames).fill(MASK_CODE); + const pass = { + confidence: new Float32Array(slots), + prediction: new BigInt64Array(slots), + held: new Float32Array(slots), + }; + const schedule = commitSchedule(steps); + const random = mulberry32(seed); + + // Collect execution tensors for cleanup in a single try/finally block + const auxTensors: Tensor[] = []; + const alloc = (...args: Parameters): Tensor => { + const t = tensor(...args); + auxTensors.push(t); + return t; + }; + + try { + // 1. Prefill the prompt and the reference into the K/V cache + const tCache = alloc('float16', [ + 2, + CACHE_LAYERS, + CACHE_HEADS, + ids.length + references, + CACHE_HEAD_DIM, + ]); + const tIds = alloc('int64', [ids.length], BigInt64Array.from(ids, BigInt)); + const tReference = alloc('int64', [CODEBOOKS, references], voice.codes); + backbone.execute('prefill', [tIds, tReference], [tCache]); + + // 2. Main passes over codebooks 0-1 + const tTokens = alloc('int64', [CODEBOOKS, frames]); + const tConfidence = alloc('float32', [2, frames]); + const tPrediction = alloc('int64', [2, frames]); + const tHeld = alloc('float32', [2, frames]); + for (let step = 0; step < steps; step++) { + tTokens.setData(tokens); + backbone.execute('step', [tTokens, tCache], [tConfidence, tPrediction, tHeld]); + tConfidence.getData(pass.confidence); + tPrediction.getData(pass.prediction); + tHeld.getData(pass.held); + samplePass(tokens, frames, pass, step, schedule, remaskRatio, random); + } + + // 3. Polish codebooks 2-7 + const tReferenceFrames = alloc('int64', [1], BigInt64Array.of(BigInt(references))); + const tFine = alloc('int64', [CODEBOOKS - 2, frames]); + tTokens.setData(tokens); + backbone.execute('polish', [tTokens, tReferenceFrames, tCache], [tFine]); + tokens.set(tFine.getData(new BigInt64Array((CODEBOOKS - 2) * frames)), slots); + + // 4. Decode + const tAudio = alloc('float32', [frames * FRAME_SAMPLES]); + tTokens.setData(tokens); + decoder.execute('forward', [tTokens], [tAudio]); + let audio: Float32Array = tAudio.getData(new Float32Array(frames * FRAME_SAMPLES)); + + // 5. Cut the end, then trim the silent edges + if (postprocess) { + const { endCut, silenceThreshold, keep } = postprocess; + const end = Math.max(0, audio.length - Math.floor(endCut * REVERB_SAMPLE_RATE)); + audio = trimSilence(audio.subarray(0, end), REVERB_SAMPLE_RATE, silenceThreshold, keep); + } + + // 6. Append the pause the chunk's final punctuation calls for + const result = new Float32Array(audio.length + Math.round(pause * REVERB_SAMPLE_RATE)); + result.set(audio); + return result; + } finally { + auxTensors.forEach((t) => t.dispose()); + } + }; + + const synthesizeChunk = wrapAsync(synthesizeChunkWorklet, runtime); + + const checkLanguage = (language: unknown, ctx: string): void => { + if (!REVERB_LANGUAGES.includes(language as ReverbLanguage)) { + throw RnExecuTorchError( + 'INVALID_ARGUMENT', + `${ctx}: Unsupported language: ${String(language)}.` + ); + } + }; + + const resolveVoice = (voice: K | ReverbVoice): ReverbVoice => { + const resolved = typeof voice === 'object' ? voice : voices[voice]; + if (!resolved) { + throw RnExecuTorchError('INVALID_ARGUMENT', `synthesize: Unknown voice: ${String(voice)}.`); + } + return validateVoice(resolved, 'INVALID_ARGUMENT', 'synthesize'); + }; + + let encoder: + | Promise<{ encode: (audio: Float32Array) => Promise; frames: Range }> + | undefined; + const loadEncoder = async () => { + if (!config.modelPaths.codecEncoder) { + throw RnExecuTorchError('INVALID_STATE', 'encodeVoice: The config names no codecEncoder.'); + } + const model = await load(config.modelPaths.codecEncoder); + try { + if (disposed) { + throw RnExecuTorchError('RESOURCE_DISPOSED', 'encodeVoice: The pipeline was disposed.'); + } + const [frames] = validateSpec(model.schema, { default: ENCODER_SPEC }).dims.range('F'); + scope.track(model); + const encodeWorklet = (audio: Float32Array): BigInt64Array => { + 'worklet'; + const count = audio.length / FRAME_SAMPLES; + const tAudio = tensor('float32', [count, FRAME_SAMPLES], audio); + const tCodes = tensor('int64', [CODEBOOKS, count]); + try { + model.execute('forward', [tAudio], [tCodes]); + return tCodes.getData(new BigInt64Array(CODEBOOKS * count)); + } finally { + tAudio.dispose(); + tCodes.dispose(); + } + }; + return { encode: wrapAsync(encodeWorklet, runtime), frames }; + } catch (error) { + model.dispose(); + throw error; + } + }; + + const encodeVoice = async ( + audio: Float32Array, + transcript: string, + language: ReverbLanguage + ): Promise => { + if (disposed) { + throw RnExecuTorchError('RESOURCE_DISPOSED', 'encodeVoice: The pipeline was disposed.'); + } + checkLanguage(language, 'encodeVoice'); + if (!transcript.trim()) { + throw RnExecuTorchError('INVALID_ARGUMENT', 'encodeVoice: The transcript cannot be empty.'); + } + encoder ??= loadEncoder().catch((error) => { + encoder = undefined; + throw error; + }); + const { encode, frames } = await encoder; + + const count = Math.floor(audio.length / FRAME_SAMPLES); + const min = Math.max(frames.min, referenceFrames.min); + const max = Math.min(frames.max, referenceFrames.max); + if (count < min || count > max) { + throw RnExecuTorchError( + 'INVALID_ARGUMENT', + `encodeVoice: The audio must last ${min / REVERB_FRAME_RATE} to ` + + `${max / REVERB_FRAME_RATE} s at ${REVERB_SAMPLE_RATE} Hz ` + + `(got ${audio.length / REVERB_SAMPLE_RATE} s).` + ); + } + const codes = await encode(audio.slice(0, count * FRAME_SAMPLES)); + + return { codes, transcript: prepareText(transcript, language), language }; + }; + + let isSynthesizing = false; + const synthesizeStop = (): void => { + isSynthesizing = false; + }; + + async function* synthesize( + text: string, + options: ReverbTtsOptions + ): AsyncGenerator { + if (isSynthesizing) { + throw RnExecuTorchError('INVALID_STATE', 'synthesize: Synthesis is already in progress.'); + } + + if (!text || !text.trim()) { + throw RnExecuTorchError('INVALID_ARGUMENT', 'synthesize: Input text cannot be empty.'); + } + + const { language } = options; + checkLanguage(language, 'synthesize'); + const voice = resolveVoice(options.voice); + const references = voice.codes.length / CODEBOOKS; + + const speed = options.speed ?? DEFAULT_SPEED; + if (!(speed >= MIN_SPEED && speed <= MAX_SPEED)) { + throw RnExecuTorchError( + 'INVALID_ARGUMENT', + `synthesize: speed must be between ${MIN_SPEED} and ${MAX_SPEED}.` + ); + } + + // A chunk at the limit must still fit the backbone's target window. + const maxChunkDuration = options.maxChunkDuration ?? DEFAULT_MAX_CHUNK_DURATION; + const maxDuration = maxFrames / REVERB_FRAME_RATE; + if (!(maxChunkDuration > 0 && maxChunkDuration <= maxDuration)) { + throw RnExecuTorchError( + 'INVALID_ARGUMENT', + `synthesize: maxChunkDuration must be above 0 and at most ${maxDuration} s.` + ); + } + + const steps = options.sampling?.steps ?? DEFAULT_STEPS; + const remaskRatio = options.sampling?.remaskRatio ?? DEFAULT_REMASK_RATIO; + const seed = options.sampling?.seed ?? Math.floor(Math.random() * 0x100000000); + if (!Number.isInteger(steps) || steps <= 0) { + throw RnExecuTorchError( + 'INVALID_ARGUMENT', + 'synthesize: steps must be a positive integer.' + ); + } + if (!(remaskRatio >= 0 && remaskRatio <= 1)) { + throw RnExecuTorchError('INVALID_ARGUMENT', 'synthesize: remaskRatio must be in [0, 1].'); + } + if (!Number.isInteger(seed)) { + throw RnExecuTorchError('INVALID_ARGUMENT', 'synthesize: seed must be an integer.'); + } + const postprocess = + options.postprocess === false + ? false + : { + endCut: options.postprocess?.endCut ?? DEFAULT_POSTPROCESS.endCut, + silenceThreshold: + options.postprocess?.silenceThreshold ?? DEFAULT_POSTPROCESS.silenceThreshold, + keep: options.postprocess?.keep ?? DEFAULT_POSTPROCESS.keep, + }; + + const encodePrompt = (chunk: string) => + tokenizer.encode(buildPrompt(language, voice.transcript, chunk)); + if (encodePrompt('').length >= promptTokens.max) { + throw RnExecuTorchError( + 'INVALID_ARGUMENT', + `synthesize: The voice's transcript leaves no room for text in the ` + + `${promptTokens.max}-token prompt.` + ); + } + const targetLength = (chunk: string) => + estimateTargetFrames(chunk, voice.transcript, references) / speed; + + isSynthesizing = true; + try { + const chunks = partitionReverbText( + prepareText(text, language), + (chunk) => targetLength(chunk) / REVERB_FRAME_RATE, + { + maxDuration: maxChunkDuration, + fits: (chunk) => encodePrompt(chunk).length <= promptTokens.max, + } + ); + + for (const [chunkIndex, chunk] of chunks.entries()) { + if (!isSynthesizing) break; + + const ids = encodePrompt(chunk); + if (ids.length > promptTokens.max) { + throw RnExecuTorchError( + 'INVALID_ARGUMENT', + `synthesize: "${chunk}" does not fit the ${promptTokens.max}-token prompt.` + ); + } + const frames = Math.min( + Math.max(roundHalfEven(targetLength(chunk)), minFrames), + maxFrames + ); + const audio = await synthesizeChunk(ids, voice, frames, { + steps, + remaskRatio, + seed: seed + chunkIndex, + postprocess, + pause: pauseAfter(chunk), + }); + yield { + audio, + sampleRate: REVERB_SAMPLE_RATE, + duration: audio.length / REVERB_SAMPLE_RATE, + chunkIndex, + totalChunks: chunks.length, + }; + } + } finally { + isSynthesizing = false; + } + } + + return { dispose, encodeVoice, synthesize, synthesizeStop }; + } catch (error) { + dispose(); + throw error; + } +} diff --git a/packages/react-native-executorch/src/extensions/speech/utils/reverbPartitioner.ts b/packages/react-native-executorch/src/extensions/speech/utils/reverbPartitioner.ts new file mode 100644 index 0000000000..0d98a8d1ba --- /dev/null +++ b/packages/react-native-executorch/src/extensions/speech/utils/reverbPartitioner.ts @@ -0,0 +1,92 @@ +/** + * Splits long text into chunks within a duration limit. + */ + +import { RnExecuTorchError } from '../../../core/error'; +import { EOS_PATTERN, PAUSE_PATTERN } from './textPartitioner'; + +const DEFAULT_MAX_DURATION = 20; +const CLOSERS = /["'”’»)\]}]+$/u; // closing quotes and brackets after a word's punctuation + +/** + * Options for {@link partitionReverbText}. + * @category Speech / Types + */ +export type ReverbPartitionOptions = { + /** Max seconds per chunk. Defaults to 20. */ + readonly maxDuration?: number; + /** A further condition every chunk must meet. */ + readonly fits?: (chunk: string) => boolean; +}; + +// Splits words into runs, each ending at a word whose final mark (past closers) matches `boundary`. +function splitAfter(words: readonly string[], boundary: RegExp): string[][] { + const runs: string[][] = [[]]; + for (const word of words) { + runs[runs.length - 1]!.push(word); + if (boundary.test(word.replace(CLOSERS, '').slice(-1))) runs.push([]); + } + return runs.filter((run) => run.length > 0); +} + +/** + * Splits text into chunks of at most `maxDuration` seconds each. Greedy, + * coarsest boundary first: whole sentences, then other punctuation, then words. + * A single word that doesn't fit is a chunk of its own. + * @category Speech / Functions + * @param text The text. + * @param duration Estimated seconds of speech of a chunk. + * @param options See {@link ReverbPartitionOptions}. + * @returns The chunks, which joined with spaces give back the text. + * @throws {RnExecuTorchError} With code `INVALID_ARGUMENT` if `maxDuration` is + * not positive. + */ +export function partitionReverbText( + text: string, + duration: (chunk: string) => number, + options?: ReverbPartitionOptions +): string[] { + const maxDuration = options?.maxDuration ?? DEFAULT_MAX_DURATION; + if (!(maxDuration > 0)) { + throw RnExecuTorchError( + 'INVALID_ARGUMENT', + 'partitionReverbText: maxDuration must be positive.' + ); + } + const fits = (words: readonly string[]): boolean => { + const chunk = words.join(' '); + return duration(chunk) <= maxDuration && (options?.fits?.(chunk) ?? true); + }; + + // A unit too long on its own is replaced by its pieces at the next finer boundary. + const boundaries = [EOS_PATTERN, PAUSE_PATTERN]; + const queue = splitAfter(text.split(/\s+/).filter(Boolean), EOS_PATTERN).map((words) => ({ + words, + level: 0, + })); + const chunks: string[] = []; + let current: string[] = []; + + while (queue.length > 0) { + const unit = queue.shift()!; + if (fits([...current, ...unit.words])) { + current = [...current, ...unit.words]; + } else if (current.length > 0) { + chunks.push(current.join(' ')); + current = []; + queue.unshift(unit); + } else if (unit.words.length === 1) { + chunks.push(unit.words[0]!); + } else { + let level = unit.level + 1; + let pieces = level < boundaries.length ? splitAfter(unit.words, boundaries[level]!) : []; + if (pieces.length < 2) { + level = boundaries.length; + pieces = unit.words.map((word) => [word]); + } + queue.unshift(...pieces.map((words) => ({ words, level }))); + } + } + if (current.length > 0) chunks.push(current.join(' ')); + return chunks; +} diff --git a/packages/react-native-executorch/src/extensions/speech/utils/reverbUtils.ts b/packages/react-native-executorch/src/extensions/speech/utils/reverbUtils.ts new file mode 100644 index 0000000000..d89d0f96db --- /dev/null +++ b/packages/react-native-executorch/src/extensions/speech/utils/reverbUtils.ts @@ -0,0 +1,192 @@ +/** + * Text preparation, length estimate, and post-processing utilities for Reverb. + */ + +import { Verbalizer, type LanguageHandle } from 'verbalis/core'; +import { language as de } from 'verbalis/de'; +import { language as en } from 'verbalis/en'; +import { language as es } from 'verbalis/es'; +import { language as fr } from 'verbalis/fr'; +import { language as it } from 'verbalis/it'; +import { language as pl } from 'verbalis/pl'; +import { language as pt } from 'verbalis/pt'; + +import { RnExecuTorchError } from '../../../core/error'; +import type { ReverbLanguage, ReverbVoice } from '../tasks/reverbTextToSpeech'; + +const BLIP_WINDOWS = 12; // an edge sound of at most 120 ms is a codec burst +const SOUND_GAP_WINDOWS = 20; // loud windows over 200 ms apart are separate sounds + +const VERBALIS_LANGUAGES: Record LanguageHandle> = { + de, + en, + es, + fr, + it, + pl, + pt, +}; +const verbalizers: Partial> = {}; + +/** + * Parses a voice file's JSON: `{ codes, transcript, language }`, the codes as + * `8 × frames` integers, row-major. + * @param json The parsed JSON. + * @returns The voice. + * @throws {RnExecuTorchError} With code `LOAD_FAILED` on a missing or + * mistyped field. + */ +export function parseVoice(json: any): ReverbVoice { + const { codes, transcript, language } = json ?? {}; + const valid = + Array.isArray(codes) && + codes.every((code) => Number.isInteger(code)) && + typeof transcript === 'string' && + typeof language === 'string'; + if (!valid) { + throw RnExecuTorchError('LOAD_FAILED', 'parseVoice: Invalid voice JSON format.'); + } + return { + codes: BigInt64Array.from(codes as number[], BigInt), + transcript, + language: language as ReverbLanguage, + }; +} + +/** + * Writes text the way it is read aloud with Verbalis: numbers, symbols, and + * abbreviations as words, and quotes, dashes, ellipses, and spaces in the forms + * Reverb was trained on. JS thread only. + * @param text The text. + * @param language The text's language. + * @returns The prepared text. + */ +export function prepareText(text: string, language: ReverbLanguage): string { + const verbalizer = (verbalizers[language] ??= new Verbalizer(VERBALIS_LANGUAGES[language]())); + return verbalizer.process(text); +} + +/** + * The backbone's prompt: the text's language, then the voice's transcript and + * the text. + * @param language The text's language. + * @param transcript The voice's transcript. + * @param text The text to speak. + * @returns The prompt. + */ +export function buildPrompt(language: ReverbLanguage, transcript: string, text: string): string { + return ( + `<|denoise|><|lang_start|>${language}<|lang_end|><|instruct_start|>None<|instruct_end|>` + + `<|text_start|>${transcript} ${text}<|text_end|>` + ); +} + +// Spoken weight by Unicode class: spaces, marks, punctuation and symbols, digits, the rest. +function spokenWeight(text: string): number { + 'worklet'; + let weight = 0; + for (const char of text) { + if (/\p{Z}/u.test(char)) weight += 0.2; + else if (/\p{M}/u.test(char)) weight += 0; + else if (/[\p{P}\p{S}]/u.test(char)) weight += 0.5; + else if (/\p{N}/u.test(char)) weight += 3.5; + else weight += 1; + } + return weight; +} + +/** + * Estimates the codec frames (25 per second) it takes to speak `text` at a + * reference clip's pace, as Echo's `estimate_frames` does. + * @category Speech / Functions + * @param text The text to speak. + * @param referenceText The reference clip's transcript. + * @param referenceFrames The reference clip's length in frames. + * @returns The estimate, at least 1. + */ +export function estimateTargetFrames( + text: string, + referenceText: string, + referenceFrames: number +): number { + 'worklet'; + let frames = (spokenWeight(text) / Math.max(spokenWeight(referenceText), 1)) * referenceFrames; + if (frames < 50) frames = 50 * Math.pow(frames / 50, 1 / 3); // short texts get relatively more time + return Math.max(1, Math.floor(frames)); +} + +/** + * Rounds half to even, like Python's `round`. + * @param value The value. + * @returns The rounded value. + */ +export function roundHalfEven(value: number): number { + 'worklet'; + const rounded = Math.round(value); + return rounded - value === 0.5 && rounded % 2 !== 0 ? rounded - 1 : rounded; +} + +/** + * Trims silent edges like Voice Studio: in 10 ms windows, dropping an isolated + * edge sound of at most 120 ms. + * @param audio The audio. + * @param sampleRate Its sampling rate in Hz. + * @param threshold Level in dBFS below which a window is silent. + * @param keep Seconds of silence kept at each edge. + * @returns A view of `audio` without the silent edges, or `audio` if all silent. + */ +export function trimSilence( + audio: Float32Array, + sampleRate: number, + threshold: number, + keep: number +): Float32Array { + 'worklet'; + const window = Math.floor(sampleRate / 100); + const windows = Math.floor(audio.length / window); + const sounds: { first: number; last: number; loud: number }[] = []; + for (let w = 0; w < windows; w++) { + let energy = 0; + for (let i = w * window; i < (w + 1) * window; i++) { + const sample = audio[i]!; + energy += sample * sample; // `** 2` makes this loop 2.5x slower in Hermes + } + if (20 * Math.log10(Math.sqrt(energy / window) + 1e-10) < threshold) continue; + const sound = sounds[sounds.length - 1]; + if (sound && w - sound.last <= SOUND_GAP_WINDOWS) { + sound.last = w; + sound.loud++; + } else { + sounds.push({ first: w, last: w, loud: 1 }); + } + } + if (sounds.length === 0) return audio; + + if (sounds.length > 1 && sounds[0]!.loud <= BLIP_WINDOWS) sounds.shift(); + if (sounds.length > 1 && sounds[sounds.length - 1]!.loud <= BLIP_WINDOWS) sounds.pop(); + const margin = roundHalfEven(keep * 100); + const start = Math.max(sounds[0]!.first - margin, 0); + const end = Math.min(sounds[sounds.length - 1]!.last + 1 + margin, windows); + return audio.subarray(start * window, end * window); +} + +/** + * Pause after a chunk by its final punctuation, in milliseconds: Kokoro's + * values, `...` taking the ellipsis's. + */ +// prettier-ignore +export const REVERB_PAUSE_MS: Record = { + '.': 375, '?': 500, '!': 250, ';': 400, '...': 1000, ',': 130, ':': 250, '-': 200, +}; + +/** + * The pause after a chunk, by its final punctuation past closing quotes. + * @param chunk The chunk. + * @returns The pause in seconds. + */ +export function pauseAfter(chunk: string): number { + 'worklet'; + const end = chunk.replace(/[\s"'”’»)\]}]+$/u, ''); + const mark = end.endsWith('...') ? '...' : end.slice(-1); + return (REVERB_PAUSE_MS[mark] ?? 0) / 1000; +} diff --git a/packages/react-native-executorch/src/extensions/speech/utils/textPartitioner.ts b/packages/react-native-executorch/src/extensions/speech/utils/textPartitioner.ts index 6651beaa34..720233a6cc 100644 --- a/packages/react-native-executorch/src/extensions/speech/utils/textPartitioner.ts +++ b/packages/react-native-executorch/src/extensions/speech/utils/textPartitioner.ts @@ -6,9 +6,17 @@ import { RnExecuTorchError } from '../../../core/error'; type Tag = 'eos' | 'pause' | 'whitespace'; -// Punctuation Regex Patterns -const EOS_PATTERN = /[.?!;…|।॥¿¡。.!?;]/; -const PAUSE_PATTERN = /[,:\u2014\u00AB\u00BB、,:〜~]/; +/** + * Sentence-ending punctuation. + * @category Speech / Constants + */ +export const EOS_PATTERN = /[.?!;…|।॥¿¡。.!?;]/; + +/** + * Punctuation pausing within a sentence: commas, colons, dashes. + * @category Speech / Constants + */ +export const PAUSE_PATTERN = /[,:\u2014\u00AB\u00BB、,:〜~]/; const WHITESPACE_PATTERN = /\s/; // Standard Partitioning Constants diff --git a/packages/react-native-executorch/src/hooks/useTextToSpeech.ts b/packages/react-native-executorch/src/hooks/useTextToSpeech.ts index 643636ee43..fbe0409d5b 100644 --- a/packages/react-native-executorch/src/hooks/useTextToSpeech.ts +++ b/packages/react-native-executorch/src/hooks/useTextToSpeech.ts @@ -10,6 +10,11 @@ import { type SupertonicTtsModel, type SupertonicTextToSpeech, } from '../extensions/speech/tasks/supertonicTextToSpeech'; +import { + createReverbTextToSpeech, + type ReverbTtsModel, + type ReverbTextToSpeech, +} from '../extensions/speech/tasks/reverbTextToSpeech'; /** * React hook to load and run the Kokoro Text-to-Speech pipeline. @@ -82,20 +87,55 @@ export function useTextToSpeech( synthesizeStop: SupertonicTextToSpeech['synthesizeStop'] | undefined; }; +/** + * React hook to download, load, and run the Reverb Text-to-Speech pipeline, + * releasing native memory on unmount or config change. + * + * For imperative usage, see {@link createReverbTextToSpeech}. + * @category Hooks + * @typeParam K Preset voice keys. + * @param config The Reverb TTS model configuration. See {@link ReverbTtsModel}. + * @param options Load and caching options. See {@link ResourceOptions}. + * @returns The same object as {@link ReverbTextToSpeech} (without `dispose`), + * combined with loading state and download progress. + * @see {@link ReverbTextToSpeech} + */ +export function useTextToSpeech( + config: ReverbTtsModel, + options?: ResourceOptions +): { + /** Whether the pipeline is loaded and ready to synthesize. */ + isReady: boolean; + /** The download or load error, if any. */ + error: Error | undefined; + /** Download progress across every asset, in percent. */ + downloadProgress: number; + /** The config with every remote URL resolved to a local path. */ + resource: ReverbTtsModel | undefined; + /** Streams synthesized audio chunks. Undefined until the pipeline is ready. */ + synthesize: ReverbTextToSpeech['synthesize'] | undefined; + /** Cancels an in-flight synthesis. Undefined until the pipeline is ready. */ + synthesizeStop: ReverbTextToSpeech['synthesizeStop'] | undefined; + /** Encodes a recording into a voice. Undefined until the pipeline is ready. */ + encodeVoice: ReverbTextToSpeech['encodeVoice'] | undefined; +}; + export function useTextToSpeech( - config: KokoroTtsModel | SupertonicTtsModel, + config: KokoroTtsModel | SupertonicTtsModel | ReverbTtsModel, options?: ResourceOptions ) { // Each config names the pipeline it belongs to, so the factory is resolved // from that tag alone. const factory = ( - ttsConfig: KokoroTtsModel | SupertonicTtsModel - ): Promise | SupertonicTextToSpeech> => { + ttsConfig: KokoroTtsModel | SupertonicTtsModel | ReverbTtsModel + ): Promise | SupertonicTextToSpeech | ReverbTextToSpeech> => { switch (ttsConfig.name) { case 'kokoro': return createKokoroTextToSpeech(ttsConfig); case 'supertonic': return createSupertonicTextToSpeech(ttsConfig); + case 'reverb': + return createReverbTextToSpeech(ttsConfig); } }; @@ -109,5 +149,7 @@ export function useTextToSpeech( resource, synthesize: model?.synthesize, synthesizeStop: model?.synthesizeStop, + // Only Reverb encodes voices. + ...(model && 'encodeVoice' in model ? { encodeVoice: model.encodeVoice } : {}), }; } diff --git a/packages/react-native-executorch/src/index.ts b/packages/react-native-executorch/src/index.ts index b5c6fc885f..b71db4a83a 100644 --- a/packages/react-native-executorch/src/index.ts +++ b/packages/react-native-executorch/src/index.ts @@ -98,6 +98,7 @@ export * from './extensions/speech/tasks/whisperSpeechToText'; export * from './extensions/speech/tasks/qwen3AsrSpeechToText'; export * from './extensions/speech/tasks/supertonicTextToSpeech'; export * from './extensions/speech/tasks/kokoroTextToSpeech'; +export * from './extensions/speech/tasks/reverbTextToSpeech'; // Vector search over embeddings export * from './extensions/vectorStore'; diff --git a/packages/react-native-executorch/tsconfig.build.json b/packages/react-native-executorch/tsconfig.build.json index cf63e46cf5..0f9f4ff56b 100644 --- a/packages/react-native-executorch/tsconfig.build.json +++ b/packages/react-native-executorch/tsconfig.build.json @@ -1,4 +1,4 @@ { "extends": "./tsconfig", - "exclude": ["example", "lib", "__tests__", "jest.config.js"] + "exclude": ["example", "lib", "__tests__", "jest.config.js", "third-party"] } diff --git a/packages/react-native-executorch/tsconfig.json b/packages/react-native-executorch/tsconfig.json index f355221ebe..d6fafac7ef 100644 --- a/packages/react-native-executorch/tsconfig.json +++ b/packages/react-native-executorch/tsconfig.json @@ -27,5 +27,6 @@ "strict": true, "target": "ESNext", "verbatimModuleSyntax": true - } + }, + "exclude": ["node_modules", "lib", "third-party"] } diff --git a/yarn.lock b/yarn.lock index 0d19f9e435..4cc305b2b2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -13001,6 +13001,7 @@ __metadata: react-native-worklets: "npm:0.13.0" test-renderer: "npm:^1.2.0" typescript: "npm:~5.9.2" + verbalis: "npm:^0.1.1" peerDependencies: "@kesha-antonov/react-native-background-downloader": ">=4.4.0" react: "*" @@ -15046,6 +15047,13 @@ __metadata: languageName: node linkType: hard +"verbalis@npm:^0.1.1": + version: 0.1.1 + resolution: "verbalis@npm:0.1.1" + checksum: 10/012f4789d08a0e3ea729db7f192f2e453e525f90bc278e780e1ca17baa82e9f47ab61c8ce7307d4742142fb218f1e57556229719164d9cb25c9631584b096922 + languageName: node + linkType: hard + "vlq@npm:^1.0.0": version: 1.0.1 resolution: "vlq@npm:1.0.1" From 1e67715401f07888f7eedcd00f2a0a056d42d027 Mon Sep 17 00:00:00 2001 From: IgorSwat Date: Mon, 16 Nov 2026 14:22:26 +0100 Subject: [PATCH 4/8] Add Reverb-TTS demo app & model registry --- apps/speech/app/_layout.tsx | 7 + apps/speech/app/index.tsx | 6 + .../app/reverb-text-to-speech/index.tsx | 1073 +++++++++++++++++ .../02-extensions/speech/02-text-to-speech.md | 60 +- .../api/__snapshots__/apiSurface.test.ts.snap | 1 + .../__tests__/api/modelRegistry.test.ts | 4 +- .../__tests__/api/modelVariants.test.ts | 3 + .../react-native-executorch/src/constants.ts | 16 + .../react-native-executorch/src/models.ts | 75 +- 9 files changed, 1235 insertions(+), 10 deletions(-) create mode 100644 apps/speech/app/reverb-text-to-speech/index.tsx diff --git a/apps/speech/app/_layout.tsx b/apps/speech/app/_layout.tsx index e98be4cd43..546f2ef6f4 100644 --- a/apps/speech/app/_layout.tsx +++ b/apps/speech/app/_layout.tsx @@ -55,6 +55,13 @@ export default function Layout() { title: 'Text-to-Speech (Kokoro)', }} /> + ); } diff --git a/apps/speech/app/index.tsx b/apps/speech/app/index.tsx index 61cd5f6097..80d5a8a344 100644 --- a/apps/speech/app/index.tsx +++ b/apps/speech/app/index.tsx @@ -35,6 +35,12 @@ export default function Home() { > Text-to-Speech (Kokoro) + router.navigate('reverb-text-to-speech/')} + > + Text-to-Speech (Reverb) + ); diff --git a/apps/speech/app/reverb-text-to-speech/index.tsx b/apps/speech/app/reverb-text-to-speech/index.tsx new file mode 100644 index 0000000000..81139a26a9 --- /dev/null +++ b/apps/speech/app/reverb-text-to-speech/index.tsx @@ -0,0 +1,1073 @@ +import React, { useCallback, useEffect, useRef, useState } from 'react'; +import { + Animated, + Platform, + ScrollView, + StyleSheet, + Text, + TextInput, + TouchableOpacity, + View, +} from 'react-native'; +import { AudioManager, AudioRecorder } from 'react-native-audio-api'; +import RNBlobUtil from 'react-native-blob-util'; +import DeviceInfo from 'react-native-device-info'; +import { + models, + useSpeechToText, + useTextToSpeech, + REVERB_DEFAULT_VOICE_NAMES, + REVERB_FRAME_RATE, + REVERB_SAMPLE_RATE, + WHISPER_SAMPLE_RATE_HZ, + type ReverbDefaultVoiceName, + type ReverbLanguage, + type ReverbVoice, +} from 'react-native-executorch'; + +import ScreenWrapper from '../../components/ScreenWrapper'; +import { ModelPicker } from '../../components/ModelPicker'; +import { ModelStatus } from '../../components/ModelStatus'; +import { Button } from '../../components/Button'; +import { useAudioPlayer } from '../../hooks/useAudioPlayer'; +import { theme } from '../../theme'; + +type ReverbVariant = Exclude; + +const MODEL_OPTIONS: { label: string; value: ReverbVariant; platform: typeof Platform.OS }[] = [ + { label: 'MLX fp16', value: 'MLX_COREML_FP16', platform: 'ios' }, + { label: 'MLX int8', value: 'MLX_COREML_INT8', platform: 'ios' }, + { label: 'MLX int4', value: 'MLX_COREML_INT4', platform: 'ios' }, + { label: 'Vulkan int4', value: 'VULKAN_INT4', platform: 'android' }, +]; + +// Transcribes the recordings a new voice is made from. +const WHISPER_MODEL = models.speechToText.WHISPER.BASE.DEFAULT; + +const LANGUAGES: { label: string; value: ReverbLanguage }[] = [ + { label: 'English', value: 'en' }, + { label: 'German', value: 'de' }, + { label: 'Spanish', value: 'es' }, + { label: 'French', value: 'fr' }, + { label: 'Italian', value: 'it' }, + { label: 'Polish', value: 'pl' }, + { label: 'Portuguese', value: 'pt' }, +]; + +// cspell:disable +const SAMPLE_TEXTS: Record = { + en: + 'Reverb speaks in any voice you give it. Record a few seconds of speech, and it reads ' + + 'your text in that voice, entirely on your device, with no internet connection.', + de: + 'Reverb spricht mit jeder Stimme, die du ihm gibst. Nimm ein paar Sekunden Sprache auf, ' + + 'und es liest deinen Text mit dieser Stimme vor, vollständig auf deinem Gerät.', + es: + 'Reverb habla con cualquier voz que le des. Graba unos segundos de voz y leerá tu texto ' + + 'con esa voz, completamente en tu dispositivo y sin conexión a internet.', + fr: + "Reverb parle avec n'importe quelle voix. Enregistrez quelques secondes de parole, et il " + + 'lira votre texte avec cette voix, entièrement sur votre appareil.', + it: + 'Reverb parla con qualsiasi voce tu gli dia. Registra qualche secondo di parlato e leggerà ' + + 'il tuo testo con quella voce, interamente sul tuo dispositivo.', + pl: + 'Reverb mówi dowolnym głosem, który mu podasz. Nagraj kilka sekund mowy, a przeczyta twój ' + + 'tekst tym głosem, w całości na twoim urządzeniu, bez połączenia z internetem.', + pt: + 'O Reverb fala com qualquer voz que lhe der. Grave alguns segundos de fala e ele lerá o seu ' + + 'texto com essa voz, inteiramente no seu dispositivo.', +}; +// cspell:enable + +const PRESETS: { name: ReverbDefaultVoiceName; title: string; subtitle: string }[] = [ + { name: 'standard_male', title: 'Standard', subtitle: 'Male · calm' }, + { name: 'angry_male', title: 'Angry', subtitle: 'Male · intense' }, + { name: 'sensual_female', title: 'Sensual', subtitle: 'Female · soft' }, +]; + +const SPEED_OPTIONS = [0.8, 0.9, 1.0, 1.1, 1.25].map((value) => ({ label: `${value}x`, value })); +const STEPS_OPTIONS = [4, 6, 8, 12, 16].map((value) => ({ + label: value === 8 ? '8 (default)' : `${value}`, + value, +})); +const VARIATION_OPTIONS = [0, 0.15, 0.25, 0.4].map((value) => ({ + label: value === 0.25 ? '0.25 (default)' : `${value}`, + value, +})); + +const MAX_RECORDING_S = 20; +const MIN_RECORDING_S = 1; +const SILENCE_DB = -45; +const SILENCE_BELOW_PEAK_DB = 25; +const METER_BARS = 24; + +type PresetInfo = { transcript: string; language: string; duration: number }; +type CustomVoice = { id: number; name: string; voice: ReverbVoice; duration: number }; +type VoiceChoice = + | { kind: 'preset'; name: ReverbDefaultVoiceName } + | { kind: 'custom'; id: number }; + +const isSimulator = DeviceInfo.isEmulatorSync(); + +const messageOf = (err: unknown) => (err instanceof Error ? err.message : String(err)); + +const formatSeconds = (s: number) => + `${Math.floor(s / 60)}:${String(Math.floor(s % 60)).padStart(2, '0')}`; + +function concat(chunks: readonly Float32Array[]): Float32Array { + const out = new Float32Array(chunks.reduce((n, chunk) => n + chunk.length, 0)); + let offset = 0; + for (const chunk of chunks) { + out.set(chunk, offset); + offset += chunk.length; + } + return out; +} + +// Linear interpolation up; down, the mean of the input samples each output covers (a crude +// anti-alias filter). +function resample(input: Float32Array, from: number, to: number): Float32Array { + if (from === to) return input; + const ratio = from / to; + const out = new Float32Array(Math.floor(input.length / ratio)); + for (let i = 0; i < out.length; i++) { + if (ratio > 1) { + const start = Math.floor(i * ratio); + const end = Math.min(input.length, Math.max(start + 1, Math.floor((i + 1) * ratio))); + let sum = 0; + for (let j = start; j < end; j++) sum += input[j]!; + out[i] = sum / (end - start); + } else { + const x = i * ratio; + const j = Math.floor(x); + out[i] = input[j]! + (input[Math.min(j + 1, input.length - 1)]! - input[j]!) * (x - j); + } + } + return out; +} + +const rms = (audio: Float32Array, start = 0, end = audio.length) => { + let sum = 0; + for (let i = start; i < end; i++) sum += audio[i]! * audio[i]!; + return Math.sqrt(sum / Math.max(1, end - start)); +}; + +// Drops the silence before the first and after the last word, keeping 0.15 s at each edge. Silence +// is relative to the loudest 20 ms: a phone's noise floor can sit above a fixed level (-42 dBFS +// measured on a Galaxy S24 FE). +function trimSilence(audio: Float32Array, rate: number): Float32Array { + const win = Math.round(rate * 0.02); + const levels: number[] = []; + for (let start = 0; start < audio.length; start += win) { + levels.push(rms(audio, start, Math.min(start + win, audio.length))); + } + const peak = levels.reduce((max, level) => Math.max(max, level), 0); + const threshold = Math.max(10 ** (SILENCE_DB / 20), peak * 10 ** (-SILENCE_BELOW_PEAK_DB / 20)); + const first = levels.findIndex((level) => level >= threshold); + if (first < 0) return audio; + let last = levels.length - 1; + while (levels[last]! < threshold) last--; + const pad = Math.round(rate * 0.15); + return audio.slice( + Math.max(0, first * win - pad), + Math.min(audio.length, (last + 1) * win + pad) + ); +} + +// Records mono audio, resampled to Reverb's 24 kHz when the device delivers another rate. +function useVoiceRecorder() { + const recorderRef = useRef(null); + const chunksRef = useRef([]); + const samplesRef = useRef(0); + const rateRef = useRef(REVERB_SAMPLE_RATE); + const [isRecording, setIsRecording] = useState(false); + const [seconds, setSeconds] = useState(0); + const [level, setLevel] = useState(0); + + const start = useCallback(async (): Promise => { + if ((await AudioManager.requestRecordingPermissions()) !== 'Granted') { + return 'Microphone access was denied.'; + } + chunksRef.current = []; + samplesRef.current = 0; + setSeconds(0); + const recorder = new AudioRecorder(); + recorder.onAudioReady( + { sampleRate: REVERB_SAMPLE_RATE, bufferLength: REVERB_SAMPLE_RATE / 10, channelCount: 1 }, + (event) => { + const samples = new Float32Array(event.buffer.getChannelData(0)); + rateRef.current = event.buffer.sampleRate; + chunksRef.current.push(samples); + samplesRef.current += samples.length; + setSeconds(samplesRef.current / rateRef.current); + const db = 20 * Math.log10(rms(samples) + 1e-9); + setLevel(Math.min(1, Math.max(0, (db + 60) / 60))); + } + ); + const result = await recorder.start(); + if (result.status === 'error') { + recorder.clearOnAudioReady(); + return result.message; + } + recorderRef.current = recorder; + setIsRecording(true); + return null; + }, []); + + const stop = useCallback(async (): Promise => { + const recorder = recorderRef.current; + if (!recorder) return null; + recorderRef.current = null; + recorder.clearOnAudioReady(); + await recorder.stop().catch(() => {}); + setIsRecording(false); + setLevel(0); + const audio = concat(chunksRef.current); + chunksRef.current = []; + return resample(audio, rateRef.current, REVERB_SAMPLE_RATE); + }, []); + + useEffect(() => { + return () => { + recorderRef.current?.stop().catch(() => {}); + }; + }, []); + + return { isRecording, seconds, level, start, stop }; +} + +function VoiceTile({ + title, + subtitle, + selected, + disabled, + onPress, + onRemove, +}: { + title: string; + subtitle: string; + selected: boolean; + disabled: boolean; + onPress: () => void; + onRemove?: () => void; +}) { + return ( + + + + + {title.charAt(0)} + + + {onRemove && !disabled && ( + + ✕ + + )} + + + {title} + + + {subtitle} + + + ); +} + +function RecordButton({ + isRecording, + disabled, + onPress, +}: { + isRecording: boolean; + disabled: boolean; + onPress: () => void; +}) { + const pulse = useRef(new Animated.Value(0)).current; + + useEffect(() => { + if (!isRecording) return; + const loop = Animated.loop( + Animated.timing(pulse, { toValue: 1, duration: 1200, useNativeDriver: true }) + ); + loop.start(); + return () => { + loop.stop(); + pulse.setValue(0); + }; + }, [isRecording, pulse]); + + return ( + + {isRecording && ( + + )} + + + + + ); +} + +function LevelMeter({ level, active }: { level: number; active: boolean }) { + const lit = active ? Math.round(level * METER_BARS) : 0; + return ( + + {Array.from({ length: METER_BARS }, (_, i) => ( + + ))} + + ); +} + +function ReverbContent() { + const [selectedModel, setSelectedModel] = useState( + Platform.OS === 'android' ? 'VULKAN_INT4' : 'MLX_COREML_FP16' + ); + const [language, setLanguage] = useState('en'); + const [text, setText] = useState(SAMPLE_TEXTS.en); + const [choice, setChoice] = useState({ kind: 'preset', name: 'standard_male' }); + const [presetInfo, setPresetInfo] = useState>>( + {} + ); + const [customVoices, setCustomVoices] = useState([]); + const nextVoiceId = useRef(1); + + const [speed, setSpeed] = useState(1.0); + const [showSampler, setShowSampler] = useState(false); + const [steps, setSteps] = useState(8); + const [remaskRatio, setRemaskRatio] = useState(0.25); + const [fixedSeed, setFixedSeed] = useState(null); + const [lastSeed, setLastSeed] = useState(null); + const [trimEdges, setTrimEdges] = useState(true); + + const [isCloneOpen, setIsCloneOpen] = useState(false); + const [recording, setRecording] = useState(null); + const [transcript, setTranscript] = useState(''); + const [recordingLanguage, setRecordingLanguage] = useState('en'); + const [cloneStep, setCloneStep] = useState<'transcribing' | 'encoding' | null>(null); + + const [isSynthesizing, setIsSynthesizing] = useState(false); + const [chunkProgress, setChunkProgress] = useState(null); + const [result, setResult] = useState(null); + const [runError, setRunError] = useState(null); + + const { isPlaying, playStream, stop } = useAudioPlayer(REVERB_SAMPLE_RATE); + const recorder = useVoiceRecorder(); + + const { isReady, downloadProgress, error, resource, synthesize, synthesizeStop, encodeVoice } = + useTextToSpeech(models.textToSpeech.REVERB[selectedModel]); + const whisper = useSpeechToText(WHISPER_MODEL, { preventLoad: !isCloneOpen }); + + useEffect(() => { + AudioManager.setAudioSessionOptions({ + iosCategory: 'playAndRecord', + iosMode: 'default', + iosOptions: ['defaultToSpeaker', 'allowBluetoothA2DP'], + }); + }, []); + + useEffect(() => { + return () => { + synthesizeStop?.(); + stop(); + }; + }, [stop, synthesizeStop]); + + // The preset voices' reference transcripts and lengths, read from their JSON files. + useEffect(() => { + const voices = resource?.voices; + if (!voices) return; + let cancelled = false; + (async () => { + const info: Partial> = {}; + for (const name of REVERB_DEFAULT_VOICE_NAMES) { + try { + const json = JSON.parse(await RNBlobUtil.fs.readFile(voices[name], 'utf8')); + info[name] = { + transcript: json.transcript, + language: json.language, + duration: json.codes.length / 8 / REVERB_FRAME_RATE, + }; + } catch { + // A preset without its description still synthesizes. + } + } + if (!cancelled) setPresetInfo(info); + })(); + return () => { + cancelled = true; + }; + }, [resource]); + + const selectedCustom = + choice.kind === 'custom' ? customVoices.find((v) => v.id === choice.id) : undefined; + const reference = + choice.kind === 'preset' + ? presetInfo[choice.name] + : selectedCustom && { + transcript: selectedCustom.voice.transcript, + language: selectedCustom.voice.language, + duration: selectedCustom.duration, + }; + + const isBusy = isSynthesizing || isPlaying || recorder.isRecording || cloneStep !== null; + const recordingSeconds = recording ? recording.length / REVERB_SAMPLE_RATE : 0; + + const transcribeRecording = async (audio: Float32Array, lang: ReverbLanguage) => { + if (!whisper.transcribe) return; + setCloneStep('transcribing'); + try { + const heard = await whisper.transcribe( + resample(audio, REVERB_SAMPLE_RATE, WHISPER_SAMPLE_RATE_HZ), + { language: lang } + ); + setTranscript(heard.trim()); + } catch (err) { + setRunError(messageOf(err)); + } finally { + setCloneStep(null); + } + }; + + const finishRecording = async () => { + const audio = await recorder.stop(); + if (!audio) return; + const trimmed = trimSilence(audio, REVERB_SAMPLE_RATE); + setRecording(trimmed); + setTranscript(''); + await transcribeRecording(trimmed, recordingLanguage); + }; + + const toggleRecording = async () => { + if (recorder.isRecording) { + await finishRecording(); + return; + } + setRunError(null); + stop(); + const failure = await recorder.start(); + if (failure) setRunError(failure); + }; + + // Stops at the longest reference the encoder takes. + useEffect(() => { + if (recorder.isRecording && recorder.seconds >= MAX_RECORDING_S) finishRecording(); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [recorder.isRecording, recorder.seconds]); + + const playRecording = async () => { + if (!recording) return; + async function* once() { + yield { audio: recording!, sampleRate: REVERB_SAMPLE_RATE }; + } + await playStream(once()).catch((err) => setRunError(messageOf(err))); + }; + + const createVoice = async () => { + if (!encodeVoice || !recording || !transcript.trim()) return; + setRunError(null); + setCloneStep('encoding'); + try { + const voice = await encodeVoice(recording, transcript.trim(), recordingLanguage); + const id = nextVoiceId.current++; + setCustomVoices((voices) => [ + ...voices, + { id, name: `My voice ${id}`, voice, duration: recordingSeconds }, + ]); + setChoice({ kind: 'custom', id }); + setRecording(null); + setTranscript(''); + setIsCloneOpen(false); + } catch (err) { + setRunError(messageOf(err)); + } finally { + setCloneStep(null); + } + }; + + const removeVoice = (id: number) => { + setCustomVoices((voices) => voices.filter((v) => v.id !== id)); + if (choice.kind === 'custom' && choice.id === id) { + setChoice({ kind: 'preset', name: 'standard_male' }); + } + }; + + const handleSynthesize = async () => { + const voice = choice.kind === 'preset' ? choice.name : selectedCustom?.voice; + if (!synthesize || !voice || isBusy || !text.trim()) return; + + setRunError(null); + setResult(null); + setChunkProgress(null); + setIsSynthesizing(true); + + const seed = fixedSeed ?? Math.floor(Math.random() * 2 ** 31); + setLastSeed(seed); + let audioSeconds = 0; + let firstAudio: number | null = null; + const t0 = performance.now(); + + try { + const chunks = synthesize(text, { + voice, + language, + speed, + sampling: { steps, remaskRatio, seed }, + postprocess: trimEdges ? undefined : false, + }); + await playStream( + chunks, + (chunk) => { + audioSeconds += chunk.duration; + setChunkProgress( + `Chunk ${chunk.chunkIndex + 1}/${chunk.totalChunks} (${chunk.duration.toFixed(1)} s)` + ); + }, + () => { + firstAudio = (performance.now() - t0) / 1000; + } + ); + const elapsed = (performance.now() - t0) / 1000; + setResult( + `${audioSeconds.toFixed(1)} s of audio · first audio after ${(firstAudio ?? elapsed).toFixed(1)} s` + ); + setChunkProgress(null); + } catch (err) { + setRunError(messageOf(err)); + } finally { + setIsSynthesizing(false); + } + }; + + const handleStop = () => { + synthesizeStop?.(); + stop(); + setIsSynthesizing(false); + }; + + const selectLanguage = (lang: ReverbLanguage) => { + if (text === SAMPLE_TEXTS[language]) setText(SAMPLE_TEXTS[lang]); + setLanguage(lang); + if (!recording) setRecordingLanguage(lang); + }; + + const whisperStatus = !whisper.isReady + ? whisper.error + ? `Whisper failed to load: ${whisper.error.message}` + : `Loading Whisper to transcribe… ${Math.round(whisper.downloadProgress)}%` + : cloneStep === 'transcribing' + ? 'Transcribing with Whisper…' + : null; + + return ( + + + Reverb Text-to-Speech + + Speech in the voice of any short recording, in 7 languages, generated on-device. Pick a + preset voice or clone your own. + + + ({ + label: m.label, + value: m.value, + disabled: isBusy || m.platform !== Platform.OS, + }))} + selectedValue={selectedModel} + onValueChange={setSelectedModel} + /> + + + {runError && ( + + {runError} + + )} + + + Voice + + {PRESETS.map(({ name, title, subtitle }) => ( + setChoice({ kind: 'preset', name })} + /> + ))} + {customVoices.map((v) => ( + setChoice({ kind: 'custom', id: v.id })} + onRemove={() => removeVoice(v.id)} + /> + ))} + setIsCloneOpen((open) => !open)} + disabled={isBusy} + activeOpacity={0.8} + > + {isCloneOpen ? '–' : '+'} + {isCloneOpen ? 'Close' : 'Clone a voice'} + + + + {reference && !isCloneOpen && ( + + + Reference · {reference.language} · {reference.duration.toFixed(1)} s + + “{reference.transcript}” + + )} + + {isCloneOpen && ( + + 1 · Record a reference + + Read anything for 5–15 seconds in a quiet room. Reverb copies the voice, the pace and + the mood of the recording. + + + + + + {formatSeconds(recorder.isRecording ? recorder.seconds : recordingSeconds)} + / {formatSeconds(MAX_RECORDING_S)} + + + + {isSimulator + ? 'Recording needs a physical device.' + : recorder.isRecording + ? 'Recording… tap to stop' + : recording + ? 'Recorded' + : 'Tap to record'} + + + + {recording && !recorder.isRecording && ( + + + {isPlaying ? '■ Stop' : '▶ Listen'} + + transcribeRecording(recording, recordingLanguage)} + disabled={!whisper.isReady || isBusy} + > + ↻ Transcribe + + + )} + + 2 · Check the transcript + ({ ...l, disabled: isBusy }))} + selectedValue={recordingLanguage} + onValueChange={setRecordingLanguage} + /> + {whisperStatus && {whisperStatus}} + + + 3 · Create the voice + +