diff --git a/apps/speech/screens/TextToSpeechScreen.tsx b/apps/speech/screens/TextToSpeechScreen.tsx index ed0a3fd5df..ea99f48c1d 100644 --- a/apps/speech/screens/TextToSpeechScreen.tsx +++ b/apps/speech/screens/TextToSpeechScreen.tsx @@ -1,4 +1,4 @@ -import React, { useEffect, useRef, useState } from 'react'; +import React, { useEffect, useMemo, useRef, useState } from 'react'; import { Text, View, @@ -81,17 +81,40 @@ const capitalize = (s: string) => s.charAt(0).toUpperCase() + s.slice(1); // (`() => TextToSpeechModelConfig`); call them to get the actual configs. const kokoroVoicesByLang = models.text_to_speech.kokoro as unknown as Record< string, - Record TextToSpeechModelConfig> + Record< + string, + (opts?: { backend?: 'xnnpack' | 'coreml' }) => TextToSpeechModelConfig + > >; -const KOKORO_VOICES: ModelOption[] = Object.entries( - kokoroVoicesByLang -).flatMap(([lang, voices]) => - Object.entries(voices).map(([name, factory]) => ({ - label: `${KOKORO_LANG_LABELS[lang] ?? lang} · ${capitalize(name)}`, - value: factory(), - })) -); +type KokoroBackend = 'xnnpack' | 'coreml'; + +// Core ML only covers the standard model, so the polish and german voices stay +// on XNNPACK and their factories throw if asked for Core ML. +const kokoroVoices = ( + backend: KokoroBackend +): ModelOption[] => + Object.entries(kokoroVoicesByLang).flatMap(([lang, voices]) => + Object.entries(voices).map(([name, factory]) => { + let value: TextToSpeechModelConfig; + try { + value = factory({ backend }); + } catch { + value = factory(); + } + return { + label: `${KOKORO_LANG_LABELS[lang] ?? lang} · ${capitalize(name)}`, + value, + }; + }) + ); + +const KOKORO_VOICES = kokoroVoices('xnnpack'); + +const KOKORO_BACKENDS: ModelOption[] = [ + { label: 'XNNPACK', value: 'xnnpack' }, + { label: 'Core ML (iOS)', value: 'coreml' }, +]; type TtsModelType = 'supertonic' | 'kokoro'; @@ -147,6 +170,21 @@ export const TextToSpeechScreen = ({ onBack }: { onBack: () => void }) => { const [selectedLang, setSelectedLang] = useState('en'); const [totalSteps, setTotalSteps] = useState(8); + const [kokoroBackend, setKokoroBackend] = useState('xnnpack'); + + const kokoroVoiceOptions = useMemo( + () => kokoroVoices(kokoroBackend), + [kokoroBackend] + ); + + const handleSelectKokoroBackend = (backend: KokoroBackend) => { + if (backend === kokoroBackend) return; + const index = kokoroVoices(kokoroBackend).findIndex( + (o) => o.value === selectedSpeaker + ); + setKokoroBackend(backend); + if (index >= 0) setSelectedSpeaker(kokoroVoices(backend)[index]!.value); + }; const model = useTextToSpeech(selectedSpeaker); @@ -306,13 +344,23 @@ export const TextToSpeechScreen = ({ onBack }: { onBack: () => void }) => { setSelectedSpeaker(m)} /> + {selectedTtsModel === 'kokoro' && Platform.OS === 'ios' && ( + + )} + {selectedTtsModel === 'supertonic' && ( <> ( // don't share the `{ modelName: string }` shape of the rest of the registry, // and have no quant/backend axis. Expose them as a plain `() => Config` // accessor so the call style stays consistent (`models.text_to_speech.en_us.heart()`). -function tts(c: C): () => C { - return () => c; +function tts( + c: C +): (opts?: { backend?: 'xnnpack' | 'coreml' }) => C { + return (opts) => { + if (opts?.backend !== 'coreml') return c; + if (c.model !== KOKORO_STANDARD) { + throw new Error( + 'Core ML is only available for the standard Kokoro model; ' + + 'the polish and german variants ship on XNNPACK only.' + ); + } + return { ...c, model: KOKORO_STANDARD_COREML } as C; + }; } type TTSBackendMap = Partial>; diff --git a/packages/react-native-executorch/src/constants/tts/models.ts b/packages/react-native-executorch/src/constants/tts/models.ts index 7191d4fe58..f304c1ed81 100644 --- a/packages/react-native-executorch/src/constants/tts/models.ts +++ b/packages/react-native-executorch/src/constants/tts/models.ts @@ -3,10 +3,29 @@ import { URL_PREFIX, PREVIOUS_VERSION_TAG, VERSION_TAG } from '../versions'; // Text to speech (tts) - Kokoro model(s) const KOKORO_MODEL_ROOT = `${URL_PREFIX}-kokoro/${PREVIOUS_VERSION_TAG}/xnnpack`; +const KOKORO_COREML_MODEL_ROOT = `${URL_PREFIX}-kokoro/${VERSION_TAG}/coreml`; const KOKORO_STANDARD_MODEL_ROOT = `${KOKORO_MODEL_ROOT}/standard`; +const KOKORO_COREML_STANDARD_MODEL_ROOT = `${KOKORO_COREML_MODEL_ROOT}/standard`; const KOKORO_POLISH_MODEL_ROOT = `${KOKORO_MODEL_ROOT}/polish`; const KOKORO_GERMAN_MODEL_ROOT = `${KOKORO_MODEL_ROOT}/german`; +/** + * The standard Kokoro instance with the synthesizer running on Core ML. iOS only. + * + * The synthesizer is the expensive half of Kokoro. On an iPhone 16 this build + * produces 7.4 s of audio in 627 ms, against 4741 ms for the XNNPACK one, at the + * cost of a one-time compile on the first call that is cached across launches. + * The duration predictor stays on XNNPACK, so this config mixes backends. + * + * Use {@link KOKORO_STANDARD} on Android. + * @category Models - Text to Speech + */ +export const KOKORO_STANDARD_COREML = { + modelName: 'kokoro' as const, + durationPredictorSource: `${KOKORO_STANDARD_MODEL_ROOT}/duration_predictor_std.pte`, + synthesizerSource: `${KOKORO_COREML_STANDARD_MODEL_ROOT}/synthesizer_coreml_fp32.pte`, +}; + /** * A standard Kokoro instance which processes the text in batches of maximum 128 tokens. * Works well with built-in languages: english, spanish, french, italian, portuguese and hindi.