diff --git a/.agents/skills/add-native-extension/SKILL.md b/.agents/skills/add-native-extension/SKILL.md index bb83842b2e..63b88969e9 100644 --- a/.agents/skills/add-native-extension/SKILL.md +++ b/.agents/skills/add-native-extension/SKILL.md @@ -120,8 +120,8 @@ namespace rnexecutorch::extensions:: } // 6. Perform the computation - const float *srcData = reinterpret_cast(src->data_.get()); - float *dstData = reinterpret_cast(dst->data_.get()); + const float *srcData = reinterpret_cast(src->data_); + float *dstData = reinterpret_cast(dst->data_); size_t size = src->size(); for (size_t i = 0; i < size; ++i) diff --git a/.cspell-wordlist.txt b/.cspell-wordlist.txt index aed3ee1e7d..159ac238c4 100644 --- a/.cspell-wordlist.txt +++ b/.cspell-wordlist.txt @@ -275,3 +275,8 @@ pcre libkleidicv thresholding binarization +subcomponents +gapless +Purna +Viram +Deergh diff --git a/apps/speech/app.json b/apps/speech/app.json new file mode 100644 index 0000000000..002040822f --- /dev/null +++ b/apps/speech/app.json @@ -0,0 +1,45 @@ +{ + "expo": { + "name": "speech", + "slug": "speech", + "version": "1.0.0", + "orientation": "portrait", + "icon": "./assets/icons/icon.png", + "userInterfaceStyle": "light", + "newArchEnabled": true, + "scheme": "rne-speech", + "splash": { + "image": "./assets/icons/splash.png", + "resizeMode": "contain", + "backgroundColor": "#ffffff" + }, + "ios": { + "supportsTablet": true, + "bundleIdentifier": "com.anonymous.speech" + }, + "android": { + "adaptiveIcon": { + "foregroundImage": "./assets/icons/adaptive-icon.png", + "backgroundColor": "#ffffff" + }, + "package": "com.anonymous.speech" + }, + "web": { + "favicon": "./assets/icons/favicon.png" + }, + "plugins": [ + "expo-router", + [ + "expo-build-properties", + { + "android": { + "minSdkVersion": 26 + }, + "ios": { + "deploymentTarget": "17.0" + } + } + ] + ] + } +} diff --git a/apps/speech/app/_layout.tsx b/apps/speech/app/_layout.tsx new file mode 100644 index 0000000000..0788f41329 --- /dev/null +++ b/apps/speech/app/_layout.tsx @@ -0,0 +1,32 @@ +import { Drawer } from 'expo-router/drawer'; +import { ColorPalette } from '../theme'; +import React from 'react'; + +export default function Layout() { + return ( + + null, + title: 'Main Menu', + drawerItemStyle: { display: 'none' }, + }} + /> + + + ); +} diff --git a/apps/speech/app/index.tsx b/apps/speech/app/index.tsx new file mode 100644 index 0000000000..7aa06e823f --- /dev/null +++ b/apps/speech/app/index.tsx @@ -0,0 +1,49 @@ +import { useRouter } from 'expo-router'; +import { View, Text, StyleSheet, TouchableOpacity } from 'react-native'; +import { ColorPalette } from '../theme'; + +export default function Home() { + const router = useRouter(); + + return ( + + Select a demo model + + router.navigate('text-to-speech/')}> + Text to Speech + + + + ); +} + +const styles = StyleSheet.create({ + container: { + flex: 1, + justifyContent: 'center', + alignItems: 'center', + backgroundColor: '#fff', + }, + headerText: { + fontSize: 18, + color: ColorPalette.strongPrimary, + margin: 20, + }, + buttonContainer: { + width: '80%', + justifyContent: 'space-evenly', + marginBottom: 20, + }, + button: { + backgroundColor: ColorPalette.strongPrimary, + borderRadius: 8, + padding: 14, + alignItems: 'center', + marginBottom: 12, + }, + buttonText: { + color: 'white', + fontSize: 16, + fontWeight: '600', + }, +}); diff --git a/apps/speech/app/text-to-speech/index.tsx b/apps/speech/app/text-to-speech/index.tsx new file mode 100644 index 0000000000..ae67f56abe --- /dev/null +++ b/apps/speech/app/text-to-speech/index.tsx @@ -0,0 +1,212 @@ +import React, { useState, useCallback, useEffect, useRef } from 'react'; +import { View, Text, TextInput, StyleSheet, ScrollView, Keyboard } from 'react-native'; +import { commonStyles, theme } from '../../theme'; +import { useTextToSpeech, models } from 'react-native-executorch'; +import ScreenWrapper from '../../components/ScreenWrapper'; +import { VoicePicker, type VoiceOption } from '../../components/VoicePicker'; +import { ModelStatus } from '../../components/ModelStatus'; +import { LatencyIndicator } from '../../components/LatencyIndicator'; +import { Button } from '../../components/Button'; +import type { TextToSpeechModel } from 'react-native-executorch'; +import { AudioManager, AudioContext, AudioBuffer } from 'react-native-audio-api'; + +const VOICE_OPTIONS: VoiceOption[] = [ + { name: 'AF Heart', lang: 'en-us', value: models.textToSpeech.KOKORO.AF_HEART }, + { name: 'AF River', lang: 'en-us', value: models.textToSpeech.KOKORO.AF_RIVER }, + { name: 'AF Sarah', lang: 'en-us', value: models.textToSpeech.KOKORO.AF_SARAH }, + { name: 'AM Adam', lang: 'en-us', value: models.textToSpeech.KOKORO.AM_ADAM }, + { name: 'AM Michael', lang: 'en-us', value: models.textToSpeech.KOKORO.AM_MICHAEL }, + { name: 'BF Emma', lang: 'en-gb', value: models.textToSpeech.KOKORO.BF_EMMA }, + { name: 'BM Daniel', lang: 'en-gb', value: models.textToSpeech.KOKORO.BM_DANIEL }, + { name: 'FF Siwis', lang: 'fr', value: models.textToSpeech.KOKORO.FF_SIWIS }, + { name: 'EF Dora', lang: 'es', value: models.textToSpeech.KOKORO.EF_DORA }, + { name: 'EM Alex', lang: 'es', value: models.textToSpeech.KOKORO.EM_ALEX }, + { name: 'IF Sara', lang: 'it', value: models.textToSpeech.KOKORO.IF_SARA }, + { name: 'IM Nicola', lang: 'it', value: models.textToSpeech.KOKORO.IM_NICOLA }, + { name: 'PF Dora', lang: 'pt', value: models.textToSpeech.KOKORO.PF_DORA }, + { name: 'PM Santa', lang: 'pt', value: models.textToSpeech.KOKORO.PM_SANTA }, + { name: 'HF Alpha', lang: 'hi', value: models.textToSpeech.KOKORO.HF_ALPHA }, + { name: 'HM Omega', lang: 'hi', value: models.textToSpeech.KOKORO.HM_OMEGA }, + { name: 'PM Mateusz', lang: 'pl', value: models.textToSpeech.KOKORO.PM_MATEUSZ }, + { name: 'DF Anna', lang: 'de', value: models.textToSpeech.KOKORO.DF_ANNA }, +]; + +const createAudioBufferFromVector = ( + audioVector: Float32Array, + audioContext: AudioContext | null = null, + sampleRate: number = 24000 +): AudioBuffer => { + if (audioContext == null) audioContext = new AudioContext({ sampleRate }); + + const audioBuffer = audioContext.createBuffer(1, audioVector.length, sampleRate); + const channelData = audioBuffer.getChannelData(0); + channelData.set(audioVector); + + return audioBuffer; +}; + +function TextToSpeechContent() { + const [selectedVoice, setSelectedVoice] = useState(VOICE_OPTIONS[0].value); + const [text, setText] = useState(''); + const [isPlaying, setIsPlaying] = useState(false); + const [latency, setLatency] = useState(null); + const [error, setError] = useState(null); + + const { isReady, downloadProgress, error: loadError, stream } = useTextToSpeech(selectedVoice); + + const audioContextRef = useRef(null); + const gainNodeRef = useRef(null); + + useEffect(() => { + AudioManager.setAudioSessionOptions({ + iosCategory: 'playAndRecord', + iosMode: 'spokenAudio', + iosOptions: ['defaultToSpeaker'], + }); + + const context = new AudioContext({ sampleRate: 24000 }); + audioContextRef.current = context; + context.suspend(); + + const gainNode = context.createGain(); + gainNode.gain.value = 2.0; + gainNode.connect(context.destination); + gainNodeRef.current = gainNode; + + return () => { + audioContextRef.current?.close(); + audioContextRef.current = null; + gainNodeRef.current = null; + }; + }, []); + + const handleSpeak = useCallback(async () => { + if (!stream || !text.trim()) return; + + Keyboard.dismiss(); + setIsPlaying(true); + setError(null); + const start = Date.now(); + + const audioContext = audioContextRef.current; + if (!audioContext) return; + + if (audioContext.state === 'suspended') { + await audioContext.resume(); + } + + try { + await stream({ + text: text.trim(), + speed: 0.9, + onBegin: () => { + setLatency(null); + }, + // Resolve once this chunk has finished playing so the next one is + // delivered only afterwards — giving gapless, sequential playback. + onNext: (audioVec: Float32Array) => + new Promise((resolve) => { + const audioBuffer = createAudioBufferFromVector(audioVec, audioContext, 24000); + + const source = audioContext.createBufferSource(); + source.buffer = audioBuffer; + + if (gainNodeRef.current) { + source.connect(gainNodeRef.current); + } else { + source.connect(audioContext.destination); + } + + source.onEnded = () => resolve(); + source.start(); + }), + onEnd: () => { + setLatency(Date.now() - start); + setIsPlaying(false); + audioContext.suspend(); + }, + }); + } catch (e: any) { + setError(e.message || String(e)); + setIsPlaying(false); + } + }, [stream, text]); + + const activeError = loadError ? String(loadError) : error; + + return ( + + + Enter text and select a voice to generate speech. + + + { + setSelectedVoice(model); + setError(null); + setLatency(null); + }} + /> + + + + + + +