From 424530c32133fbd0393637d783319d76e1b9949a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20S=C5=82uszniak?= Date: Thu, 2 Jul 2026 17:39:10 +0200 Subject: [PATCH 1/5] [RNE Rewrite] feat: add voice activity detection pipeline (#1249) Port the VAD feature to the rewrite as a pure-TypeScript pipeline on top of the core model.execute primitive (no new C++): - src/extensions/speech/tasks/vad.ts: createVAD runner replicating the native FSMN-VAD algorithm (framing + Hann window + pre-emphasis, chunked inference, thresholding / min-duration / padding / merge). Segments are returned in seconds. Relies on the get_dynamic_dims relaxed input validation for the dynamic frame dimension; the fsmn-vad model is re-exported with it. - src/extensions/speech/vadStreamer.ts: pure streaming state machine driving onSpeechBegin / onSpeechEnd over an accumulating buffer. - src/hooks/useVAD.ts: hook wrapping createVAD + streamer lifecycle. - Register models.vad.FSMN_VAD and export the speech extension. - apps/speech: expo-router demo (mirrors apps/nlp) with a real-time mic VAD screen via react-native-audio-api. --- .cspell-wordlist.txt | 4 + apps/speech/app.json | 51 +++ apps/speech/app/_layout.tsx | 32 ++ apps/speech/app/index.tsx | 51 +++ apps/speech/app/vad/index.tsx | 227 +++++++++++++ apps/speech/assets/icons/adaptive-icon.png | Bin 0 -> 17547 bytes apps/speech/assets/icons/executorch.svg | 9 + apps/speech/assets/icons/favicon.png | Bin 0 -> 1466 bytes apps/speech/assets/icons/icon.png | Bin 0 -> 22380 bytes apps/speech/assets/icons/splash.png | Bin 0 -> 47346 bytes apps/speech/babel.config.js | 7 + apps/speech/components/Button.tsx | 102 ++++++ apps/speech/components/ModelStatus.tsx | 69 ++++ apps/speech/components/ScreenWrapper.tsx | 8 + apps/speech/declarations.d.ts | 5 + apps/speech/index.ts | 8 + apps/speech/metro.config.js | 21 ++ apps/speech/package.json | 49 +++ apps/speech/theme.ts | 76 +++++ apps/speech/tsconfig.json | 15 + .../src/extensions/speech/index.ts | 2 + .../src/extensions/speech/tasks/vad.ts | 303 ++++++++++++++++++ .../src/extensions/speech/vadStreamer.ts | 131 ++++++++ .../src/hooks/useVAD.ts | 94 ++++++ packages/react-native-executorch/src/index.ts | 4 + .../react-native-executorch/src/models.ts | 14 + yarn.lock | 59 ++++ 27 files changed, 1341 insertions(+) create mode 100644 apps/speech/app.json create mode 100644 apps/speech/app/_layout.tsx create mode 100644 apps/speech/app/index.tsx create mode 100644 apps/speech/app/vad/index.tsx create mode 100644 apps/speech/assets/icons/adaptive-icon.png create mode 100644 apps/speech/assets/icons/executorch.svg create mode 100644 apps/speech/assets/icons/favicon.png create mode 100644 apps/speech/assets/icons/icon.png create mode 100644 apps/speech/assets/icons/splash.png create mode 100644 apps/speech/babel.config.js create mode 100644 apps/speech/components/Button.tsx create mode 100644 apps/speech/components/ModelStatus.tsx create mode 100644 apps/speech/components/ScreenWrapper.tsx create mode 100644 apps/speech/declarations.d.ts create mode 100644 apps/speech/index.ts create mode 100644 apps/speech/metro.config.js create mode 100644 apps/speech/package.json create mode 100644 apps/speech/theme.ts create mode 100644 apps/speech/tsconfig.json create mode 100644 packages/react-native-executorch/src/extensions/speech/index.ts create mode 100644 packages/react-native-executorch/src/extensions/speech/tasks/vad.ts create mode 100644 packages/react-native-executorch/src/extensions/speech/vadStreamer.ts create mode 100644 packages/react-native-executorch/src/hooks/useVAD.ts diff --git a/.cspell-wordlist.txt b/.cspell-wordlist.txt index 61b9142ad5..18106d44c7 100644 --- a/.cspell-wordlist.txt +++ b/.cspell-wordlist.txt @@ -278,3 +278,7 @@ binarization bugprone NOLINTNEXTLINE nullopt + +hann +preemphasis +coeff diff --git a/apps/speech/app.json b/apps/speech/app.json new file mode 100644 index 0000000000..c53f558446 --- /dev/null +++ b/apps/speech/app.json @@ -0,0 +1,51 @@ +{ + "expo": { + "name": "speech", + "slug": "speech", + "version": "1.0.0", + "orientation": "portrait", + "icon": "./assets/icons/icon.png", + "userInterfaceStyle": "light", + "newArchEnabled": true, + "scheme": "rne-speech", + "splash": { + "image": "./assets/icons/splash.png", + "resizeMode": "contain", + "backgroundColor": "#ffffff" + }, + "ios": { + "supportsTablet": true, + "bundleIdentifier": "com.anonymous.speech", + "infoPlist": { + "NSMicrophoneUsageDescription": "This app uses the microphone to detect voice activity." + } + }, + "android": { + "adaptiveIcon": { + "foregroundImage": "./assets/icons/adaptive-icon.png", + "backgroundColor": "#ffffff" + }, + "package": "com.anonymous.speech", + "permissions": [ + "android.permission.RECORD_AUDIO" + ] + }, + "web": { + "favicon": "./assets/icons/favicon.png" + }, + "plugins": [ + "expo-router", + [ + "expo-build-properties", + { + "android": { + "minSdkVersion": 26 + }, + "ios": { + "deploymentTarget": "17.0" + } + } + ] + ] + } +} diff --git a/apps/speech/app/_layout.tsx b/apps/speech/app/_layout.tsx new file mode 100644 index 0000000000..bd4a75cc71 --- /dev/null +++ b/apps/speech/app/_layout.tsx @@ -0,0 +1,32 @@ +import { Drawer } from 'expo-router/drawer'; +import { ColorPalette } from '../theme'; +import React from 'react'; + +export default function Layout() { + return ( + + null, + title: 'Main Menu', + drawerItemStyle: { display: 'none' }, + }} + /> + + + ); +} diff --git a/apps/speech/app/index.tsx b/apps/speech/app/index.tsx new file mode 100644 index 0000000000..453b8e3863 --- /dev/null +++ b/apps/speech/app/index.tsx @@ -0,0 +1,51 @@ +import { useRouter } from 'expo-router'; +import { View, Text, StyleSheet, TouchableOpacity } from 'react-native'; +import { ColorPalette } from '../theme'; +import ExecutorchLogo from '../assets/icons/executorch.svg'; + +export default function Home() { + const router = useRouter(); + + return ( + + + Select a demo + + router.navigate('vad/')}> + Voice Activity Detection + + + + ); +} + +const styles = StyleSheet.create({ + container: { + flex: 1, + justifyContent: 'center', + alignItems: 'center', + backgroundColor: '#fff', + }, + headerText: { + fontSize: 18, + color: ColorPalette.strongPrimary, + margin: 20, + }, + buttonContainer: { + width: '80%', + justifyContent: 'space-evenly', + marginBottom: 20, + }, + button: { + backgroundColor: ColorPalette.strongPrimary, + borderRadius: 8, + padding: 14, + alignItems: 'center', + marginBottom: 12, + }, + buttonText: { + color: 'white', + fontSize: 16, + fontWeight: '600', + }, +}); diff --git a/apps/speech/app/vad/index.tsx b/apps/speech/app/vad/index.tsx new file mode 100644 index 0000000000..4cbf2eaebe --- /dev/null +++ b/apps/speech/app/vad/index.tsx @@ -0,0 +1,227 @@ +import React, { useEffect, useRef, useState } from 'react'; +import { View, Text, StyleSheet, ScrollView, Platform } from 'react-native'; +import { useVAD, models } from 'react-native-executorch'; +import { AudioManager, AudioRecorder } from 'react-native-audio-api'; +import DeviceInfo from 'react-native-device-info'; + +import ScreenWrapper from '../../components/ScreenWrapper'; +import { ModelStatus } from '../../components/ModelStatus'; +import { Button } from '../../components/Button'; +import { theme } from '../../theme'; + +const SAMPLE_RATE = 16000; +const isSimulator = DeviceInfo.isEmulatorSync(); + +function VADContent() { + const { isReady, downloadProgress, error, stream, streamInsert, streamStop } = useVAD( + models.vad.FSMN_VAD + ); + + const [isStreaming, setIsStreaming] = useState(false); + const [isSpeaking, setIsSpeaking] = useState(false); + const [hasMicPermission, setHasMicPermission] = useState(false); + const [runError, setRunError] = useState(null); + const [logs, setLogs] = useState([]); + + const recorder = useRef(new AudioRecorder()); + const logScrollRef = useRef(null); + + const addLog = (message: string) => { + setLogs((prev) => [...prev, `${new Date().toLocaleTimeString()}: ${message}`]); + }; + + useEffect(() => { + AudioManager.setAudioSessionOptions({ + iosCategory: 'playAndRecord', + iosMode: 'spokenAudio', + iosOptions: ['allowBluetoothHFP', 'defaultToSpeaker'], + }); + AudioManager.requestRecordingPermissions().then((status) => + setHasMicPermission(status === 'Granted') + ); + }, []); + + const handleStart = async () => { + if (isStreaming || !isReady) return; + + if (!hasMicPermission) { + setRunError('Microphone permission denied. Please enable it in Settings.'); + return; + } + + setRunError(null); + setLogs([]); + setIsStreaming(true); + addLog('Starting VAD stream…'); + + recorder.current.onAudioReady( + { sampleRate: SAMPLE_RATE, bufferLength: 1600, channelCount: 1 }, + ({ buffer }) => streamInsert(buffer.getChannelData(0)) + ); + + try { + await AudioManager.setAudioSessionActivity(true); + const started = await recorder.current.start(); + if (started.status === 'error') { + throw new Error(started.message); + } + + await stream({ + onSpeechBegin: () => { + setIsSpeaking(true); + addLog('Speech detected (begin)'); + }, + onSpeechEnd: () => { + setIsSpeaking(false); + addLog('Silence detected (end)'); + }, + options: { timeout: 100, detectionMargin: 300 }, + }); + } catch (e) { + setRunError(e instanceof Error ? e.message : String(e)); + setIsStreaming(false); + } + }; + + const handleStop = async () => { + await recorder.current.stop(); + streamStop(); + setIsStreaming(false); + setIsSpeaking(false); + addLog('VAD stream stopped'); + }; + + const streamDisabled = isSimulator || !isReady; + + return ( + + + Voice Activity Detection + + Streams microphone audio through the FSMN-VAD model and reports when speech begins and + ends in real time. + + + + + {runError && ( + + {runError} + + )} + + + + + + {isSpeaking ? 'SPEAKING' : 'SILENT'} + + + + {isStreaming ? ( +