diff --git a/.cspell-wordlist.txt b/.cspell-wordlist.txt
index 61b9142ad5..6bc184a669 100644
--- a/.cspell-wordlist.txt
+++ b/.cspell-wordlist.txt
@@ -278,3 +278,8 @@ binarization
bugprone
NOLINTNEXTLINE
nullopt
+
+hann
+preemphasis
+coeff
+Silero
diff --git a/apps/speech/app.json b/apps/speech/app.json
new file mode 100644
index 0000000000..c53f558446
--- /dev/null
+++ b/apps/speech/app.json
@@ -0,0 +1,51 @@
+{
+ "expo": {
+ "name": "speech",
+ "slug": "speech",
+ "version": "1.0.0",
+ "orientation": "portrait",
+ "icon": "./assets/icons/icon.png",
+ "userInterfaceStyle": "light",
+ "newArchEnabled": true,
+ "scheme": "rne-speech",
+ "splash": {
+ "image": "./assets/icons/splash.png",
+ "resizeMode": "contain",
+ "backgroundColor": "#ffffff"
+ },
+ "ios": {
+ "supportsTablet": true,
+ "bundleIdentifier": "com.anonymous.speech",
+ "infoPlist": {
+ "NSMicrophoneUsageDescription": "This app uses the microphone to detect voice activity."
+ }
+ },
+ "android": {
+ "adaptiveIcon": {
+ "foregroundImage": "./assets/icons/adaptive-icon.png",
+ "backgroundColor": "#ffffff"
+ },
+ "package": "com.anonymous.speech",
+ "permissions": [
+ "android.permission.RECORD_AUDIO"
+ ]
+ },
+ "web": {
+ "favicon": "./assets/icons/favicon.png"
+ },
+ "plugins": [
+ "expo-router",
+ [
+ "expo-build-properties",
+ {
+ "android": {
+ "minSdkVersion": 26
+ },
+ "ios": {
+ "deploymentTarget": "17.0"
+ }
+ }
+ ]
+ ]
+ }
+}
diff --git a/apps/speech/app/_layout.tsx b/apps/speech/app/_layout.tsx
new file mode 100644
index 0000000000..bd4a75cc71
--- /dev/null
+++ b/apps/speech/app/_layout.tsx
@@ -0,0 +1,32 @@
+import { Drawer } from 'expo-router/drawer';
+import { ColorPalette } from '../theme';
+import React from 'react';
+
+export default function Layout() {
+ return (
+
+ null,
+ title: 'Main Menu',
+ drawerItemStyle: { display: 'none' },
+ }}
+ />
+
+
+ );
+}
diff --git a/apps/speech/app/index.tsx b/apps/speech/app/index.tsx
new file mode 100644
index 0000000000..453b8e3863
--- /dev/null
+++ b/apps/speech/app/index.tsx
@@ -0,0 +1,51 @@
+import { useRouter } from 'expo-router';
+import { View, Text, StyleSheet, TouchableOpacity } from 'react-native';
+import { ColorPalette } from '../theme';
+import ExecutorchLogo from '../assets/icons/executorch.svg';
+
+export default function Home() {
+ const router = useRouter();
+
+ return (
+
+
+ Select a demo
+
+ router.navigate('vad/')}>
+ Voice Activity Detection
+
+
+
+ );
+}
+
+const styles = StyleSheet.create({
+ container: {
+ flex: 1,
+ justifyContent: 'center',
+ alignItems: 'center',
+ backgroundColor: '#fff',
+ },
+ headerText: {
+ fontSize: 18,
+ color: ColorPalette.strongPrimary,
+ margin: 20,
+ },
+ buttonContainer: {
+ width: '80%',
+ justifyContent: 'space-evenly',
+ marginBottom: 20,
+ },
+ button: {
+ backgroundColor: ColorPalette.strongPrimary,
+ borderRadius: 8,
+ padding: 14,
+ alignItems: 'center',
+ marginBottom: 12,
+ },
+ buttonText: {
+ color: 'white',
+ fontSize: 16,
+ fontWeight: '600',
+ },
+});
diff --git a/apps/speech/app/vad/index.tsx b/apps/speech/app/vad/index.tsx
new file mode 100644
index 0000000000..734a681b85
--- /dev/null
+++ b/apps/speech/app/vad/index.tsx
@@ -0,0 +1,228 @@
+import React, { useEffect, useRef, useState } from 'react';
+import { View, Text, StyleSheet, ScrollView, Platform } from 'react-native';
+import { useVAD, models } from 'react-native-executorch';
+import { AudioManager, AudioRecorder } from 'react-native-audio-api';
+import DeviceInfo from 'react-native-device-info';
+
+import ScreenWrapper from '../../components/ScreenWrapper';
+import { ModelStatus } from '../../components/ModelStatus';
+import { Button } from '../../components/Button';
+import { theme } from '../../theme';
+
+// Record at the model's expected sample rate rather than hardcoding it.
+const SAMPLE_RATE = models.vad.FSMN_VAD.featureConfig.sampleRate;
+const isSimulator = DeviceInfo.isEmulatorSync();
+
+function VADContent() {
+ const { isReady, downloadProgress, error, stream, streamInsert, streamStop } = useVAD(
+ models.vad.FSMN_VAD
+ );
+
+ const [isStreaming, setIsStreaming] = useState(false);
+ const [isSpeaking, setIsSpeaking] = useState(false);
+ const [hasMicPermission, setHasMicPermission] = useState(false);
+ const [runError, setRunError] = useState(null);
+ const [logs, setLogs] = useState([]);
+
+ const recorder = useRef(new AudioRecorder());
+ const logScrollRef = useRef(null);
+
+ const addLog = (message: string) => {
+ setLogs((prev) => [...prev, `${new Date().toLocaleTimeString()}: ${message}`]);
+ };
+
+ useEffect(() => {
+ AudioManager.setAudioSessionOptions({
+ iosCategory: 'playAndRecord',
+ iosMode: 'spokenAudio',
+ iosOptions: ['allowBluetoothHFP', 'defaultToSpeaker'],
+ });
+ AudioManager.requestRecordingPermissions().then((status) =>
+ setHasMicPermission(status === 'Granted')
+ );
+ }, []);
+
+ const handleStart = async () => {
+ if (isStreaming || !isReady) return;
+
+ if (!hasMicPermission) {
+ setRunError('Microphone permission denied. Please enable it in Settings.');
+ return;
+ }
+
+ setRunError(null);
+ setLogs([]);
+ setIsStreaming(true);
+ addLog('Starting VAD stream…');
+
+ recorder.current.onAudioReady(
+ { sampleRate: SAMPLE_RATE, bufferLength: 1600, channelCount: 1 },
+ ({ buffer }) => streamInsert(buffer.getChannelData(0))
+ );
+
+ try {
+ await AudioManager.setAudioSessionActivity(true);
+ const started = await recorder.current.start();
+ if (started.status === 'error') {
+ throw new Error(started.message);
+ }
+
+ await stream({
+ onSpeechBegin: () => {
+ setIsSpeaking(true);
+ addLog('Speech detected (begin)');
+ },
+ onSpeechEnd: () => {
+ setIsSpeaking(false);
+ addLog('Silence detected (end)');
+ },
+ options: { timeout: 100, detectionMargin: 300 },
+ });
+ } catch (e) {
+ setRunError(e instanceof Error ? e.message : String(e));
+ setIsStreaming(false);
+ }
+ };
+
+ const handleStop = async () => {
+ await recorder.current.stop();
+ streamStop();
+ setIsStreaming(false);
+ setIsSpeaking(false);
+ addLog('VAD stream stopped');
+ };
+
+ const streamDisabled = isSimulator || !isReady;
+
+ return (
+
+
+ Voice Activity Detection
+
+ Streams microphone audio through the FSMN-VAD model and reports when speech begins and
+ ends in real time.
+
+
+
+
+ {runError && (
+
+ {runError}
+
+ )}
+
+
+
+
+
+ {isSpeaking ? 'SPEAKING' : 'SILENT'}
+
+
+
+ {isStreaming ? (
+
+ ) : (
+
+ )}
+
+
+
+ VAD events
+ logScrollRef.current?.scrollToEnd({ animated: true })}
+ >
+ {logs.length > 0 ? (
+ logs.map((log, i) => (
+
+ {log}
+
+ ))
+ ) : (
+ No events logged yet…
+ )}
+
+
+
+ );
+}
+
+export default function VADScreen() {
+ return (
+
+
+
+ );
+}
+
+const styles = StyleSheet.create({
+ container: { flex: 1, backgroundColor: theme.colors.background },
+ content: { padding: theme.spacing.large, paddingBottom: 40 },
+ card: {
+ backgroundColor: theme.colors.cardBackground,
+ borderRadius: theme.radius.large,
+ padding: 20,
+ marginBottom: 20,
+ borderWidth: 1,
+ borderColor: theme.colors.lightBorder,
+ },
+ cardTitle: {
+ fontSize: theme.typography.title.fontSize,
+ fontWeight: theme.typography.title.fontWeight,
+ color: theme.colors.strongPrimary,
+ marginBottom: 8,
+ },
+ cardDescription: {
+ fontSize: 14,
+ color: theme.colors.textMuted,
+ lineHeight: 20,
+ marginBottom: 16,
+ },
+ visualizer: { alignItems: 'center', marginBottom: 20 },
+ indicator: {
+ width: 96,
+ height: 96,
+ borderRadius: 48,
+ marginBottom: 16,
+ },
+ speaking: { backgroundColor: '#22c55e' },
+ silent: { backgroundColor: '#e9ecef' },
+ visualizerText: { fontSize: 22, fontWeight: '800', letterSpacing: 2 },
+ speakingText: { color: '#22c55e' },
+ silentText: { color: theme.colors.textPlaceholder },
+ sectionTitle: { fontSize: 16, fontWeight: '700', color: '#212529', marginBottom: 10 },
+ logScroll: {
+ maxHeight: 180,
+ backgroundColor: '#f8fafc',
+ borderRadius: theme.radius.small,
+ borderWidth: 1,
+ borderColor: theme.colors.lightBorder,
+ padding: 12,
+ },
+ logText: {
+ fontSize: 12,
+ fontFamily: Platform.OS === 'ios' ? 'Menlo' : 'monospace',
+ color: '#334155',
+ marginBottom: 2,
+ },
+ emptyText: { color: theme.colors.textPlaceholder, fontStyle: 'italic' },
+ errorContainer: {
+ backgroundColor: theme.colors.errorBackground,
+ padding: 12,
+ borderRadius: theme.radius.small,
+ marginBottom: 20,
+ },
+ errorText: { color: theme.colors.errorText, fontSize: 14, textAlign: 'center' },
+});
diff --git a/apps/speech/assets/icons/adaptive-icon.png b/apps/speech/assets/icons/adaptive-icon.png
new file mode 100644
index 0000000000..03d6f6b6c6
Binary files /dev/null and b/apps/speech/assets/icons/adaptive-icon.png differ
diff --git a/apps/speech/assets/icons/executorch.svg b/apps/speech/assets/icons/executorch.svg
new file mode 100644
index 0000000000..e548ea4201
--- /dev/null
+++ b/apps/speech/assets/icons/executorch.svg
@@ -0,0 +1,9 @@
+
diff --git a/apps/speech/assets/icons/favicon.png b/apps/speech/assets/icons/favicon.png
new file mode 100644
index 0000000000..e75f697b18
Binary files /dev/null and b/apps/speech/assets/icons/favicon.png differ
diff --git a/apps/speech/assets/icons/icon.png b/apps/speech/assets/icons/icon.png
new file mode 100644
index 0000000000..a0b1526fc7
Binary files /dev/null and b/apps/speech/assets/icons/icon.png differ
diff --git a/apps/speech/assets/icons/splash.png b/apps/speech/assets/icons/splash.png
new file mode 100644
index 0000000000..0e89705a94
Binary files /dev/null and b/apps/speech/assets/icons/splash.png differ
diff --git a/apps/speech/babel.config.js b/apps/speech/babel.config.js
new file mode 100644
index 0000000000..6b2006979c
--- /dev/null
+++ b/apps/speech/babel.config.js
@@ -0,0 +1,7 @@
+module.exports = function (api) {
+ api.cache(true);
+ return {
+ presets: ['babel-preset-expo'],
+ plugins: ['react-native-worklets/plugin'],
+ };
+};
diff --git a/apps/speech/components/Button.tsx b/apps/speech/components/Button.tsx
new file mode 100644
index 0000000000..6c4c5d8081
--- /dev/null
+++ b/apps/speech/components/Button.tsx
@@ -0,0 +1,102 @@
+import React from 'react';
+import {
+ TouchableOpacity,
+ Text,
+ ActivityIndicator,
+ StyleSheet,
+ type StyleProp,
+ type ViewStyle,
+} from 'react-native';
+import { theme } from '../theme';
+
+export interface ButtonProps {
+ title: string;
+ onPress: () => void;
+ variant?: 'primary' | 'secondary' | 'accent';
+ disabled?: boolean;
+ loading?: boolean;
+ style?: StyleProp;
+}
+
+export function Button({
+ title,
+ onPress,
+ variant = 'primary',
+ disabled = false,
+ loading = false,
+ style,
+}: ButtonProps) {
+ const buttonStyles = [
+ styles.button,
+ variant === 'primary' && styles.primary,
+ variant === 'secondary' && styles.secondary,
+ variant === 'accent' && styles.accent,
+ disabled && styles.disabled,
+ style,
+ ];
+
+ const textStyles = [
+ styles.text,
+ variant === 'primary' && styles.textPrimary,
+ variant === 'secondary' && styles.textSecondary,
+ variant === 'accent' && styles.textPrimary,
+ disabled && styles.textDisabled,
+ ];
+
+ return (
+
+ {loading ? (
+
+ ) : (
+ {title}
+ )}
+
+ );
+}
+
+const styles = StyleSheet.create({
+ button: {
+ flex: 1,
+ paddingVertical: 14,
+ borderRadius: theme.radius.medium,
+ alignItems: 'center',
+ justifyContent: 'center',
+ },
+ primary: {
+ backgroundColor: theme.colors.strongPrimary,
+ },
+ secondary: {
+ backgroundColor: theme.colors.secondary,
+ borderColor: theme.colors.strongPrimary,
+ borderWidth: 1.5,
+ },
+ accent: {
+ backgroundColor: theme.colors.accent,
+ },
+ disabled: {
+ backgroundColor: '#aaa',
+ borderColor: '#aaa',
+ opacity: 0.6,
+ },
+ text: {
+ fontSize: 15,
+ fontWeight: '600',
+ },
+ textPrimary: {
+ color: theme.colors.textPrimary,
+ },
+ textSecondary: {
+ color: theme.colors.textSecondary,
+ },
+ textDisabled: {
+ color: '#666',
+ },
+});
diff --git a/apps/speech/components/ModelStatus.tsx b/apps/speech/components/ModelStatus.tsx
new file mode 100644
index 0000000000..0234ac53f8
--- /dev/null
+++ b/apps/speech/components/ModelStatus.tsx
@@ -0,0 +1,69 @@
+import React from 'react';
+import { View, Text, ActivityIndicator, StyleSheet } from 'react-native';
+import { theme } from '../theme';
+
+export interface ModelStatusProps {
+ isReady: boolean;
+ downloadProgress?: number | null;
+ error?: string | null;
+ modelTypeLabel?: string;
+}
+
+export function ModelStatus({
+ isReady,
+ downloadProgress,
+ error,
+ modelTypeLabel = 'model',
+}: ModelStatusProps) {
+ if (error) {
+ return (
+
+ {error}
+
+ );
+ }
+
+ if (!isReady) {
+ return (
+
+
+
+ Downloading {modelTypeLabel}...{' '}
+ {downloadProgress ? `${Math.round(downloadProgress)}%` : '0%'}
+
+
+ );
+ }
+
+ return null;
+}
+
+const styles = StyleSheet.create({
+ statusBox: {
+ flexDirection: 'row',
+ alignItems: 'center',
+ backgroundColor: '#ffe8d6',
+ paddingHorizontal: 16,
+ paddingVertical: 10,
+ borderRadius: theme.radius.small,
+ marginBottom: 16,
+ width: '100%',
+ },
+ statusIndicator: {
+ marginRight: 8,
+ },
+ statusText: { fontSize: 13, color: '#a0522d', fontWeight: '500' },
+ errorContainer: {
+ backgroundColor: theme.colors.errorBackground,
+ padding: 12,
+ borderRadius: theme.radius.small,
+ marginVertical: 8,
+ alignSelf: 'stretch',
+ marginBottom: 16,
+ },
+ errorText: {
+ color: theme.colors.errorText,
+ fontSize: 14,
+ textAlign: 'center',
+ },
+});
diff --git a/apps/speech/components/ScreenWrapper.tsx b/apps/speech/components/ScreenWrapper.tsx
new file mode 100644
index 0000000000..31f70e4442
--- /dev/null
+++ b/apps/speech/components/ScreenWrapper.tsx
@@ -0,0 +1,8 @@
+import { useIsFocused } from 'expo-router';
+import { PropsWithChildren } from 'react';
+
+export default function ScreenWrapper({ children }: PropsWithChildren) {
+ const isFocused = useIsFocused();
+
+ return isFocused ? <>{children} : null;
+}
diff --git a/apps/speech/declarations.d.ts b/apps/speech/declarations.d.ts
new file mode 100644
index 0000000000..85e178f497
--- /dev/null
+++ b/apps/speech/declarations.d.ts
@@ -0,0 +1,5 @@
+declare module '*.svg' {
+ import { SvgProps } from 'react-native-svg';
+ const content: React.FV;
+ export default content;
+}
diff --git a/apps/speech/index.ts b/apps/speech/index.ts
new file mode 100644
index 0000000000..3f443dcf95
--- /dev/null
+++ b/apps/speech/index.ts
@@ -0,0 +1,8 @@
+import { registerRootComponent } from 'expo';
+
+import App from './app';
+
+// registerRootComponent calls AppRegistry.registerComponent('main', () => App);
+// It also ensures that whether you load the app in Expo Go or in a native build,
+// the environment is set up appropriately
+registerRootComponent(App);
diff --git a/apps/speech/metro.config.js b/apps/speech/metro.config.js
new file mode 100644
index 0000000000..f8ab2ab96d
--- /dev/null
+++ b/apps/speech/metro.config.js
@@ -0,0 +1,21 @@
+// Learn more https://docs.expo.io/guides/customizing-metro
+const { getDefaultConfig } = require('expo/metro-config');
+
+/** @type {import('expo/metro-config').MetroConfig} */
+const config = getDefaultConfig(__dirname);
+
+const { transformer, resolver } = config;
+
+config.transformer = {
+ ...transformer,
+ babelTransformerPath: require.resolve('react-native-svg-transformer/expo'),
+};
+config.resolver = {
+ ...resolver,
+ assetExts: resolver.assetExts.filter((ext) => ext !== 'svg'),
+ sourceExts: [...resolver.sourceExts, 'svg'],
+};
+
+config.resolver.assetExts.push('pte');
+
+module.exports = config;
diff --git a/apps/speech/package.json b/apps/speech/package.json
new file mode 100644
index 0000000000..9bbe0197d2
--- /dev/null
+++ b/apps/speech/package.json
@@ -0,0 +1,49 @@
+{
+ "name": "speech",
+ "version": "1.0.0",
+ "main": "expo-router/entry",
+ "react-native-executorch": {
+ "features": [
+ "vad"
+ ]
+ },
+ "scripts": {
+ "start": "expo start",
+ "android": "expo run:android",
+ "ios": "expo run:ios",
+ "web": "expo start --web",
+ "typecheck": "tsc",
+ "lint": "eslint . --ext .ts,.tsx --fix",
+ "postinstall": "yarn run -T patch-package --patch-dir ../../patches"
+ },
+ "dependencies": {
+ "@react-navigation/drawer": "^7.9.4",
+ "@react-navigation/native": "^7.2.2",
+ "expo": "~56.0.9",
+ "expo-build-properties": "~56.0.17",
+ "expo-constants": "~56.0.17",
+ "expo-linking": "~56.0.13",
+ "expo-router": "~56.2.9",
+ "react": "19.2.3",
+ "react-native": "0.85.3",
+ "react-native-audio-api": "0.13.1",
+ "react-native-device-info": "^15.0.2",
+ "react-native-drawer-layout": "^4.2.2",
+ "react-native-executorch": "workspace:*",
+ "react-native-gesture-handler": "~2.31.1",
+ "react-native-reanimated": "4.4.0",
+ "react-native-safe-area-context": "~5.7.0",
+ "react-native-screens": "~4.25.2",
+ "react-native-svg": "15.15.4",
+ "react-native-worklets": "0.9.1"
+ },
+ "devDependencies": {
+ "@babel/core": "^7.29.0",
+ "@react-native/metro-config": "^0.86.0",
+ "@types/react": "~19.2.0",
+ "babel-preset-expo": "~56.0.14",
+ "react-native-svg-transformer": "^1.5.3",
+ "react-refresh": "^0.14.0"
+ },
+ "private": true
+}
diff --git a/apps/speech/theme.ts b/apps/speech/theme.ts
new file mode 100644
index 0000000000..765ed34ae1
--- /dev/null
+++ b/apps/speech/theme.ts
@@ -0,0 +1,76 @@
+import { StyleSheet } from 'react-native';
+
+export const ColorPalette = {
+ primary: '#001A72',
+ strongPrimary: '#020F3C',
+};
+
+export const theme = {
+ colors: {
+ primary: ColorPalette.primary,
+ strongPrimary: ColorPalette.strongPrimary,
+ secondary: '#ffffff',
+ accent: '#1a73e8',
+ background: '#f5f5f5',
+ cardBackground: '#ffffff',
+ placeholderBackground: '#eaeaea',
+ border: '#ccc',
+ lightBorder: '#e9ecef',
+ errorBackground: '#ffe3e3',
+ errorText: '#d63031',
+ textPrimary: '#ffffff',
+ textSecondary: '#000000',
+ textMuted: '#666666',
+ textPlaceholder: '#868e96',
+ },
+ radius: {
+ small: 8,
+ medium: 12,
+ large: 16,
+ },
+ spacing: {
+ small: 8,
+ medium: 12,
+ large: 16,
+ },
+ typography: {
+ title: {
+ fontSize: 22,
+ fontWeight: '700' as const,
+ },
+ body: {
+ fontSize: 14,
+ color: '#333333',
+ },
+ },
+};
+
+export const commonStyles = StyleSheet.create({
+ container: {
+ flex: 1,
+ backgroundColor: theme.colors.background,
+ },
+ contentContainer: {
+ padding: theme.spacing.large,
+ alignItems: 'center',
+ },
+ title: {
+ fontSize: theme.typography.title.fontSize,
+ fontWeight: theme.typography.title.fontWeight,
+ color: theme.colors.strongPrimary,
+ marginBottom: theme.spacing.large,
+ },
+ buttonRow: {
+ flexDirection: 'row',
+ gap: theme.spacing.small,
+ width: '100%',
+ marginBottom: theme.spacing.large,
+ },
+ description: {
+ fontSize: theme.typography.body.fontSize,
+ color: theme.colors.textMuted,
+ textAlign: 'center',
+ marginBottom: theme.spacing.large,
+ paddingHorizontal: theme.spacing.medium,
+ },
+});
diff --git a/apps/speech/tsconfig.json b/apps/speech/tsconfig.json
new file mode 100644
index 0000000000..47026ce434
--- /dev/null
+++ b/apps/speech/tsconfig.json
@@ -0,0 +1,15 @@
+{
+ "extends": "../../tsconfig.json",
+ "compilerOptions": {
+ "strict": true,
+ "allowJs": true,
+ "module": "preserve",
+ "moduleDetection": "force",
+ "moduleResolution": "bundler",
+ "customConditions": ["react-native"],
+ "noEmit": true,
+ "paths": {
+ "react-native-executorch": ["../../packages/react-native-executorch/src"]
+ }
+ }
+}
diff --git a/packages/react-native-executorch/android/CMakeLists.txt b/packages/react-native-executorch/android/CMakeLists.txt
index a14fb9625b..433e4523d6 100644
--- a/packages/react-native-executorch/android/CMakeLists.txt
+++ b/packages/react-native-executorch/android/CMakeLists.txt
@@ -26,6 +26,7 @@ find_package(ReactAndroid REQUIRED CONFIG)
file(GLOB CORE_SOURCES ${CPP_DIR}/core/*.cpp)
file(GLOB MATH_SOURCES ${CPP_DIR}/extensions/math/*.cpp)
file(GLOB NLP_SOURCES ${CPP_DIR}/extensions/nlp/*.cpp)
+file(GLOB SPEECH_SOURCES ${CPP_DIR}/extensions/speech/*.cpp)
file(GLOB OPENCV_SOURCES ${CPP_DIR}/extensions/cv/*.cpp)
set(RNE_SOURCES
@@ -33,6 +34,7 @@ set(RNE_SOURCES
${CORE_SOURCES}
${MATH_SOURCES}
${NLP_SOURCES}
+ ${SPEECH_SOURCES}
cpp-adapter.cpp
)
diff --git a/packages/react-native-executorch/cpp/RnExecutorch.cpp b/packages/react-native-executorch/cpp/RnExecutorch.cpp
index 064d20e808..df1bcb9d4d 100644
--- a/packages/react-native-executorch/cpp/RnExecutorch.cpp
+++ b/packages/react-native-executorch/cpp/RnExecutorch.cpp
@@ -3,6 +3,7 @@
#include "core/install.h"
#include "extensions/math/install.h"
#include "extensions/nlp/install.h"
+#include "extensions/speech/install.h"
#ifdef RNE_ENABLE_OPENCV
#include "extensions/cv/install.h"
@@ -20,6 +21,7 @@ void install(jsi::Runtime &jsiRuntime) {
#endif
rnexecutorch::extensions::math::install(jsiRuntime, module);
rnexecutorch::extensions::nlp::install(jsiRuntime, module);
+ rnexecutorch::extensions::speech::install(jsiRuntime, module);
jsiRuntime.global().setProperty(jsiRuntime, "__rnexecutorch_jsi__", std::move(module));
}
diff --git a/packages/react-native-executorch/cpp/extensions/speech/install.cpp b/packages/react-native-executorch/cpp/extensions/speech/install.cpp
new file mode 100644
index 0000000000..ae6675527e
--- /dev/null
+++ b/packages/react-native-executorch/cpp/extensions/speech/install.cpp
@@ -0,0 +1,14 @@
+#include "install.h"
+#include "operations.h"
+
+namespace rnexecutorch::extensions::speech {
+namespace jsi = facebook::jsi;
+
+void install(jsi::Runtime &rt, jsi::Object &module) {
+ jsi::Object speechModule(rt);
+
+ install_frameWaveform(rt, speechModule);
+
+ module.setProperty(rt, "speech", speechModule);
+}
+} // namespace rnexecutorch::extensions::speech
diff --git a/packages/react-native-executorch/cpp/extensions/speech/install.h b/packages/react-native-executorch/cpp/extensions/speech/install.h
new file mode 100644
index 0000000000..177d377dd8
--- /dev/null
+++ b/packages/react-native-executorch/cpp/extensions/speech/install.h
@@ -0,0 +1,7 @@
+#pragma once
+
+#include
+
+namespace rnexecutorch::extensions::speech {
+void install(facebook::jsi::Runtime &rt, facebook::jsi::Object &module);
+} // namespace rnexecutorch::extensions::speech
diff --git a/packages/react-native-executorch/cpp/extensions/speech/operations.cpp b/packages/react-native-executorch/cpp/extensions/speech/operations.cpp
new file mode 100644
index 0000000000..c3fd4d7c2f
--- /dev/null
+++ b/packages/react-native-executorch/cpp/extensions/speech/operations.cpp
@@ -0,0 +1,99 @@
+#include "operations.h"
+
+#include
+#include
+#include
+
+#include "core/tensor.h"
+#include "core/tensor_helpers.h"
+
+namespace rnexecutorch::extensions::speech {
+namespace jsi = facebook::jsi;
+namespace conversions = rnexecutorch::core::conversions;
+namespace tensor = rnexecutorch::core::tensor;
+using rnexecutorch::core::types::DType;
+
+// Slices a mono waveform into overlapping frames, applying per-frame
+// mean-removal, a pre-emphasis filter and a Hann window, writing each frame into
+// a zero-padded row of `dst`. Mirrors the reference FSMN-VAD feature extraction.
+// The whole per-frame inner loop dominates the VAD pipeline (~85% of a detect()
+// call on device), so it lives in native code per the extension guidelines.
+void install_frameWaveform(jsi::Runtime &rt, jsi::Object &module) {
+ const auto *name = "frameWaveform";
+ auto fnBody = [](jsi::Runtime &rt, const jsi::Value & /*thisVal*/, const jsi::Value *args, size_t count) -> jsi::Value {
+ if (count != 7) {
+ throw jsi::JSError(rt, "Usage: frameWaveform(waveform, hann, dst, startSample, numFrames, hopLength, preemphasis)");
+ }
+
+ auto waveform = tensor::fromJs(rt, "frameWaveform: waveform", args[0], DType::float32, std::nullopt);
+ auto hann = tensor::fromJs(rt, "frameWaveform: hann", args[1], DType::float32, std::nullopt);
+ auto dst = tensor::fromJs(rt, "frameWaveform: dst", args[2], DType::float32, std::nullopt);
+ auto startSample = conversions::asType(rt, "frameWaveform: startSample", args[3]);
+ auto numFrames = conversions::asType(rt, "frameWaveform: numFrames", args[4]);
+ auto hopLength = conversions::asType(rt, "frameWaveform: hopLength", args[5]);
+ auto preemphasis = static_cast(conversions::asType(rt, "frameWaveform: preemphasis", args[6]));
+
+ tensor::checkNotSameTensor(rt, "frameWaveform: waveform", waveform, "frameWaveform: dst", dst);
+ tensor::checkNotSameTensor(rt, "frameWaveform: hann", hann, "frameWaveform: dst", dst);
+ auto waveLock = tensor::tryLockShared(rt, "frameWaveform: waveform", waveform);
+ auto hannLock = tensor::tryLockShared(rt, "frameWaveform: hann", hann);
+ auto dstLock = tensor::tryLockUnique(rt, "frameWaveform: dst", dst);
+
+ if (dst->shape_.size() != 2) {
+ throw jsi::JSError(rt, "frameWaveform: dst must be 2D [frames, fftLength]");
+ }
+ const auto frameLength = static_cast(hann->numel_);
+ const int64_t chunkFrames = dst->shape_[0];
+ const int64_t fftLength = dst->shape_[1];
+ if (frameLength > fftLength) {
+ throw jsi::JSError(rt, "frameWaveform: hann length exceeds dst fftLength");
+ }
+ if (numFrames < 0 || numFrames > chunkFrames) {
+ throw jsi::JSError(rt, "frameWaveform: numFrames out of dst frame capacity");
+ }
+
+ const auto waveLen = static_cast(waveform->numel_);
+ if (numFrames > 0) {
+ const int64_t lastSample = startSample + (numFrames - 1) * hopLength + frameLength - 1;
+ if (startSample < 0 || lastSample >= waveLen) {
+ throw jsi::JSError(rt, "frameWaveform: frame window out of waveform bounds");
+ }
+ }
+
+ const int64_t leftPad = (fftLength - frameLength) / 2;
+ const auto *wave = reinterpret_cast(waveform->data_.get());
+ const auto *win = reinterpret_cast(hann->data_.get());
+ auto *out = reinterpret_cast(dst->data_.get());
+
+ // Zero the full destination so both the intra-frame padding around each
+ // window and any trailing padding rows (numFrames < chunkFrames) are 0.
+ std::fill(out, out + dst->numel_, 0.0f);
+
+ for (int64_t f = 0; f < numFrames; ++f) {
+ float *base = out + f * fftLength + leftPad;
+ const float *start = wave + startSample + f * hopLength;
+
+ // Pass 1: mean over the raw window.
+ float sum = 0.0f;
+ for (int64_t j = 0; j < frameLength; ++j) {
+ sum += start[j];
+ }
+ const float mean = sum / static_cast(frameLength);
+
+ // Pass 2: fused mean-removal + pre-emphasis + Hann. Pre-emphasis of a
+ // mean-subtracted signal, `(raw[j]-mean) - c*(raw[j-1]-mean)`, equals
+ // `raw[j] - c*raw[j-1] - mean*(1-c)`, so every output reads only raw
+ // input samples — no serial dependency, and the loop vectorizes.
+ const float meanBias = mean * (1.0f - preemphasis);
+ base[0] = (start[0] - mean) * win[0];
+ for (int64_t j = 1; j < frameLength; ++j) {
+ base[j] = (start[j] - preemphasis * start[j - 1] - meanBias) * win[j];
+ }
+ }
+
+ return jsi::Value(rt, args[2]);
+ };
+
+ module.setProperty(rt, name, jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, name), 7, fnBody));
+}
+} // namespace rnexecutorch::extensions::speech
diff --git a/packages/react-native-executorch/cpp/extensions/speech/operations.h b/packages/react-native-executorch/cpp/extensions/speech/operations.h
new file mode 100644
index 0000000000..dbe5986fcc
--- /dev/null
+++ b/packages/react-native-executorch/cpp/extensions/speech/operations.h
@@ -0,0 +1,7 @@
+#pragma once
+
+#include
+
+namespace rnexecutorch::extensions::speech {
+void install_frameWaveform(facebook::jsi::Runtime &rt, facebook::jsi::Object &module);
+} // namespace rnexecutorch::extensions::speech
diff --git a/packages/react-native-executorch/src/extensions/speech/index.ts b/packages/react-native-executorch/src/extensions/speech/index.ts
new file mode 100644
index 0000000000..270d75e5c1
--- /dev/null
+++ b/packages/react-native-executorch/src/extensions/speech/index.ts
@@ -0,0 +1,2 @@
+export * from './tasks/vad';
+export * from './vadStreamer';
diff --git a/packages/react-native-executorch/src/extensions/speech/ops.ts b/packages/react-native-executorch/src/extensions/speech/ops.ts
new file mode 100644
index 0000000000..6269f399fc
--- /dev/null
+++ b/packages/react-native-executorch/src/extensions/speech/ops.ts
@@ -0,0 +1,40 @@
+import { rnexecutorchJsi } from '../../native/bridge';
+import { type Tensor } from '../../core/tensor';
+
+/**
+ * Native framing op: slices a mono `waveform` into `numFrames` overlapping
+ * frames starting at sample `startSample`, applying per-frame mean-removal, a
+ * pre-emphasis filter and the `hann` window, and writing each frame into a
+ * zero-padded row of `dst` (shape `[frames, fftLength]`). `dst` is fully zeroed
+ * first, so rows beyond `numFrames` stay zero (padding). The frame length is
+ * taken from `hann`'s length and the padded width from `dst`'s last dimension.
+ * @category Typescript API
+ * @param waveform Input audio samples, shape `[length]`.
+ * @param hann Precomputed Hann window, shape `[frameLength]`.
+ * @param dst Pre-allocated destination, shape `[frames, fftLength]`.
+ * @param startSample Index of the first sample of the first frame.
+ * @param numFrames Number of frames to write (must be `<= dst.shape[0]`).
+ * @param hopLength Samples between consecutive frames.
+ * @param preemphasis Pre-emphasis filter coefficient.
+ * @returns The `dst` tensor, for convenience.
+ */
+export function frameWaveform(
+ waveform: Tensor,
+ hann: Tensor,
+ dst: Tensor,
+ startSample: number,
+ numFrames: number,
+ hopLength: number,
+ preemphasis: number
+): Tensor {
+ 'worklet';
+ return rnexecutorchJsi.speech.frameWaveform(
+ waveform,
+ hann,
+ dst,
+ startSample,
+ numFrames,
+ hopLength,
+ preemphasis
+ );
+}
diff --git a/packages/react-native-executorch/src/extensions/speech/tasks/vad.ts b/packages/react-native-executorch/src/extensions/speech/tasks/vad.ts
new file mode 100644
index 0000000000..ad5d84fd87
--- /dev/null
+++ b/packages/react-native-executorch/src/extensions/speech/tasks/vad.ts
@@ -0,0 +1,309 @@
+import type { WorkletRuntime } from 'react-native-worklets';
+
+import { tensor } from '../../../core/tensor';
+import { loadModel } from '../../../core/model';
+import { validateModelSchema, SymbolicTensor } from '../../../core/modelSchema';
+import { wrapAsync } from '../../../core/runtime';
+import { frameWaveform } from '../ops';
+
+/**
+ * Feature-extraction geometry describing how a raw waveform is turned into the
+ * per-frame input a VAD model expects. These values are model-specific — they
+ * must match how the `.pte` was trained — and are supplied by the model config
+ * (see the `models` registry), not hardcoded in the pipeline.
+ * @category Types
+ * @property {number} sampleRate - Expected input sample rate in Hz (e.g. `16000`).
+ * @property {number} frameLength - Analysis window length in samples (e.g. `400`, 25 ms).
+ * @property {number} hopLength - Samples between consecutive windows (e.g. `160`, 10 ms).
+ * @property {number} fftLength - Zero-padded window length fed to the model (e.g. `512`).
+ * @property {number} preemphasis - Pre-emphasis filter coefficient (e.g. `0.97`).
+ * @property {number} minFrames - Minimum frames the model accepts per forward pass (e.g. `100`).
+ */
+export type VADFeatureConfig = {
+ readonly sampleRate: number;
+ readonly frameLength: number;
+ readonly hopLength: number;
+ readonly fftLength: number;
+ readonly preemphasis: number;
+ readonly minFrames: number;
+};
+
+/**
+ * Tunable thresholds controlling how per-frame speech probabilities are turned
+ * into speech {@link Segment}s.
+ * @category Types
+ * @property {number} [speechThreshold] - Minimum speech probability (0-1) for a
+ * frame to count as speech. Defaults to `0.6`.
+ * @property {number} [minSpeechDurationMs] - Minimum duration a region must stay
+ * above the threshold to open a segment. Defaults to `250`.
+ * @property {number} [minSilenceDurationMs] - Minimum duration below the
+ * threshold required to close a segment. Defaults to `100`.
+ * @property {number} [speechPadMs] - Padding added to both ends of every
+ * detected segment. Defaults to `30`.
+ * @property {number} [mergeGapMs] - Segments closer than this gap are merged
+ * into one. Defaults to `0`.
+ */
+export type VADOptions = {
+ readonly speechThreshold?: number;
+ readonly minSpeechDurationMs?: number;
+ readonly minSilenceDurationMs?: number;
+ readonly speechPadMs?: number;
+ readonly mergeGapMs?: number;
+};
+
+/**
+ * Model configuration required to instantiate a VAD task runner.
+ * @category Types
+ * @property {string} modelPath - Local path or remote URL of the `.pte` model.
+ * @property {VADFeatureConfig} featureConfig - Model-specific feature-extraction
+ * geometry.
+ * @property {VADOptions} [defaultOptions] - Detection thresholds tuned for this
+ * model; overridable per `detect` call. Falls back to the library defaults.
+ */
+export type VADModel = {
+ readonly modelPath: string;
+ readonly featureConfig: VADFeatureConfig;
+ readonly defaultOptions?: VADOptions;
+};
+
+/**
+ * A detected speech region, with start and end expressed in seconds.
+ * @category Types
+ */
+export type Segment = {
+ readonly start: number;
+ readonly end: number;
+};
+
+// Resolved options with defaults applied. Kept separate so `detectWorklet` can
+// read every field unconditionally.
+type ResolvedOptions = Required;
+
+// Library fallback thresholds (Silero-style). A model may override any of these
+// via `VADModel.defaultOptions`, and callers via the `detect` options argument.
+const DEFAULT_OPTIONS: ResolvedOptions = {
+ speechThreshold: 0.6,
+ minSpeechDurationMs: 250,
+ minSilenceDurationMs: 100,
+ speechPadMs: 30,
+ mergeGapMs: 0,
+};
+
+// A speech region measured in raw sample indices (internal to postprocessing).
+type SampleSegment = { start: number; end: number };
+
+// Periodic Hann window used to reduce spectral leakage on each frame. Ported
+// from `dsp::hannWindow` (periodic definition, divides by `size`).
+function hannWindow(size: number): Float32Array {
+ 'worklet';
+ const window = new Float32Array(size);
+ for (let i = 0; i < size; i++) {
+ window[i] = 0.5 * (1 - Math.cos((2 * Math.PI * i) / size));
+ }
+ return window;
+}
+
+// Converts per-frame non-speech probabilities into padded speech segments in
+// sample units. Mirrors `VoiceActivityDetection::postprocess` (excluding the
+// final merge step). `scores[i]` holds the non-speech probability of frame `i`,
+// so the speech probability is `1 - scores[i]`.
+function scoresToSegments(
+ scores: Float32Array,
+ opts: ResolvedOptions,
+ hopLength: number,
+ hopLengthMs: number
+): SampleSegment[] {
+ 'worklet';
+ const threshold = opts.speechThreshold;
+ const minSpeechHops = Math.floor(opts.minSpeechDurationMs / hopLengthMs);
+ const minSilenceHops = Math.floor(opts.minSilenceDurationMs / hopLengthMs);
+ const speechPadHops = Math.floor(opts.speechPadMs / hopLengthMs);
+
+ const segments: SampleSegment[] = [];
+ let triggered = false;
+ let startSegment = -1;
+ let potentialStart = -1;
+ let potentialEnd = -1;
+
+ for (let i = 0; i < scores.length; i++) {
+ const score = 1 - scores[i]!;
+ if (!triggered) {
+ if (score >= threshold) {
+ if (potentialStart === -1) potentialStart = i;
+ else if (i - potentialStart >= minSpeechHops) {
+ triggered = true;
+ startSegment = potentialStart;
+ potentialStart = -1;
+ }
+ } else {
+ potentialStart = -1;
+ }
+ } else if (score < threshold) {
+ if (potentialEnd === -1) potentialEnd = i;
+ else if (i - potentialEnd >= minSilenceHops) {
+ triggered = false;
+ segments.push({ start: startSegment, end: potentialEnd });
+ potentialEnd = -1;
+ }
+ } else {
+ potentialEnd = -1;
+ }
+ }
+ if (triggered) segments.push({ start: startSegment, end: scores.length });
+
+ for (const segment of segments) {
+ segment.start = (segment.start > speechPadHops ? segment.start - speechPadHops : 0) * hopLength;
+ segment.end = Math.min(segment.end + speechPadHops, scores.length) * hopLength;
+ }
+ return segments;
+}
+
+// Merges adjacent segments separated by a gap of at most `maxMergeGap` samples.
+// Mirrors `utils::mergeSegments`.
+function mergeSegments(segments: SampleSegment[], maxMergeGap: number): SampleSegment[] {
+ 'worklet';
+ if (segments.length === 0) return segments;
+
+ const merged: SampleSegment[] = [{ start: segments[0]!.start, end: segments[0]!.end }];
+ for (let i = 1; i < segments.length; i++) {
+ const last = merged[merged.length - 1]!;
+ const current = segments[i]!;
+ if (current.start < last.end || current.start - last.end <= maxMergeGap) {
+ last.end = Math.max(last.end, current.end);
+ } else {
+ merged.push({ start: current.start, end: current.end });
+ }
+ }
+ return merged;
+}
+
+/**
+ * Creates a Voice Activity Detection runner for executing local VAD models.
+ *
+ * It loads the model, validates its input/output signature, pre-allocates the
+ * static output tensor and registers a disposal hook. The whole pipeline —
+ * feature extraction, chunked inference and segment postprocessing — runs in
+ * TypeScript on top of the core `model.execute` primitive, parameterized by the
+ * model's {@link VADFeatureConfig}.
+ *
+ * The model exposes a dynamic frame dimension: each forward pass accepts
+ * `[frames, fftLength]` (up to the model-declared maximum) and returns per-frame
+ * class probabilities. Long inputs are split into chunks; a short final chunk is
+ * zero-padded up to the model minimum and its padding scores are discarded.
+ * @category Typescript API
+ * @param config VAD task configuration containing the model path and feature
+ * config.
+ * @param runtime Optional worklet runtime thread on which to run the model
+ * execution.
+ * @returns A promise resolving to an object containing detection and disposal
+ * controls.
+ */
+export async function createVAD(
+ config: VADModel,
+ runtime?: WorkletRuntime
+): Promise<{
+ /**
+ * Releases all allocated native resources.
+ */
+ dispose: () => void;
+ /**
+ * Asynchronously detects speech segments within a mono waveform sampled at the
+ * model's configured sample rate.
+ * @param waveform The input audio samples.
+ * @param options Optional per-call overrides of the detection thresholds.
+ * @returns A promise resolving to the detected speech segments, in seconds.
+ */
+ detect: (waveform: Float32Array, options?: VADOptions) => Promise;
+ /**
+ * Synchronous version of {@link detect} to be executed directly on the caller
+ * or worklet thread.
+ */
+ detectWorklet: (waveform: Float32Array, options?: VADOptions) => Segment[];
+}> {
+ const { modelPath, featureConfig: fc } = config;
+ const samplesPerMs = fc.sampleRate / 1000;
+ const hopLengthMs = fc.hopLength / samplesPerMs;
+ const modelDefaults: ResolvedOptions = { ...DEFAULT_OPTIONS, ...config.defaultOptions };
+
+ const model = await wrapAsync(loadModel, runtime)(modelPath);
+
+ // Input: [frames, fftLength] with a dynamic frame count. Output: per-frame
+ // class probabilities, either [1, frames, classes] or [frames, classes]. Class
+ // 0 is the non-speech class.
+ const meta = validateModelSchema(
+ model,
+ 'forward',
+ [SymbolicTensor('float32', ['N', fc.fftLength])],
+ [SymbolicTensor('float32', [1, 'F', 'C'], ['F', 'C'])]
+ );
+ const maxFrames = meta.inputTensorMeta[0]!.shape[0]!;
+ const outShape = meta.outputTensorMeta[0]!.shape;
+ const numClass = outShape[outShape.length - 1]!;
+
+ // The output tensor is validated against the declared shape exactly, so it is
+ // pre-allocated once at that shape. Its frame capacity caps the chunk size so
+ // a full chunk's output can never overflow it. The Hann window is uploaded
+ // once and reused by the native framing op across every call.
+ const tensors = [
+ tensor('float32', outShape),
+ tensor('float32', [fc.frameLength], hannWindow(fc.frameLength)),
+ ] as const;
+ const [tOutput, tHann] = tensors;
+ const outBuffer = new Float32Array(tOutput.numel);
+ const chunkCapacity = Math.min(maxFrames, Math.floor(tOutput.numel / numClass));
+
+ const dispose = () => {
+ tensors.forEach((t) => t.dispose());
+ model.dispose();
+ };
+
+ const detectWorklet = (waveform: Float32Array, options?: VADOptions): Segment[] => {
+ 'worklet';
+ if (waveform.length < fc.frameLength) return [];
+
+ const opts: ResolvedOptions = { ...modelDefaults, ...options };
+ const numFrames = Math.floor((waveform.length - fc.frameLength) / fc.hopLength);
+ if (numFrames <= 0) return [];
+
+ const scores = new Float32Array(numFrames);
+ // Upload the waveform once; the native op frames slices of it per chunk.
+ const tWaveform = tensor('float32', [waveform.length], waveform);
+ try {
+ let offset = 0;
+ while (offset < numFrames) {
+ const realFrames = Math.min(numFrames - offset, chunkCapacity);
+ const chunkFrames = Math.max(realFrames, fc.minFrames);
+ const tInput = tensor('float32', [chunkFrames, fc.fftLength]);
+ try {
+ frameWaveform(
+ tWaveform,
+ tHann,
+ tInput,
+ offset * fc.hopLength,
+ realFrames,
+ fc.hopLength,
+ fc.preemphasis
+ );
+ model.execute('forward', [tInput], [tOutput]);
+ tOutput.getData(outBuffer);
+ for (let i = 0; i < realFrames; i++) {
+ scores[offset + i] = outBuffer[i * numClass]!;
+ }
+ } finally {
+ tInput.dispose();
+ }
+ offset += realFrames;
+ }
+ } finally {
+ tWaveform.dispose();
+ }
+
+ const raw = scoresToSegments(scores, opts, fc.hopLength, hopLengthMs);
+ const merged = mergeSegments(raw, opts.mergeGapMs * samplesPerMs);
+ return merged.map((s) => ({ start: s.start / fc.sampleRate, end: s.end / fc.sampleRate }));
+ };
+
+ const detect = wrapAsync(detectWorklet, runtime);
+
+ return { detect, detectWorklet, dispose };
+}
diff --git a/packages/react-native-executorch/src/extensions/speech/vadStreamer.ts b/packages/react-native-executorch/src/extensions/speech/vadStreamer.ts
new file mode 100644
index 0000000000..35eff420b6
--- /dev/null
+++ b/packages/react-native-executorch/src/extensions/speech/vadStreamer.ts
@@ -0,0 +1,139 @@
+import type { Segment } from './tasks/vad';
+
+// Detection runs over a bounded sliding window of the most recent audio rather
+// than the whole growing session. The streaming decision only depends on recent
+// context (whether speech is ongoing at the buffer end), and model cost scales
+// with window length — so a bounded window keeps per-tick latency flat and low
+// (~3x cheaper than a 10s buffer on device) instead of growing without limit.
+// 2.5s is well above FSMN's receptive field (~200ms) and the min-speech-duration
+// (250ms), so detection quality is unaffected. Expressed in seconds and scaled
+// by the stream's sample rate.
+const DETECTION_WINDOW_SECONDS = 2.5;
+
+/**
+ * Options controlling the streaming detection loop.
+ * @category Types
+ * @property {number} [timeout] - Delay in milliseconds between successive
+ * inferences over the accumulated buffer. Defaults to `100`.
+ * @property {number} [detectionMargin] - How recent (in milliseconds) the last
+ * detected speech segment must be, relative to the end of the buffer, for speech
+ * to still be considered ongoing. Defaults to `100`.
+ * @property {() => void | Promise} [onSpeechBegin] - Called when speech
+ * starts (silence to speech transition).
+ * @property {() => void | Promise} [onSpeechEnd] - Called when speech ends
+ * (speech to silence transition).
+ */
+export type VADStreamOptions = {
+ readonly timeout?: number;
+ readonly detectionMargin?: number;
+ readonly onSpeechBegin?: () => void | Promise;
+ readonly onSpeechEnd?: () => void | Promise;
+};
+
+/**
+ * A live streaming VAD session driving speech begin/end callbacks.
+ * @category Types
+ */
+export type VADStreamer = {
+ /** Starts the detection loop. Resolves once the session is stopped. */
+ start: () => Promise;
+ /** Appends an audio chunk to the streaming buffer. */
+ insert: (chunk: Float32Array) => void;
+ /** Stops the detection loop and clears the buffer. */
+ stop: () => void;
+};
+
+/**
+ * Creates a pure (React-independent) streaming VAD session on top of a `detect`
+ * function (typically `createVAD(...).detect`).
+ *
+ * It accumulates inserted audio into an internal buffer and, every `timeout`
+ * milliseconds, runs detection over the buffer and fires `onSpeechBegin` /
+ * `onSpeechEnd` on transitions. Ticks never overlap: the next tick is scheduled
+ * only after the previous detection resolves. Ported from
+ * `VoiceActivityDetection::stream`.
+ * @category Typescript API
+ * @param detect The async detection function to run over the buffer.
+ * @param sampleRate Sample rate (Hz) of the inserted audio, used to convert the
+ * buffer length to seconds and to bound the buffer.
+ * @param options Streaming configuration and callbacks.
+ * @returns A {@link VADStreamer} controlling the session lifecycle.
+ */
+export function createVadStreamer(
+ detect: (waveform: Float32Array) => Promise,
+ sampleRate: number,
+ options?: VADStreamOptions
+): VADStreamer {
+ const timeout = options?.timeout ?? 100;
+ const detectionMargin = options?.detectionMargin ?? 100;
+ const windowSamples = DETECTION_WINDOW_SECONDS * sampleRate;
+
+ let buffer = new Float32Array(0);
+ let running = false;
+ let isSpeaking = false;
+ let timer: ReturnType | null = null;
+ let resolveFinished: (() => void) | null = null;
+
+ const insert = (chunk: Float32Array) => {
+ // `buffer` is replaced (never mutated in place), so any snapshot handed to a
+ // running detection stays valid. Older audio beyond the detection window is
+ // dropped so the buffer — and each tick's model cost — stays bounded.
+ const next = new Float32Array(buffer.length + chunk.length);
+ next.set(buffer);
+ next.set(chunk, buffer.length);
+ buffer = next.length > windowSamples ? next.slice(next.length - windowSamples) : next;
+ };
+
+ const tick = async () => {
+ if (!running) return;
+
+ let speaking = false;
+ const snapshot = buffer;
+ try {
+ const segments = await detect(snapshot);
+ if (segments.length > 0) {
+ const bufferEndSec = snapshot.length / sampleRate;
+ const diffMs = (bufferEndSec - segments[segments.length - 1]!.end) * 1000;
+ speaking = diffMs <= detectionMargin;
+ }
+ } catch {
+ // Detection may throw if the model is disposed mid-stream; treat as
+ // non-speech and let the next tick (if any) recover.
+ }
+
+ if (!running) return;
+
+ if (speaking && !isSpeaking) {
+ isSpeaking = true;
+ await options?.onSpeechBegin?.();
+ } else if (!speaking && isSpeaking) {
+ isSpeaking = false;
+ await options?.onSpeechEnd?.();
+ }
+
+ if (running) timer = setTimeout(tick, timeout);
+ };
+
+ const start = () =>
+ new Promise((resolve) => {
+ if (running) {
+ resolve();
+ return;
+ }
+ running = true;
+ resolveFinished = resolve;
+ timer = setTimeout(tick, timeout);
+ });
+
+ const stop = () => {
+ running = false;
+ if (timer) clearTimeout(timer);
+ timer = null;
+ buffer = new Float32Array(0);
+ isSpeaking = false;
+ resolveFinished?.();
+ resolveFinished = null;
+ };
+
+ return { start, insert, stop };
+}
diff --git a/packages/react-native-executorch/src/hooks/useVAD.ts b/packages/react-native-executorch/src/hooks/useVAD.ts
new file mode 100644
index 0000000000..3f094c48e1
--- /dev/null
+++ b/packages/react-native-executorch/src/hooks/useVAD.ts
@@ -0,0 +1,94 @@
+import { useEffect, useRef } from 'react';
+
+import { useModel } from './useModel';
+import { useResourceDownload } from './useResourceDownload';
+import { createVAD, type VADModel } from '../extensions/speech/tasks/vad';
+import {
+ createVadStreamer,
+ type VADStreamOptions,
+ type VADStreamer,
+} from '../extensions/speech/vadStreamer';
+
+/**
+ * Input for {@link useVAD}'s `stream` method: begin/end callbacks plus streaming
+ * options.
+ * @category Types
+ */
+export type VADStreamInput = {
+ readonly onSpeechBegin?: () => void | Promise;
+ readonly onSpeechEnd?: () => void | Promise;
+ readonly options?: Pick;
+};
+
+/**
+ * React hook to load and run a Voice Activity Detection model.
+ *
+ * This hook manages downloading (if it's a remote URL) and loading the model
+ * file, compiling it, tracking download progress and compilation errors, and
+ * cleaning up native model memory when the component unmounts or configuration
+ * changes. It also owns the lifecycle of the streaming session created by
+ * `stream`.
+ * @category Hooks
+ * @param config The VAD model configuration.
+ * @param options Hook options.
+ * @param options.preventLoad If true, prevents downloading and compiling the
+ * model.
+ * @returns An object containing the model's loading state, error, download
+ * progress, one-shot detection functions, and streaming controls.
+ */
+export function useVAD(config: VADModel, options?: { preventLoad?: boolean }) {
+ const { localPath, downloadProgress, downloadError } = useResourceDownload(
+ config.modelPath,
+ options?.preventLoad
+ );
+ const { model, error } = useModel(
+ createVAD,
+ localPath ? { ...config, modelPath: localPath } : null,
+ [localPath]
+ );
+
+ const streamerRef = useRef(null);
+
+ // Tear down any active streaming session when the model changes or the
+ // component unmounts.
+ useEffect(
+ () => () => {
+ streamerRef.current?.stop();
+ streamerRef.current = null;
+ },
+ [model]
+ );
+
+ const stream = (input: VADStreamInput): Promise => {
+ if (!model?.detect) {
+ throw new Error('useVAD: model is not loaded yet');
+ }
+ streamerRef.current?.stop();
+ const streamer = createVadStreamer(model.detect, config.featureConfig.sampleRate, {
+ ...input.options,
+ onSpeechBegin: input.onSpeechBegin,
+ onSpeechEnd: input.onSpeechEnd,
+ });
+ streamerRef.current = streamer;
+ return streamer.start();
+ };
+
+ const streamInsert = (chunk: Float32Array) => streamerRef.current?.insert(chunk);
+
+ const streamStop = () => {
+ streamerRef.current?.stop();
+ streamerRef.current = null;
+ };
+
+ return {
+ isReady: !!model,
+ error: downloadError || error,
+ downloadProgress,
+ localPath,
+ detect: model?.detect,
+ detectWorklet: model?.detectWorklet,
+ stream,
+ streamInsert,
+ streamStop,
+ };
+}
diff --git a/packages/react-native-executorch/src/index.ts b/packages/react-native-executorch/src/index.ts
index 00557be78b..1eb30e1eca 100644
--- a/packages/react-native-executorch/src/index.ts
+++ b/packages/react-native-executorch/src/index.ts
@@ -6,6 +6,7 @@ export * from './hooks/useInstanceSegmenter';
export * from './hooks/useKeypointDetector';
export * from './hooks/useObjectDetector';
export * from './hooks/useTokenizer';
+export * from './hooks/useVAD';
export * from './hooks/useResourceDownload';
export * from './hooks/useModel';
@@ -21,6 +22,8 @@ export * from './extensions/cv/tasks/instanceSegmentation';
export * from './extensions/cv/tasks/keypointDetection';
export * from './extensions/cv/tasks/objectDetection';
export * from './extensions/nlp/tasks/tokenization';
+export * from './extensions/speech/tasks/vad';
+export * from './extensions/speech/vadStreamer';
// Core primitives — for library builders and power users
export { tensor } from './core/tensor';
@@ -44,6 +47,7 @@ export { defaultWorkletRuntime, wrapAsync } from './core/runtime';
export * as math from './extensions/math';
export * as cv from './extensions/cv';
export * as nlp from './extensions/nlp';
+export * as speech from './extensions/speech';
// Utils
export * from './utils';
diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts
index 3397288e23..4bb5dd2ed5 100644
--- a/packages/react-native-executorch/src/models.ts
+++ b/packages/react-native-executorch/src/models.ts
@@ -4,6 +4,7 @@ import type { StyleTransferModel } from './extensions/cv/tasks/styleTransfer';
import type { SemanticSegmentationModel } from './extensions/cv/tasks/semanticSegmentation';
import type { KeypointDetectorModel } from './extensions/cv/tasks/keypointDetection';
import type { InstanceSegmenterModel } from './extensions/cv/tasks/instanceSegmentation';
+import type { VADModel } from './extensions/speech/tasks/vad';
import {
IMAGENET_NORM,
IMAGENET1K_LABELS,
@@ -528,6 +529,24 @@ const YOLO26_XLARGE_SEG_640_XNNPACK_FP32: InstanceSegmenterModel<'xyxy', CocoCla
opts: YOLO26_SEG_OPTS,
};
+// =============================================================================
+// Voice Activity Detection
+// =============================================================================
+// FSMN-VAD feature extraction: 16 kHz mono, 25 ms window / 10 ms hop, windows
+// zero-padded to 512, 0.97 pre-emphasis, minimum 100 frames per forward pass.
+const FSMN_VAD_FEATURE_CONFIG = {
+ sampleRate: 16000,
+ frameLength: 400,
+ hopLength: 160,
+ fftLength: 512,
+ preemphasis: 0.97,
+ minFrames: 100,
+};
+const FSMN_VAD_XNNPACK_FP32: VADModel = {
+ modelPath: `${BASE_URL}-fsmn-vad/${NEXT_VERSION_TAG}/xnnpack/fsmn_vad_xnnpack_fp32.pte`,
+ featureConfig: FSMN_VAD_FEATURE_CONFIG,
+};
+
// =============================================================================
// Tokenizers
// =============================================================================
@@ -734,6 +753,12 @@ export const models = {
},
},
},
+ vad: {
+ FSMN_VAD: {
+ ...FSMN_VAD_XNNPACK_FP32,
+ XNNPACK_FP32: FSMN_VAD_XNNPACK_FP32,
+ },
+ },
tokenizer: {
ALL_MINILM_L6_V2: ALL_MINILM_L6_V2_TOKENIZER,
},
diff --git a/yarn.lock b/yarn.lock
index bc05759538..503786b0dc 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -12764,6 +12764,24 @@ __metadata:
languageName: node
linkType: hard
+"react-native-audio-api@npm:0.13.1":
+ version: 0.13.1
+ resolution: "react-native-audio-api@npm:0.13.1"
+ dependencies:
+ semver: "npm:^7.7.3"
+ peerDependencies:
+ react: "*"
+ react-native: "*"
+ react-native-worklets: ">= 0.6.0"
+ peerDependenciesMeta:
+ react-native-worklets:
+ optional: true
+ bin:
+ setup-rn-audio-api-web: scripts/setup-rn-audio-api-web.js
+ checksum: 10/30675d28706f4cf03beb925120a6745d858f7dead4bc0df2dd734d6d0feaafc14924bbb1dd261c4f158a4501743f8abe72e029aadac8759b442480c01850ca88
+ languageName: node
+ linkType: hard
+
"react-native-builder-bob@npm:^0.40.18":
version: 0.40.18
resolution: "react-native-builder-bob@npm:0.40.18"
@@ -12796,6 +12814,15 @@ __metadata:
languageName: node
linkType: hard
+"react-native-device-info@npm:^15.0.2":
+ version: 15.0.2
+ resolution: "react-native-device-info@npm:15.0.2"
+ peerDependencies:
+ react-native: "*"
+ checksum: 10/9f945a16bd1aebefc60868e22552375bc630eb98d5505d8cdc847d7841a226540408e7ea174cbb4eb5bfec17235eaca731d154082f64e4935e2a4d2e0244b39b
+ languageName: node
+ linkType: hard
+
"react-native-drawer-layout@npm:^4.2.2, react-native-drawer-layout@npm:^4.2.5":
version: 4.2.5
resolution: "react-native-drawer-layout@npm:4.2.5"
@@ -14036,6 +14063,38 @@ __metadata:
languageName: node
linkType: hard
+"speech@workspace:apps/speech":
+ version: 0.0.0-use.local
+ resolution: "speech@workspace:apps/speech"
+ dependencies:
+ "@babel/core": "npm:^7.29.0"
+ "@react-native/metro-config": "npm:^0.86.0"
+ "@react-navigation/drawer": "npm:^7.9.4"
+ "@react-navigation/native": "npm:^7.2.2"
+ "@types/react": "npm:~19.2.0"
+ babel-preset-expo: "npm:~56.0.14"
+ expo: "npm:~56.0.9"
+ expo-build-properties: "npm:~56.0.17"
+ expo-constants: "npm:~56.0.17"
+ expo-linking: "npm:~56.0.13"
+ expo-router: "npm:~56.2.9"
+ react: "npm:19.2.3"
+ react-native: "npm:0.85.3"
+ react-native-audio-api: "npm:0.13.1"
+ react-native-device-info: "npm:^15.0.2"
+ react-native-drawer-layout: "npm:^4.2.2"
+ react-native-executorch: "workspace:*"
+ react-native-gesture-handler: "npm:~2.31.1"
+ react-native-reanimated: "npm:4.4.0"
+ react-native-safe-area-context: "npm:~5.7.0"
+ react-native-screens: "npm:~4.25.2"
+ react-native-svg: "npm:15.15.4"
+ react-native-svg-transformer: "npm:^1.5.3"
+ react-native-worklets: "npm:0.9.1"
+ react-refresh: "npm:^0.14.0"
+ languageName: unknown
+ linkType: soft
+
"split-on-first@npm:^1.0.0":
version: 1.1.0
resolution: "split-on-first@npm:1.1.0"